From 6a07214262eceee15aea2155a649d3e6b7399385 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Sun, 7 Mar 2021 22:13:52 +0000
Subject: [PATCH 01/16] attempt to get otel tracing to work with tempo

in particular try to fetch TraceIds while inside a span.
this might be a dumb idea, but need the id somehow...
---
 Cargo.lock       | 450 ++++++++++++++++++++++++++++++++++-------------
 Cargo.toml       |   6 +-
 src/lib.rs       |  20 +--
 src/main.rs      |   6 +-
 src/manager.rs   |  13 +-
 src/telemetry.rs |  43 +++++
 6 files changed, 387 insertions(+), 151 deletions(-)
 create mode 100644 src/telemetry.rs

diff --git a/Cargo.lock b/Cargo.lock
index bd98933..e883a93 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,7 +17,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e"
 dependencies = [
  "bitflags",
- "bytes",
+ "bytes 1.0.1",
  "futures-core",
  "futures-sink",
  "log",
@@ -41,7 +41,7 @@ dependencies = [
  "base64",
  "bitflags",
  "brotli2",
- "bytes",
+ "bytes 1.0.1",
  "bytestring",
  "cookie",
  "derive_more",
@@ -183,7 +183,7 @@ dependencies = [
  "actix-web-codegen",
  "ahash",
  "awc",
- "bytes",
+ "bytes 1.0.1",
  "derive_more",
  "either",
  "encoding_rs",
@@ -213,15 +213,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "addr2line"
-version = "0.14.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
-dependencies = [
- "gimli",
-]
-
 [[package]]
 name = "adler"
 version = "0.2.3"
@@ -257,12 +248,50 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
+
 [[package]]
 name = "array_tool"
 version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
 
+[[package]]
+name = "async-stream"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -291,7 +320,7 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "base64",
- "bytes",
+ "bytes 1.0.1",
  "cfg-if",
  "derive_more",
  "futures-core",
@@ -304,20 +333,6 @@ dependencies = [
  "serde_urlencoded",
 ]
 
-[[package]]
-name = "backtrace"
-version = "0.3.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
-dependencies = [
- "addr2line",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
 [[package]]
 name = "base-x"
 version = "0.2.8"
@@ -377,6 +392,12 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
+[[package]]
+name = "bytes"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
+
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -389,7 +410,7 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
 ]
 
 [[package]]
@@ -418,33 +439,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "color-eyre"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b29030875fd8376e4a28ef497790d5b4a7843d8d1396bf08ce46f5eec562c5c"
-dependencies = [
- "backtrace",
- "color-spantrace",
- "eyre",
- "indenter",
- "once_cell",
- "owo-colors",
- "tracing-error",
-]
-
-[[package]]
-name = "color-spantrace"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6eee477a4a8a72f4addd4de416eb56d54bc307b284d6601bafdee1f4ea462d1"
-dependencies = [
- "once_cell",
- "owo-colors",
- "tracing-core",
- "tracing-error",
-]
-
 [[package]]
 name = "const_fn"
 version = "0.4.5"
@@ -458,20 +452,22 @@ dependencies = [
  "actix-rt",
  "actix-web",
  "chrono",
- "color-eyre",
  "futures",
  "k8s-openapi",
  "kube",
  "kube-runtime",
+ "opentelemetry",
+ "opentelemetry-otlp",
  "prometheus",
  "schemars",
  "serde",
  "serde_json",
  "serde_yaml",
- "snafu",
+ "thiserror",
  "tokio",
  "tracing",
  "tracing-futures",
+ "tracing-opentelemetry",
  "tracing-subscriber",
 ]
 
@@ -667,14 +663,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "eyre"
-version = "0.6.5"
+name = "fixedbitset"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b"
-dependencies = [
- "indenter",
- "once_cell",
-]
+checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
 [[package]]
 name = "flate2"
@@ -832,19 +824,13 @@ dependencies = [
  "wasi",
 ]
 
-[[package]]
-name = "gimli"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
-
 [[package]]
 name = "h2"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -864,6 +850,15 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
+[[package]]
+name = "heck"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -879,7 +874,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "fnv",
  "itoa",
 ]
@@ -890,7 +885,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "http",
 ]
 
@@ -921,10 +916,11 @@ version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "futures-channel",
  "futures-core",
  "futures-util",
+ "h2",
  "http",
  "http-body",
  "httparse",
@@ -956,7 +952,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "hyper",
  "native-tls",
  "tokio",
@@ -980,12 +976,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "indenter"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4d5eb2e114fec2b7fe0fadc22888ad2658789bb7acac4dbee9cf8389f971ec8"
-
 [[package]]
 name = "indexmap"
 version = "1.6.1"
@@ -1005,12 +995,39 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
 
+[[package]]
+name = "js-sys"
+version = "0.3.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "jsonpath_lib"
 version = "0.2.6"
@@ -1031,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bcc1f973542059e6d5a6d63de6a9539d0ec784f82b2327f3c1915d33200bc6a4"
 dependencies = [
  "base64",
- "bytes",
+ "bytes 1.0.1",
  "chrono",
  "serde",
  "serde-value",
@@ -1046,7 +1063,7 @@ checksum = "8d47a55e9f881dc5027dcaf026670fa24b41f67926ab6517e2155488fe9c012a"
 dependencies = [
  "Inflector",
  "base64",
- "bytes",
+ "bytes 1.0.1",
  "chrono",
  "dirs-next",
  "either",
@@ -1209,6 +1226,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "multimap"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333"
+
 [[package]]
 name = "native-tls"
 version = "0.2.7"
@@ -1265,12 +1288,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "object"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
-
 [[package]]
 name = "once_cell"
 version = "1.5.2"
@@ -1316,6 +1333,38 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "514d24875c140ed269eecc2d1b56d7b71b573716922a763c317fb1b1b4b58f15"
+dependencies = [
+ "async-trait",
+ "futures",
+ "js-sys",
+ "lazy_static",
+ "percent-encoding",
+ "pin-project 1.0.5",
+ "rand",
+ "thiserror",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff889e68a02000126fa635a311abc329ce128c576b7dddc4cd3270f3c240afd0"
+dependencies = [
+ "async-trait",
+ "futures",
+ "opentelemetry",
+ "prost 0.7.0",
+ "thiserror",
+ "tokio",
+ "tonic",
+ "tonic-build",
+]
+
 [[package]]
 name = "ordered-float"
 version = "2.1.1"
@@ -1325,12 +1374,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "owo-colors"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55"
-
 [[package]]
 name = "parking_lot"
 version = "0.11.1"
@@ -1373,6 +1416,16 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "petgraph"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.27"
@@ -1473,6 +1526,80 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "prost"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212"
+dependencies = [
+ "bytes 0.5.6",
+ "prost-derive 0.6.1",
+]
+
+[[package]]
+name = "prost"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2"
+dependencies = [
+ "bytes 1.0.1",
+ "prost-derive 0.7.0",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26"
+dependencies = [
+ "bytes 0.5.6",
+ "heck",
+ "itertools 0.8.2",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.6.1",
+ "prost-types",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72"
+dependencies = [
+ "anyhow",
+ "itertools 0.8.2",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4"
+dependencies = [
+ "anyhow",
+ "itertools 0.9.0",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa"
+dependencies = [
+ "bytes 0.5.6",
+ "prost 0.6.1",
+]
+
 [[package]]
 name = "protobuf"
 version = "2.22.0"
@@ -1596,12 +1723,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "rustc-demangle"
-version = "0.1.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -1962,18 +2083,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2060,10 +2181,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a"
 dependencies = [
  "autocfg",
- "bytes",
+ "bytes 1.0.1",
  "libc",
  "memchr",
  "mio",
+ "num_cpus",
  "once_cell",
  "parking_lot",
  "pin-project-lite",
@@ -2103,13 +2225,24 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "futures-core",
  "futures-sink",
  "log",
@@ -2118,6 +2251,47 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ba8f479158947373b6df40cf48f4779bb25c99ca3c661bd95e0ab1963ad8b0e"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "base64",
+ "bytes 1.0.1",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "percent-encoding",
+ "pin-project 1.0.5",
+ "prost 0.7.0",
+ "prost-derive 0.7.0",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19970cf58f3acc820962be74c4021b8bbc8e8a1c4e3a02095d0aa60cde5f3633"
+dependencies = [
+ "proc-macro2",
+ "prost-build",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tower"
 version = "0.4.6"
@@ -2126,8 +2300,12 @@ checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap",
  "pin-project 1.0.5",
+ "rand",
+ "slab",
  "tokio",
+ "tokio-stream",
  "tokio-util",
  "tower-layer",
  "tower-service",
@@ -2179,16 +2357,6 @@ dependencies = [
  "lazy_static",
 ]
 
-[[package]]
-name = "tracing-error"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24"
-dependencies = [
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "tracing-futures"
 version = "0.2.5"
@@ -2210,6 +2378,19 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccdf13c28f1654fe806838f28c5b9cb23ca4c0eae71450daa489f50e523ceb1"
+dependencies = [
+ "opentelemetry",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-serde"
 version = "0.1.2"
@@ -2272,6 +2453,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"
@@ -2320,9 +2507,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.70"
+version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be"
+checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -2330,9 +2517,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.70"
+version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bc45447f0d4573f3d65720f636bbcc3dd6ce920ed704670118650bcd47764c7"
+checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8"
 dependencies = [
  "bumpalo",
  "lazy_static",
@@ -2345,9 +2532,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.70"
+version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b8853882eef39593ad4174dd26fc9865a64e84026d223f63bb2c42affcbba2c"
+checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2355,9 +2542,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.70"
+version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4133b5e7f2a531fa413b3a1695e925038a05a71cf67e87dafa295cb645a01385"
+checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2368,9 +2555,18 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.70"
+version = "0.2.71"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1"
+
+[[package]]
+name = "which"
+version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64"
+checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "winapi"
diff --git a/Cargo.toml b/Cargo.toml
index ed1fbc2..2982700 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,13 +31,15 @@ k8s-openapi = { version = "0.11.0", features = ["v1_19"], default-features=false
 serde = { version = "1.0.123", features = ["derive"] }
 serde_json = "1.0.64"
 chrono = { version = "0.4.19", features = ["serde"] }
-color-eyre = "0.5.10"
-snafu = "0.6.10"
+thiserror = "1.0.24"
 tracing = "0.1.25"
 tracing-subscriber = { version = "0.2.16", features = ["json"] }
 tracing-futures = "0.2.5"
 schemars = { version = "0.8.0", features = ["chrono"] }
 serde_yaml = "0.8.17"
+tracing-opentelemetry = "0.11.0"
+opentelemetry = { version = "0.12.0", features = ["trace"] }
+opentelemetry-otlp = { version = "0.5.0", features = ["async", "tokio"] }
 
 # local dev
 #kube = { path = "../kube-rs/kube", features = ["derive"] }
diff --git a/src/lib.rs b/src/lib.rs
index 7897188..75563e9 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,19 +1,14 @@
 #![warn(rust_2018_idioms)]
 #![allow(unused_imports)]
+use thiserror::Error;
 
-use snafu::{Backtrace, OptionExt, ResultExt, Snafu};
-#[derive(Debug, Snafu)]
+#[derive(Error, Debug)]
 pub enum Error {
-    #[snafu(display("Failed to patch Foo: {}", source))]
-    FooPatchFailed {
-        source: kube::Error,
-        backtrace: Backtrace,
-    },
+    #[error("Kube Api Error: {0}")]
+    KubeError(#[source] kube::Error),
 
-    SerializationFailed {
-        source: serde_json::Error,
-        backtrace: Backtrace,
-    },
+    #[error("SerializationError: {0}")]
+    SerializationError(#[source] serde_json::Error),
 }
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
@@ -23,3 +18,6 @@ pub use manager::Manager;
 
 /// Generated type, for crdgen
 pub use manager::Foo;
+
+/// Log and trace integrations
+pub mod telemetry;
diff --git a/src/main.rs b/src/main.rs
index e2fc278..eb89725 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -31,13 +31,9 @@ async fn index(c: Data<Manager>, _req: HttpRequest) -> impl Responder {
 
 #[actix_rt::main]
 async fn main() -> Result<()> {
-    tracing_subscriber::fmt()
-        .with_max_level(tracing::Level::INFO)
-        .json()
-        .init();
     let client = kube::Client::try_default().await.expect("create client");
     let (manager, drainer) = Manager::new(client).await;
-
+    telemetry::init().expect("init telemetry");
     let server = HttpServer::new(move || {
         App::new()
             .data(manager.clone())
diff --git a/src/manager.rs b/src/manager.rs
index d0750fc..031d3f6 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -1,4 +1,4 @@
-use crate::{Error, FooPatchFailed, Result, SerializationFailed};
+use crate::{Error, Result, telemetry};
 use chrono::prelude::*;
 use futures::{future::BoxFuture, FutureExt, StreamExt};
 use kube::{
@@ -11,10 +11,9 @@ use prometheus::{default_registry, proto::MetricFamily, register_int_counter, In
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use snafu::{Backtrace, OptionExt, ResultExt, Snafu};
 use std::sync::Arc;
 use tokio::{sync::RwLock, time::Duration};
-use tracing::{debug, error, info, instrument, trace, warn};
+use tracing::{debug, error, info, instrument, trace, warn, event, Level};
 
 /// Our Foo custom resource spec
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
@@ -44,11 +43,13 @@ struct Data {
 
 #[instrument(skip(ctx))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
+    let tid = telemetry::get_trace_id();
+
     let client = ctx.get_ref().client.clone();
     ctx.get_ref().state.write().await.last_event = Utc::now();
     let name = Meta::name(&foo);
     let ns = Meta::namespace(&foo).expect("foo is namespaced");
-    debug!("Reconcile Foo {}: {:?}", name, foo);
+    info!("Reconcile {} traceId={}", name, tid);
     let foos: Api<Foo> = Api::namespaced(client, &ns);
 
     let new_status = Patch::Apply(json!({
@@ -60,7 +61,7 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
         }
     }));
     let ps = PatchParams::apply("cntrlr").force();
-    let _o = foos.patch_status(&name, &ps, &new_status).await.context(FooPatchFailed)?;
+    let _o = foos.patch_status(&name, &ps, &new_status).await.map_err(Error::KubeError)?;
 
     ctx.get_ref().metrics.handled_events.inc();
 
@@ -70,7 +71,7 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
     })
 }
 fn error_policy(error: &Error, _ctx: Context<Data>) -> ReconcilerAction {
-    warn!("reconcile failed: {}", error);
+    warn!("reconcile failed: {:?}", error);
     ReconcilerAction {
         requeue_after: Some(Duration::from_secs(360)),
     }
diff --git a/src/telemetry.rs b/src/telemetry.rs
new file mode 100644
index 0000000..f276d33
--- /dev/null
+++ b/src/telemetry.rs
@@ -0,0 +1,43 @@
+use crate::{Error, Result};
+//use tracing_attributes::instrument;
+use tracing_subscriber::prelude::*;
+use tracing_subscriber::{Registry, fmt::format::FmtSpan};
+use tracing_subscriber::{fmt, EnvFilter};
+
+pub fn init() -> std::result::Result<(), Box<dyn std::error::Error>> {
+    let otlp_endpoint = std::env::var("OPENTELEMETRY_ENDPOINT_URL")
+        //.unwrap_or("http://grafana-agent-traces.monitoring.svc.cluster.local:55680")
+        .unwrap_or("http://0.0.0.0:55680".to_string());
+
+    let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
+        // NB: need to port-forward the service
+        // k port-forward -n monitoring service/grafana-agent-traces 55680:55680
+        .with_endpoint(&otlp_endpoint)
+        .install()?;
+
+    // Finish layers
+    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+    let logger = tracing_subscriber::fmt::layer();//.with_span_events(FmtSpan::ACTIVE);
+
+    let filter_layer = EnvFilter::try_from_default_env()
+        .or_else(|_| EnvFilter::try_new("info"))?;
+
+    // Register all subscribers
+    let subscriber = Registry::default()
+        .with(filter_layer)
+        .with(telemetry)
+        .with(logger);
+
+    tracing::subscriber::set_global_default(subscriber)?;
+    Ok(())
+}
+
+// attempt to get TraceId by span name via opentelemetry
+// feels a bit like a backdoor, but it's not passed through all the layers
+// ideally it would be exposed by tracing::Span
+pub fn get_trace_id() -> String {
+    use opentelemetry::trace::{SpanContext, TraceContextExt, Tracer};
+    opentelemetry::global::tracer("registry").in_span("reconcile", |cx| {
+        cx.span().span_context().trace_id().to_hex()
+    })
+}

From 9237cda8be928241c5472db30ca240ebb3d31acd Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Mon, 8 Mar 2021 00:24:49 +0000
Subject: [PATCH 02/16] save commands

---
 Makefile         | 5 ++++-
 src/telemetry.rs | 4 +---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 1a45e16..c7d2325 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,11 @@ install:
 	cargo run --bin crdgen > yaml/foo-crd.yaml
 	kubectl apply -f yaml/foo-crd.yaml
 
+forward-tempo:
+	kubectl port-forward -n monitoring service/grafana-agent-traces 55680:55680
+
 run:
-	cargo run
+	OPENTELEMETRY_ENDPOINT_URL=http://0.0.0.0:55680 RUST_LOG=trace,hyper=info,tower=info cargo run
 
 compile:
 	docker run --rm \
diff --git a/src/telemetry.rs b/src/telemetry.rs
index f276d33..7726c07 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -10,8 +10,6 @@ pub fn init() -> std::result::Result<(), Box<dyn std::error::Error>> {
         .unwrap_or("http://0.0.0.0:55680".to_string());
 
     let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
-        // NB: need to port-forward the service
-        // k port-forward -n monitoring service/grafana-agent-traces 55680:55680
         .with_endpoint(&otlp_endpoint)
         .install()?;
 
@@ -37,7 +35,7 @@ pub fn init() -> std::result::Result<(), Box<dyn std::error::Error>> {
 // ideally it would be exposed by tracing::Span
 pub fn get_trace_id() -> String {
     use opentelemetry::trace::{SpanContext, TraceContextExt, Tracer};
-    opentelemetry::global::tracer("registry").in_span("reconcile", |cx| {
+    opentelemetry::global::tracer("controller").in_span("reconcile", |cx| {
         cx.span().span_context().trace_id().to_hex()
     })
 }

From 02c1326e3c78027947147f889f5083fe69ffe192 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Mon, 8 Mar 2021 11:00:07 +0000
Subject: [PATCH 03/16] move telemetry to main so it stays around

able to get traceids within the reconciler,
however, now the app cannot be ctrl-c'd for some reason...
---
 src/main.rs      | 28 +++++++++++++++++++++++++++-
 src/manager.rs   |  8 ++++----
 src/telemetry.rs | 39 +++------------------------------------
 3 files changed, 34 insertions(+), 41 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index eb89725..b019d0d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,6 +2,9 @@
 pub use controller::*;
 use prometheus::{Encoder, TextEncoder};
 use tracing::{debug, error, info, trace, warn};
+use tracing_subscriber::prelude::*;
+use tracing_subscriber::{fmt, EnvFilter};
+use tracing_subscriber::{Registry, fmt::format::FmtSpan};
 
 use actix_web::{
     get, middleware,
@@ -31,9 +34,32 @@ async fn index(c: Data<Manager>, _req: HttpRequest) -> impl Responder {
 
 #[actix_rt::main]
 async fn main() -> Result<()> {
+    let otlp_endpoint = std::env::var("OPENTELEMETRY_ENDPOINT_URL")
+        //.unwrap_or("http://grafana-agent-traces.monitoring.svc.cluster.local:55680")
+        .unwrap_or("http://0.0.0.0:55680".to_string());
+
+    let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
+        .with_endpoint(&otlp_endpoint)
+        .install().unwrap();
+
+    // Finish layers
+    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+    let logger = tracing_subscriber::fmt::layer();//.with_span_events(FmtSpan::ACTIVE);
+
+    let filter_layer = EnvFilter::try_from_default_env()
+        .or_else(|_| EnvFilter::try_new("info")).unwrap();
+
+    // Register all subscribers
+    let collector = Registry::default()
+        .with(filter_layer)
+        .with(telemetry)
+        .with(logger);
+    tracing::subscriber::set_global_default(collector).unwrap();
+
     let client = kube::Client::try_default().await.expect("create client");
     let (manager, drainer) = Manager::new(client).await;
-    telemetry::init().expect("init telemetry");
+
+    let app_root = tracing::span!(tracing::Level::INFO, "app_start");
     let server = HttpServer::new(move || {
         App::new()
             .data(manager.clone())
diff --git a/src/manager.rs b/src/manager.rs
index 031d3f6..f1a3121 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::sync::Arc;
 use tokio::{sync::RwLock, time::Duration};
-use tracing::{debug, error, info, instrument, trace, warn, event, Level};
+use tracing::{debug, error, info, instrument, trace, warn, event, Level, field, Span};
 
 /// Our Foo custom resource spec
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
@@ -41,15 +41,15 @@ struct Data {
     metrics: Metrics,
 }
 
-#[instrument(skip(ctx))]
+#[instrument(skip(ctx), fields(traceId))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
-    let tid = telemetry::get_trace_id();
+    Span::current().record("traceId", &field::display(&telemetry::get_trace_id()));
 
     let client = ctx.get_ref().client.clone();
     ctx.get_ref().state.write().await.last_event = Utc::now();
     let name = Meta::name(&foo);
     let ns = Meta::namespace(&foo).expect("foo is namespaced");
-    info!("Reconcile {} traceId={}", name, tid);
+    info!("Reconcile {}", name);
     let foos: Api<Foo> = Api::namespaced(client, &ns);
 
     let new_status = Patch::Apply(json!({
diff --git a/src/telemetry.rs b/src/telemetry.rs
index 7726c07..54e3fc0 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -1,41 +1,8 @@
 use crate::{Error, Result};
-//use tracing_attributes::instrument;
-use tracing_subscriber::prelude::*;
-use tracing_subscriber::{Registry, fmt::format::FmtSpan};
-use tracing_subscriber::{fmt, EnvFilter};
 
-pub fn init() -> std::result::Result<(), Box<dyn std::error::Error>> {
-    let otlp_endpoint = std::env::var("OPENTELEMETRY_ENDPOINT_URL")
-        //.unwrap_or("http://grafana-agent-traces.monitoring.svc.cluster.local:55680")
-        .unwrap_or("http://0.0.0.0:55680".to_string());
-
-    let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
-        .with_endpoint(&otlp_endpoint)
-        .install()?;
-
-    // Finish layers
-    let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-    let logger = tracing_subscriber::fmt::layer();//.with_span_events(FmtSpan::ACTIVE);
-
-    let filter_layer = EnvFilter::try_from_default_env()
-        .or_else(|_| EnvFilter::try_new("info"))?;
-
-    // Register all subscribers
-    let subscriber = Registry::default()
-        .with(filter_layer)
-        .with(telemetry)
-        .with(logger);
-
-    tracing::subscriber::set_global_default(subscriber)?;
-    Ok(())
-}
-
-// attempt to get TraceId by span name via opentelemetry
-// feels a bit like a backdoor, but it's not passed through all the layers
-// ideally it would be exposed by tracing::Span
+///  Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack
 pub fn get_trace_id() -> String {
     use opentelemetry::trace::{SpanContext, TraceContextExt, Tracer};
-    opentelemetry::global::tracer("controller").in_span("reconcile", |cx| {
-        cx.span().span_context().trace_id().to_hex()
-    })
+    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    tracing::Span::current().context().span().span_context().trace_id().to_hex()
 }

From 6074629ba5357a4e8ecc27aca2a9eb1f30a1e8af Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Mon, 8 Mar 2021 16:28:00 +0000
Subject: [PATCH 04/16] document

---
 src/main.rs      | 1 -
 src/telemetry.rs | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index b019d0d..7aeb39e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -59,7 +59,6 @@ async fn main() -> Result<()> {
     let client = kube::Client::try_default().await.expect("create client");
     let (manager, drainer) = Manager::new(client).await;
 
-    let app_root = tracing::span!(tracing::Level::INFO, "app_start");
     let server = HttpServer::new(move || {
         App::new()
             .data(manager.clone())
diff --git a/src/telemetry.rs b/src/telemetry.rs
index 54e3fc0..c1c42fb 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -2,7 +2,7 @@ use crate::{Error, Result};
 
 ///  Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack
 pub fn get_trace_id() -> String {
-    use opentelemetry::trace::{SpanContext, TraceContextExt, Tracer};
-    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    use tracing_opentelemetry::OpenTelemetrySpanExt; // tracing::Span to opentelemetry::Context
+    use opentelemetry::trace::TraceContextExt; // opentelemetry::Context -> opentelemetry::trace::Span
     tracing::Span::current().context().span().span_context().trace_id().to_hex()
 }

From ce9dc9a4f9edf26f9be1a395603278b18643b888 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Mon, 8 Mar 2021 16:28:18 +0000
Subject: [PATCH 05/16] make it work with grcp-sys

currently does not work with tonic for some reason: https://github.com/open-telemetry/opentelemetry-rust/issues/473
---
 Cargo.lock | 686 +++++++++++++++++++++++++++++------------------------
 Cargo.toml |  12 +-
 2 files changed, 386 insertions(+), 312 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e883a93..1565ddd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,7 +17,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e"
 dependencies = [
  "bitflags",
- "bytes 1.0.1",
+ "bytes",
  "futures-core",
  "futures-sink",
  "log",
@@ -41,7 +41,7 @@ dependencies = [
  "base64",
  "bitflags",
  "brotli2",
- "bytes 1.0.1",
+ "bytes",
  "bytestring",
  "cookie",
  "derive_more",
@@ -136,9 +136,9 @@ dependencies = [
 
 [[package]]
 name = "actix-tls"
-version = "3.0.0-beta.3"
+version = "3.0.0-beta.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322b22e9238d065f331af0585089de8c36978fcc56f888054add4e4365b9916b"
+checksum = "d2b1455e3f7a26d40cfc1080b571f41e8165e5a88e937ed579f7a4b3d55b0370"
 dependencies = [
  "actix-codec",
  "actix-rt",
@@ -183,7 +183,7 @@ dependencies = [
  "actix-web-codegen",
  "ahash",
  "awc",
- "bytes 1.0.1",
+ "bytes",
  "derive_more",
  "either",
  "encoding_rs",
@@ -213,21 +213,30 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "addr2line"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
 name = "ahash"
-version = "0.7.0"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efa60d2eadd8b12a996add391db32bd1153eac697ba4869660c0016353611426"
+checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957"
 dependencies = [
  "getrandom",
  "once_cell",
- "version_check",
+ "version_check 0.9.2",
 ]
 
 [[package]]
@@ -248,44 +257,17 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "anyhow"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
-
 [[package]]
 name = "array_tool"
 version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
 
-[[package]]
-name = "async-stream"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c"
-dependencies = [
- "async-stream-impl",
- "futures-core",
-]
-
-[[package]]
-name = "async-stream-impl"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "async-trait"
-version = "0.1.42"
+version = "0.1.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d"
+checksum = "7e098e9c493fdf92832223594d9a164f96bdf17ba81a42aff86f85c76768726a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -320,8 +302,8 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "base64",
- "bytes 1.0.1",
- "cfg-if",
+ "bytes",
+ "cfg-if 1.0.0",
  "derive_more",
  "futures-core",
  "log",
@@ -333,6 +315,20 @@ dependencies = [
  "serde_urlencoded",
 ]
 
+[[package]]
+name = "backtrace"
+version = "0.3.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
+dependencies = [
+ "addr2line",
+ "cfg-if 1.0.0",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base-x"
 version = "0.2.8"
@@ -345,6 +341,25 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bindgen"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "cfg-if 0.1.10",
+ "clang-sys",
+ "lazy_static",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -360,6 +375,15 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "boringssl-src"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c212e3cd725f096e4de48d4ded0909cb62cb92c1a3420ff0e95edb46390a7736"
+dependencies = [
+ "cmake",
+]
+
 [[package]]
 name = "brotli-sys"
 version = "0.3.2"
@@ -382,9 +406,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.6.0"
+version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9"
+checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
 
 [[package]]
 name = "byteorder"
@@ -392,12 +416,6 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
-[[package]]
-name = "bytes"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
-
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -410,14 +428,29 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
 ]
 
 [[package]]
 name = "cc"
-version = "1.0.66"
+version = "1.0.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+
+[[package]]
+name = "cexpr"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 
 [[package]]
 name = "cfg-if"
@@ -439,6 +472,26 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "cmake"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "const_fn"
 version = "0.4.5"
@@ -473,13 +526,13 @@ dependencies = [
 
 [[package]]
 name = "cookie"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f"
+checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951"
 dependencies = [
  "percent-encoding",
  "time 0.2.25",
- "version_check",
+ "version_check 0.9.2",
 ]
 
 [[package]]
@@ -510,7 +563,7 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -554,7 +607,7 @@ version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "num_cpus",
 ]
 
@@ -595,7 +648,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "dirs-sys-next",
 ]
 
@@ -646,7 +699,7 @@ version = "0.8.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -663,10 +716,26 @@ dependencies = [
 ]
 
 [[package]]
-name = "fixedbitset"
-version = "0.2.0"
+name = "failure"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
+checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
+dependencies = [
+ "backtrace",
+ "failure_derive",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
 
 [[package]]
 name = "flate2"
@@ -674,7 +743,7 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "crc32fast",
  "libc",
  "miniz_oxide",
@@ -703,9 +772,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00"
+checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
 dependencies = [
  "matches",
  "percent-encoding",
@@ -810,7 +879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
 dependencies = [
  "typenum",
- "version_check",
+ "version_check 0.9.2",
 ]
 
 [[package]]
@@ -819,18 +888,69 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
  "wasi",
 ]
 
 [[package]]
-name = "h2"
+name = "gimli"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
+
+[[package]]
+name = "glob"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5"
+checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+
+[[package]]
+name = "grpcio"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dfdbacbc27f875402f878928c7e9dc22ee26d01a56a9faf655f7090e7e25109"
+dependencies = [
+ "futures",
+ "grpcio-sys",
+ "libc",
+ "log",
+ "parking_lot",
+ "protobuf",
+]
+
+[[package]]
+name = "grpcio-compiler"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad76ae5de2e2d85fe8f45e072f7909e22fa9ffaf0d53b280825783720dab051c"
+dependencies = [
+ "protobuf",
+]
+
+[[package]]
+name = "grpcio-sys"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01cf1bf66e791876cfb406952895d87912eab70e858a5474c3608e5297509eba"
+dependencies = [
+ "bindgen",
+ "boringssl-src",
+ "cc",
+ "cmake",
+ "libc",
+ "libz-sys",
+ "pkg-config",
+ "walkdir",
+]
+
+[[package]]
+name = "h2"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -841,7 +961,6 @@ dependencies = [
  "tokio",
  "tokio-util",
  "tracing",
- "tracing-futures",
 ]
 
 [[package]]
@@ -850,15 +969,6 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
-[[package]]
-name = "heck"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -874,7 +984,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "fnv",
  "itoa",
 ]
@@ -885,7 +995,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "http",
 ]
 
@@ -916,11 +1026,10 @@ version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
  "http",
  "http-body",
  "httparse",
@@ -952,7 +1061,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "hyper",
  "native-tls",
  "tokio",
@@ -967,9 +1076,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de910d521f7cc3135c4de8db1cb910e0b5ed1dc6f57c381cd07e8e661ce10094"
+checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21"
 dependencies = [
  "matches",
  "unicode-bidi",
@@ -978,9 +1087,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.6.1"
+version = "1.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b"
+checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -992,25 +1101,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
 dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "itertools"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itertools"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
-dependencies = [
- "either",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -1048,7 +1139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bcc1f973542059e6d5a6d63de6a9539d0ec784f82b2327f3c1915d33200bc6a4"
 dependencies = [
  "base64",
- "bytes 1.0.1",
+ "bytes",
  "chrono",
  "serde",
  "serde-value",
@@ -1063,7 +1154,7 @@ checksum = "8d47a55e9f881dc5027dcaf026670fa24b41f67926ab6517e2155488fe9c012a"
 dependencies = [
  "Inflector",
  "base64",
- "bytes 1.0.1",
+ "bytes",
  "chrono",
  "dirs-next",
  "either",
@@ -1138,9 +1229,31 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.86"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a"
+
+[[package]]
+name = "libloading"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
+checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
+dependencies = [
+ "cc",
+ "winapi",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
 
 [[package]]
 name = "linked-hash-map"
@@ -1163,7 +1276,7 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -1195,9 +1308,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
  "adler",
  "autocfg",
@@ -1205,9 +1318,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.7.7"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7"
+checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a"
 dependencies = [
  "libc",
  "log",
@@ -1226,12 +1339,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "multimap"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333"
-
 [[package]]
 name = "native-tls"
 version = "0.2.7"
@@ -1250,6 +1357,16 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "nom"
+version = "4.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
+dependencies = [
+ "memchr",
+ "version_check 0.1.5",
+]
+
 [[package]]
 name = "ntapi"
 version = "0.3.6"
@@ -1288,11 +1405,17 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
+
 [[package]]
 name = "once_cell"
-version = "1.5.2"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
+checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
 
 [[package]]
 name = "opaque-debug"
@@ -1307,7 +1430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70"
 dependencies = [
  "bitflags",
- "cfg-if",
+ "cfg-if 1.0.0",
  "foreign-types",
  "lazy_static",
  "libc",
@@ -1357,12 +1480,13 @@ checksum = "ff889e68a02000126fa635a311abc329ce128c576b7dddc4cd3270f3c240afd0"
 dependencies = [
  "async-trait",
  "futures",
+ "grpcio",
  "opentelemetry",
- "prost 0.7.0",
+ "protobuf",
+ "protobuf-codegen",
+ "protoc-grpcio",
  "thiserror",
  "tokio",
- "tonic",
- "tonic-build",
 ]
 
 [[package]]
@@ -1387,23 +1511,29 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
+checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "instant",
  "libc",
- "redox_syscall 0.1.57",
+ "redox_syscall",
  "smallvec",
  "winapi",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
 [[package]]
 name = "pem"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c220d01f863d13d96ca82359d1e81e64a7c6bf0637bcde7b2349630addf0c6"
+checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb"
 dependencies = [
  "base64",
  "once_cell",
@@ -1416,16 +1546,6 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
-[[package]]
-name = "petgraph"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
-dependencies = [
- "fixedbitset",
- "indexmap",
-]
-
 [[package]]
 name = "pin-project"
 version = "0.4.27"
@@ -1468,9 +1588,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827"
+checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
 
 [[package]]
 name = "pin-utils"
@@ -1517,7 +1637,7 @@ version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "fnv",
  "lazy_static",
  "memchr",
@@ -1527,85 +1647,44 @@ dependencies = [
 ]
 
 [[package]]
-name = "prost"
-version = "0.6.1"
+name = "protobuf"
+version = "2.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212"
-dependencies = [
- "bytes 0.5.6",
- "prost-derive 0.6.1",
-]
+checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86"
 
 [[package]]
-name = "prost"
-version = "0.7.0"
+name = "protobuf-codegen"
+version = "2.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2"
+checksum = "e8217a1652dbc91d19c509c558234145faed729191a966896414e5889f62d543"
 dependencies = [
- "bytes 1.0.1",
- "prost-derive 0.7.0",
+ "protobuf",
 ]
 
 [[package]]
-name = "prost-build"
-version = "0.6.1"
+name = "protoc"
+version = "2.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26"
+checksum = "57628f7456fe5fff1591901fb8b594e1584750cb857da23b65c456b42670524d"
 dependencies = [
- "bytes 0.5.6",
- "heck",
- "itertools 0.8.2",
  "log",
- "multimap",
- "petgraph",
- "prost 0.6.1",
- "prost-types",
- "tempfile",
  "which",
 ]
 
 [[package]]
-name = "prost-derive"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72"
-dependencies = [
- "anyhow",
- "itertools 0.8.2",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4"
-dependencies = [
- "anyhow",
- "itertools 0.9.0",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.6.1"
+name = "protoc-grpcio"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa"
+checksum = "af5d484461b7f14e103742f21014cc519f6f3711b05d50b3e86d912467a3f8b7"
 dependencies = [
- "bytes 0.5.6",
- "prost 0.6.1",
+ "failure",
+ "grpcio-compiler",
+ "protobuf",
+ "protobuf-codegen",
+ "protoc",
+ "tempfile",
 ]
 
-[[package]]
-name = "protobuf"
-version = "2.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86"
-
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -1614,9 +1693,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quote"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
 dependencies = [
  "proc-macro2",
 ]
@@ -1645,9 +1724,9 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
+checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
 dependencies = [
  "getrandom",
 ]
@@ -1663,15 +1742,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.1.57"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
-
-[[package]]
-name = "redox_syscall"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570"
+checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
 dependencies = [
  "bitflags",
 ]
@@ -1683,7 +1756,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
  "getrandom",
- "redox_syscall 0.2.4",
+ "redox_syscall",
 ]
 
 [[package]]
@@ -1723,6 +1796,18 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -1738,6 +1823,15 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.19"
@@ -1781,9 +1875,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "security-framework"
-version = "2.0.0"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69"
+checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d"
 dependencies = [
  "bitflags",
  "core-foundation",
@@ -1794,9 +1888,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.0.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b"
+checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -1819,9 +1913,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "serde"
-version = "1.0.123"
+version = "1.0.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
+checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f"
 dependencies = [
  "serde_derive",
 ]
@@ -1838,9 +1932,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.123"
+version = "1.0.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
+checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1896,12 +1990,12 @@ dependencies = [
 
 [[package]]
 name = "sha-1"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4b312c3731e3fe78a185e6b9b911a7aa715b8e31cce117975219aab2acf285d"
+checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f"
 dependencies = [
  "block-buffer",
- "cfg-if",
+ "cfg-if 1.0.0",
  "cpuid-bool",
  "digest",
  "opaque-debug",
@@ -1922,6 +2016,12 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "shlex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.3.0"
@@ -1972,7 +2072,7 @@ version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
  "winapi",
 ]
@@ -1983,7 +2083,7 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8"
 dependencies = [
- "version_check",
+ "version_check 0.9.2",
 ]
 
 [[package]]
@@ -2049,25 +2149,37 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.60"
+version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
+checksum = "123a78a3596b24fee53a6464ce52d8ecbf62241e6294c7e7fe12086cd161f512"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-xid",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "unicode-xid",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
  "rand",
- "redox_syscall 0.2.4",
+ "redox_syscall",
  "remove_dir_all",
  "winapi",
 ]
@@ -2132,7 +2244,7 @@ dependencies = [
  "standback",
  "stdweb",
  "time-macros",
- "version_check",
+ "version_check 0.9.2",
  "winapi",
 ]
 
@@ -2181,7 +2293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a"
 dependencies = [
  "autocfg",
- "bytes 1.0.1",
+ "bytes",
  "libc",
  "memchr",
  "mio",
@@ -2225,24 +2337,13 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-stream"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea"
-dependencies = [
- "futures-core",
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b"
 dependencies = [
- "bytes 1.0.1",
+ "bytes",
  "futures-core",
  "futures-sink",
  "log",
@@ -2251,47 +2352,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tonic"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ba8f479158947373b6df40cf48f4779bb25c99ca3c661bd95e0ab1963ad8b0e"
-dependencies = [
- "async-stream",
- "async-trait",
- "base64",
- "bytes 1.0.1",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
- "percent-encoding",
- "pin-project 1.0.5",
- "prost 0.7.0",
- "prost-derive 0.7.0",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tower",
- "tower-service",
- "tracing",
- "tracing-futures",
-]
-
-[[package]]
-name = "tonic-build"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19970cf58f3acc820962be74c4021b8bbc8e8a1c4e3a02095d0aa60cde5f3633"
-dependencies = [
- "proc-macro2",
- "prost-build",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "tower"
 version = "0.4.6"
@@ -2300,12 +2360,8 @@ checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66"
 dependencies = [
  "futures-core",
  "futures-util",
- "indexmap",
  "pin-project 1.0.5",
- "rand",
- "slab",
  "tokio",
- "tokio-stream",
  "tokio-util",
  "tower-layer",
  "tower-service",
@@ -2330,7 +2386,7 @@ version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -2453,12 +2509,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"
@@ -2467,9 +2517,9 @@ checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
 
 [[package]]
 name = "url"
-version = "2.2.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e"
+checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -2483,12 +2533,29 @@ version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
 
+[[package]]
+name = "version_check"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
+
 [[package]]
 name = "version_check"
 version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
 
+[[package]]
+name = "walkdir"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.0"
@@ -2511,7 +2578,7 @@ version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "wasm-bindgen-macro",
 ]
 
@@ -2561,11 +2628,12 @@ checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1"
 
 [[package]]
 name = "which"
-version = "3.1.1"
+version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
+checksum = "87c14ef7e1b8b8ecfc75d5eca37949410046e66f15d185c01d70824f1f8111ef"
 dependencies = [
  "libc",
+ "thiserror",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 2982700..59c276e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,6 +15,11 @@ doc = false
 name = "crdgen"
 path = "src/crdgen.rs"
 
+[[bin]]
+doc = false
+name = "blah"
+path = "blah.rs"
+
 [lib]
 name = "controller"
 path = "src/lib.rs"
@@ -32,14 +37,15 @@ serde = { version = "1.0.123", features = ["derive"] }
 serde_json = "1.0.64"
 chrono = { version = "0.4.19", features = ["serde"] }
 thiserror = "1.0.24"
+schemars = { version = "0.8.0", features = ["chrono"] }
+serde_yaml = "0.8.17"
 tracing = "0.1.25"
 tracing-subscriber = { version = "0.2.16", features = ["json"] }
 tracing-futures = "0.2.5"
-schemars = { version = "0.8.0", features = ["chrono"] }
-serde_yaml = "0.8.17"
 tracing-opentelemetry = "0.11.0"
 opentelemetry = { version = "0.12.0", features = ["trace"] }
-opentelemetry-otlp = { version = "0.5.0", features = ["async", "tokio"] }
+opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "openssl"], default-features=false }
+#opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "async"] }
 
 # local dev
 #kube = { path = "../kube-rs/kube", features = ["derive"] }

From ecfa2f4cf93c8a8e82c4cb9150085f1aa11d368f Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Tue, 9 Mar 2021 00:36:53 +0000
Subject: [PATCH 06/16] more loki + tracing compat

---
 src/main.rs          | 5 ++---
 src/manager.rs       | 4 ++--
 yaml/deployment.yaml | 3 +++
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 7aeb39e..7383424 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -35,8 +35,7 @@ async fn index(c: Data<Manager>, _req: HttpRequest) -> impl Responder {
 #[actix_rt::main]
 async fn main() -> Result<()> {
     let otlp_endpoint = std::env::var("OPENTELEMETRY_ENDPOINT_URL")
-        //.unwrap_or("http://grafana-agent-traces.monitoring.svc.cluster.local:55680")
-        .unwrap_or("http://0.0.0.0:55680".to_string());
+        .expect("Need a otel tracing collector configured");
 
     let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
         .with_endpoint(&otlp_endpoint)
@@ -44,7 +43,7 @@ async fn main() -> Result<()> {
 
     // Finish layers
     let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-    let logger = tracing_subscriber::fmt::layer();//.with_span_events(FmtSpan::ACTIVE);
+    let logger = tracing_subscriber::fmt::layer();
 
     let filter_layer = EnvFilter::try_from_default_env()
         .or_else(|_| EnvFilter::try_new("info")).unwrap();
diff --git a/src/manager.rs b/src/manager.rs
index f1a3121..151a1e6 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -41,9 +41,9 @@ struct Data {
     metrics: Metrics,
 }
 
-#[instrument(skip(ctx), fields(traceId))]
+#[instrument(skip(ctx), fields(traceID, name = "reconcile"))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
-    Span::current().record("traceId", &field::display(&telemetry::get_trace_id()));
+    Span::current().record("traceID", &field::display(&telemetry::get_trace_id()));
 
     let client = ctx.get_ref().client.clone();
     ctx.get_ref().state.write().await.last_event = Utc::now();
diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml
index 8dc67dc..7997ad6 100644
--- a/yaml/deployment.yaml
+++ b/yaml/deployment.yaml
@@ -91,6 +91,9 @@ spec:
         - name: http
           containerPort: 8080
           protocol: TCP
+        env:
+        - name: OPENTELEMETRY_ENDPOINT_URL
+          value: "grafana-agent-traces.monitoring.svc.cluster.local:55680"
         readinessProbe:
           httpGet:
             path: /health

From 3f694d5ac7bd6e2af351aa565a2354086c69209e Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Tue, 9 Mar 2021 13:26:32 +0000
Subject: [PATCH 07/16] simplify tracing output

---
 src/main.rs    | 10 ++++++----
 src/manager.rs |  7 ++++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 7383424..cb96c47 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -50,14 +50,16 @@ async fn main() -> Result<()> {
 
     // Register all subscribers
     let collector = Registry::default()
-        .with(filter_layer)
         .with(telemetry)
-        .with(logger);
+        .with(logger)
+        .with(filter_layer);
+
     tracing::subscriber::set_global_default(collector).unwrap();
 
-    let client = kube::Client::try_default().await.expect("create client");
-    let (manager, drainer) = Manager::new(client).await;
+    // Start kubernetes controller
+    let (manager, drainer) = Manager::new().await;
 
+    // Start web server
     let server = HttpServer::new(move || {
         App::new()
             .data(manager.clone())
diff --git a/src/manager.rs b/src/manager.rs
index 151a1e6..b477f7e 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -41,7 +41,7 @@ struct Data {
     metrics: Metrics,
 }
 
-#[instrument(skip(ctx), fields(traceID, name = "reconcile"))]
+#[instrument(skip(ctx), fields(traceID))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
     Span::current().record("traceID", &field::display(&telemetry::get_trace_id()));
 
@@ -49,7 +49,7 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
     ctx.get_ref().state.write().await.last_event = Utc::now();
     let name = Meta::name(&foo);
     let ns = Meta::namespace(&foo).expect("foo is namespaced");
-    info!("Reconcile {}", name);
+    info!("Reconciling Foo \"{}\"", name);
     let foos: Api<Foo> = Api::namespaced(client, &ns);
 
     let new_status = Patch::Apply(json!({
@@ -119,7 +119,8 @@ impl Manager {
     ///
     /// This returns a `Manager` that drives a `Controller` + a future to be awaited
     /// It is up to `main` to wait for the controller stream.
-    pub async fn new(client: Client) -> (Self, BoxFuture<'static, ()>) {
+    pub async fn new() -> (Self, BoxFuture<'static, ()>) {
+        let client = Client::try_default().await.expect("create client");
         let metrics = Metrics::new();
         let state = Arc::new(RwLock::new(State::new()));
         let context = Context::new(Data {

From 6245a07696df6a2e7bda6a9cc41ff7c92ba124d6 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Tue, 9 Mar 2021 13:37:51 +0000
Subject: [PATCH 08/16] pin deps, not getting tonic to work

---
 Cargo.lock | 10 ++++------
 Cargo.toml |  6 ++++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1565ddd..d8aaf49 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1149,8 +1149,7 @@ dependencies = [
 [[package]]
 name = "kube"
 version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d47a55e9f881dc5027dcaf026670fa24b41f67926ab6517e2155488fe9c012a"
+source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
 dependencies = [
  "Inflector",
  "base64",
@@ -1179,14 +1178,14 @@ dependencies = [
  "tokio-native-tls",
  "tokio-util",
  "tower",
+ "tracing",
  "url",
 ]
 
 [[package]]
 name = "kube-derive"
 version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c069cf7fcfe148819d01d1ace64ed82d0bef2c5c27efd2dbc073ec4d210becb9"
+source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
 dependencies = [
  "Inflector",
  "darling",
@@ -1199,8 +1198,7 @@ dependencies = [
 [[package]]
 name = "kube-runtime"
 version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69d85c47fe6b8d7c4c3b634614d498a841028fc1ccca4c748e1801266848127b"
+source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
 dependencies = [
  "dashmap",
  "derivative",
diff --git a/Cargo.toml b/Cargo.toml
index 59c276e..2ae45e5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,8 +30,8 @@ actix-web = "4.0.0-beta.1"
 futures = "0.3.13"
 tokio = { version = "1.1.0", features = ["macros"] }
 prometheus = "0.12.0"
-kube = { version = "0.51.0", features = ["derive"] }
-kube-runtime = "0.51.0"
+#kube = { version = "0.51.0", features = ["derive"] }
+#kube-runtime = "0.51.0"
 k8s-openapi = { version = "0.11.0", features = ["v1_19"], default-features=false }
 serde = { version = "1.0.123", features = ["derive"] }
 serde_json = "1.0.64"
@@ -50,3 +50,5 @@ opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "open
 # local dev
 #kube = { path = "../kube-rs/kube", features = ["derive"] }
 #kube-runtime = { path = "../kube-rs/kube-runtime" }
+kube = { git = "https://github.com/clux/kube-rs.git", branch = "tracing-test", features = ["derive"] }
+kube-runtime = { git = "https://github.com/clux/kube-rs.git", branch = "tracing-test" }

From 242e706df3a4324f35174d17b249f90aaddd4c8a Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Tue, 9 Mar 2021 15:28:31 +0000
Subject: [PATCH 09/16] make a docker image

---
 Cargo.lock           |  6 +++---
 Cargo.toml           |  9 ++-------
 Dockerfile           | 14 ++++++++++----
 Makefile             | 22 +++++++++-------------
 yaml/deployment.yaml |  2 +-
 5 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d8aaf49..dcdcd57 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1149,7 +1149,7 @@ dependencies = [
 [[package]]
 name = "kube"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
+source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
  "Inflector",
  "base64",
@@ -1185,7 +1185,7 @@ dependencies = [
 [[package]]
 name = "kube-derive"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
+source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
  "Inflector",
  "darling",
@@ -1198,7 +1198,7 @@ dependencies = [
 [[package]]
 name = "kube-runtime"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?branch=tracing-test#6544c829009d046f3fd633557f04d28f49042cd7"
+source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
  "dashmap",
  "derivative",
diff --git a/Cargo.toml b/Cargo.toml
index 2ae45e5..104e3d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,11 +15,6 @@ doc = false
 name = "crdgen"
 path = "src/crdgen.rs"
 
-[[bin]]
-doc = false
-name = "blah"
-path = "blah.rs"
-
 [lib]
 name = "controller"
 path = "src/lib.rs"
@@ -50,5 +45,5 @@ opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "open
 # local dev
 #kube = { path = "../kube-rs/kube", features = ["derive"] }
 #kube-runtime = { path = "../kube-rs/kube-runtime" }
-kube = { git = "https://github.com/clux/kube-rs.git", branch = "tracing-test", features = ["derive"] }
-kube-runtime = { git = "https://github.com/clux/kube-rs.git", branch = "tracing-test" }
+kube = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522", features = ["derive"] }
+kube-runtime = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522" }
diff --git a/Dockerfile b/Dockerfile
index 38a57d3..3fd9dca 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,10 @@
-FROM gcr.io/distroless/static:nonroot
-COPY --chown=nonroot:nonroot ./controller /app/
-EXPOSE 8080
-ENTRYPOINT ["/app/controller"]
+FROM rust:1.50 as builder
+WORKDIR /usr/src/controller
+RUN apt-get update && apt-get install cmake protobuf-compiler -y
+COPY . .
+RUN cargo install --locked --path .
+
+FROM debian:buster-slim
+RUN apt-get update && apt-get install -y openssl && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /usr/local/cargo/bin/controller /usr/local/bin/controller
+CMD ["controller"]
diff --git a/Makefile b/Makefile
index c7d2325..526d1ca 100644
--- a/Makefile
+++ b/Makefile
@@ -13,21 +13,9 @@ forward-tempo:
 	kubectl port-forward -n monitoring service/grafana-agent-traces 55680:55680
 
 run:
-	OPENTELEMETRY_ENDPOINT_URL=http://0.0.0.0:55680 RUST_LOG=trace,hyper=info,tower=info cargo run
-
-compile:
-	docker run --rm \
-		-v cargo-cache:/root/.cargo \
-		-v $$PWD:/volume \
-		-w /volume \
-		-it clux/muslrust:stable \
-		cargo build --release
-	sudo chown $$USER:$$USER -R target
-	mv target/x86_64-unknown-linux-musl/release/controller .
+	OPENTELEMETRY_ENDPOINT_URL=0.0.0.0:55680 RUST_LOG=debug,kube=trace,hyper=info,tower=info cargo run
 
 build:
-	@echo "Reusing built binary in current directory from make compile"
-	@ls -lah ./controller
 	docker build -t $(REPO)/$(NAME):$(VERSION) .
 
 tag-latest: build
@@ -42,3 +30,11 @@ tag-semver: build
 		docker tag $(REPO)/$(NAME):$(VERSION) $(REPO)/$(NAME):$(SEMVER_VERSION) ; \
 		docker push $(REPO)/$(NAME):$(SEMVER_VERSION) ; \
 	fi
+
+# Helpers for debugging with tempo as an otel collector
+forward-tempo-metrics:
+	kubectl port-forward -n monitoring service/grafana-agent-traces 8080:8080
+check-tempo-metrics:
+	curl http://0.0.0.0:8080/metrics -s |grep -E "^tempo_receiver_accepted_span"
+	# can verify that spans are received from metrics on the grafana-agent-traces
+    # tempo_receiver_accepted_spans{receiver="otlp",tempo_config="default",transport="grpc"} 4
diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml
index 7997ad6..2f8932c 100644
--- a/yaml/deployment.yaml
+++ b/yaml/deployment.yaml
@@ -78,7 +78,7 @@ spec:
       serviceAccountName: foo-controller
       containers:
       - name: foo-controller
-        image: "clux/controller:latest"
+        image: "clux/controller:6245a07696df6a2e7bda6a9cc41ff7c92ba124d6"
         imagePullPolicy: Always
         resources:
           limits:

From b99ad0bfbf4ae75f03323bff2796572d4257bd96 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Tue, 9 Mar 2021 15:33:16 +0000
Subject: [PATCH 10/16] use json logging + fix imports

---
 src/main.rs          | 6 ++----
 yaml/deployment.yaml | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index cb96c47..4b635cb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,9 +2,7 @@
 pub use controller::*;
 use prometheus::{Encoder, TextEncoder};
 use tracing::{debug, error, info, trace, warn};
-use tracing_subscriber::prelude::*;
-use tracing_subscriber::{fmt, EnvFilter};
-use tracing_subscriber::{Registry, fmt::format::FmtSpan};
+use tracing_subscriber::{EnvFilter, Registry, prelude::*};
 
 use actix_web::{
     get, middleware,
@@ -43,7 +41,7 @@ async fn main() -> Result<()> {
 
     // Finish layers
     let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-    let logger = tracing_subscriber::fmt::layer();
+    let logger = tracing_subscriber::fmt::layer().json();
 
     let filter_layer = EnvFilter::try_from_default_env()
         .or_else(|_| EnvFilter::try_new("info")).unwrap();
diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml
index 2f8932c..15831c4 100644
--- a/yaml/deployment.yaml
+++ b/yaml/deployment.yaml
@@ -78,7 +78,7 @@ spec:
       serviceAccountName: foo-controller
       containers:
       - name: foo-controller
-        image: "clux/controller:6245a07696df6a2e7bda6a9cc41ff7c92ba124d6"
+        image: "clux/controller:b330f691d74d6afc8e72eab325a526b7b8b124cc"
         imagePullPolicy: Always
         resources:
           limits:

From 7237271cba4e41f275b7a921a53c8e8972c0d08f Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Wed, 10 Mar 2021 10:10:18 +0000
Subject: [PATCH 11/16] get svc name in span look

---
 src/manager.rs       | 9 +++------
 yaml/deployment.yaml | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/manager.rs b/src/manager.rs
index b477f7e..0ca219c 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -41,7 +41,7 @@ struct Data {
     metrics: Metrics,
 }
 
-#[instrument(skip(ctx), fields(traceID))]
+#[instrument(skip(ctx), fields(traceID, service.name = "foo-controller"))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
     Span::current().record("traceID", &field::display(&telemetry::get_trace_id()));
 
@@ -49,7 +49,6 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
     ctx.get_ref().state.write().await.last_event = Utc::now();
     let name = Meta::name(&foo);
     let ns = Meta::namespace(&foo).expect("foo is namespaced");
-    info!("Reconciling Foo \"{}\"", name);
     let foos: Api<Foo> = Api::namespaced(client, &ns);
 
     let new_status = Patch::Apply(json!({
@@ -64,6 +63,7 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
     let _o = foos.patch_status(&name, &ps, &new_status).await.map_err(Error::KubeError)?;
 
     ctx.get_ref().metrics.handled_events.inc();
+    info!("Reconciled Foo \"{}\" in {}", name, ns);
 
     // If no events were received, check back every 30 minutes
     Ok(ReconcilerAction {
@@ -135,10 +135,7 @@ impl Manager {
         let drainer = Controller::new(foos, ListParams::default())
             .run(reconcile, error_policy, context)
             .filter_map(|x| async move { std::result::Result::ok(x) })
-            .for_each(|o| {
-                info!("Reconciled {:?}", o);
-                futures::future::ready(())
-            })
+            .for_each(|_| futures::future::ready(()))
             .boxed();
         // what we do with the controller stream from .run() ^^ does not matter
         // but we do need to consume it, hence general printing + return future
diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml
index 15831c4..cd3b1df 100644
--- a/yaml/deployment.yaml
+++ b/yaml/deployment.yaml
@@ -78,7 +78,7 @@ spec:
       serviceAccountName: foo-controller
       containers:
       - name: foo-controller
-        image: "clux/controller:b330f691d74d6afc8e72eab325a526b7b8b124cc"
+        image: "clux/controller:b99ad0bfbf4ae75f03323bff2796572d4257bd96"
         imagePullPolicy: Always
         resources:
           limits:

From d0485ea7d993450d37aaa2f710ffbcba1fb0367d Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Wed, 10 Mar 2021 11:32:03 +0000
Subject: [PATCH 12/16] push a working demo image

---
 yaml/deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yaml/deployment.yaml b/yaml/deployment.yaml
index cd3b1df..b3e19f8 100644
--- a/yaml/deployment.yaml
+++ b/yaml/deployment.yaml
@@ -78,7 +78,7 @@ spec:
       serviceAccountName: foo-controller
       containers:
       - name: foo-controller
-        image: "clux/controller:b99ad0bfbf4ae75f03323bff2796572d4257bd96"
+        image: "clux/controller:demo-otel"
         imagePullPolicy: Always
         resources:
           limits:

From 2e6849b80b191d1c9b7bacfb02e506102d2f4cf2 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Wed, 10 Mar 2021 22:19:42 +0000
Subject: [PATCH 13/16] make service name visible in tempo tracing

---
 rustfmt.toml   |  2 +-
 src/main.rs    | 18 +++++++++++++-----
 src/manager.rs | 11 +++++++----
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/rustfmt.toml b/rustfmt.toml
index e596591..1546e4d 100644
--- a/rustfmt.toml
+++ b/rustfmt.toml
@@ -1,6 +1,6 @@
 overflow_delimited_expr = true
 newline_style = "Unix"
-merge_imports = true
+imports_granularity = "Crate"
 reorder_impl_items = true
 fn_single_line = false
 blank_lines_upper_bound = 2
diff --git a/src/main.rs b/src/main.rs
index 4b635cb..a975023 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,7 +2,7 @@
 pub use controller::*;
 use prometheus::{Encoder, TextEncoder};
 use tracing::{debug, error, info, trace, warn};
-use tracing_subscriber::{EnvFilter, Registry, prelude::*};
+use tracing_subscriber::{prelude::*, EnvFilter, Registry};
 
 use actix_web::{
     get, middleware,
@@ -32,19 +32,27 @@ async fn index(c: Data<Manager>, _req: HttpRequest) -> impl Responder {
 
 #[actix_rt::main]
 async fn main() -> Result<()> {
-    let otlp_endpoint = std::env::var("OPENTELEMETRY_ENDPOINT_URL")
-        .expect("Need a otel tracing collector configured");
+    let otlp_endpoint =
+        std::env::var("OPENTELEMETRY_ENDPOINT_URL").expect("Need a otel tracing collector configured");
 
     let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
         .with_endpoint(&otlp_endpoint)
-        .install().unwrap();
+        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
+            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
+                "service.name",
+                "foo-controller",
+            )]),
+        ))
+        .install()
+        .unwrap();
 
     // Finish layers
     let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
     let logger = tracing_subscriber::fmt::layer().json();
 
     let filter_layer = EnvFilter::try_from_default_env()
-        .or_else(|_| EnvFilter::try_new("info")).unwrap();
+        .or_else(|_| EnvFilter::try_new("info"))
+        .unwrap();
 
     // Register all subscribers
     let collector = Registry::default()
diff --git a/src/manager.rs b/src/manager.rs
index 0ca219c..51c50ed 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -1,4 +1,4 @@
-use crate::{Error, Result, telemetry};
+use crate::{telemetry, Error, Result};
 use chrono::prelude::*;
 use futures::{future::BoxFuture, FutureExt, StreamExt};
 use kube::{
@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::sync::Arc;
 use tokio::{sync::RwLock, time::Duration};
-use tracing::{debug, error, info, instrument, trace, warn, event, Level, field, Span};
+use tracing::{debug, error, event, field, info, instrument, trace, warn, Level, Span};
 
 /// Our Foo custom resource spec
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
@@ -41,7 +41,7 @@ struct Data {
     metrics: Metrics,
 }
 
-#[instrument(skip(ctx), fields(traceID, service.name = "foo-controller"))]
+#[instrument(skip(ctx), fields(traceID))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
     Span::current().record("traceID", &field::display(&telemetry::get_trace_id()));
 
@@ -60,7 +60,10 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
         }
     }));
     let ps = PatchParams::apply("cntrlr").force();
-    let _o = foos.patch_status(&name, &ps, &new_status).await.map_err(Error::KubeError)?;
+    let _o = foos
+        .patch_status(&name, &ps, &new_status)
+        .await
+        .map_err(Error::KubeError)?;
 
     ctx.get_ref().metrics.handled_events.inc();
     info!("Reconciled Foo \"{}\" in {}", name, ns);

From 8409c7c44837fdc4a140d40e6507bc26ce3d5f93 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Sun, 14 Mar 2021 18:01:54 +0000
Subject: [PATCH 14/16] use own fork of prometheus for exemplar support

---
 Cargo.lock       |  5 ++---
 Cargo.toml       |  4 +++-
 src/manager.rs   | 45 ++++++++++++++++++++++++++++++++++++++++-----
 src/telemetry.rs | 11 ++++++++---
 4 files changed, 53 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dcdcd57..4355c2c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1631,9 +1631,8 @@ dependencies = [
 
 [[package]]
 name = "prometheus"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
+version = "0.11.0"
+source = "git+https://github.com/clux/rust-prometheus.git?branch=master#543e2389ef205e0daa044031358fb23338e0b694"
 dependencies = [
  "cfg-if 1.0.0",
  "fnv",
diff --git a/Cargo.toml b/Cargo.toml
index 104e3d6..e87b1fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,7 @@ actix-rt = "2.1.0"
 actix-web = "4.0.0-beta.1"
 futures = "0.3.13"
 tokio = { version = "1.1.0", features = ["macros"] }
-prometheus = "0.12.0"
+#prometheus = "0.12.0"
 #kube = { version = "0.51.0", features = ["derive"] }
 #kube-runtime = "0.51.0"
 k8s-openapi = { version = "0.11.0", features = ["v1_19"], default-features=false }
@@ -47,3 +47,5 @@ opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "open
 #kube-runtime = { path = "../kube-rs/kube-runtime" }
 kube = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522", features = ["derive"] }
 kube-runtime = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522" }
+prometheus = { git = "https://github.com/clux/rust-prometheus.git", branch = "master" }
+#prometheus = { path = "../rust-prometheus" }
diff --git a/src/manager.rs b/src/manager.rs
index 51c50ed..5b12504 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -7,12 +7,18 @@ use kube::{
     CustomResource,
 };
 use kube_runtime::controller::{Context, Controller, ReconcilerAction};
-use prometheus::{default_registry, proto::MetricFamily, register_int_counter, IntCounter};
+use prometheus::{
+    default_registry, proto::MetricFamily, register_histogram_vec, register_int_counter, Exemplar,
+    HistogramOpts, HistogramVec, IntCounter,
+};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use std::sync::Arc;
-use tokio::{sync::RwLock, time::Duration};
+use std::{collections::HashMap, sync::Arc};
+use tokio::{
+    sync::RwLock,
+    time::{Duration, Instant},
+};
 use tracing::{debug, error, event, field, info, instrument, trace, warn, Level, Span};
 
 /// Our Foo custom resource spec
@@ -43,7 +49,9 @@ struct Data {
 
 #[instrument(skip(ctx), fields(traceID))]
 async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Error> {
-    Span::current().record("traceID", &field::display(&telemetry::get_trace_id()));
+    let trace_id = telemetry::get_trace_id();
+    Span::current().record("traceID", &field::display(&trace_id));
+    let start = Instant::now();
 
     let client = ctx.get_ref().client.clone();
     ctx.get_ref().state.write().await.last_event = Utc::now();
@@ -65,6 +73,16 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
         .await
         .map_err(Error::KubeError)?;
 
+    let duration = duration_to_seconds(start.elapsed());
+
+    let mut exemplar_labels = HashMap::new();
+    exemplar_labels.insert("traceID".into(), trace_id);
+    let ex = Exemplar::new_with_labels(duration, exemplar_labels);
+    ctx.get_ref()
+        .metrics
+        .reconcile_duration
+        .with_label_values(&[])
+        .observe_with_exemplar(duration, ex);
     ctx.get_ref().metrics.handled_events.inc();
     info!("Reconciled Foo \"{}\" in {}", name, ns);
 
@@ -84,11 +102,21 @@ fn error_policy(error: &Error, _ctx: Context<Data>) -> ReconcilerAction {
 #[derive(Clone)]
 pub struct Metrics {
     pub handled_events: IntCounter,
+    pub reconcile_duration: HistogramVec,
 }
 impl Metrics {
     fn new() -> Self {
+        let reconcile_histogram = register_histogram_vec!(
+            "foo_controller_reconcile_duration_seconds",
+            "The duration of reconcile to complete in seconds",
+            &[],
+            vec![0.01, 0.1, 0.25, 0.5, 1., 5., 15., 60.]
+        )
+        .unwrap();
+
         Metrics {
-            handled_events: register_int_counter!("handled_events", "handled events").unwrap(),
+            handled_events: register_int_counter!("foo_controller_handled_events", "handled events").unwrap(),
+            reconcile_duration: reconcile_histogram,
         }
     }
 }
@@ -156,3 +184,10 @@ impl Manager {
         self.state.read().await.clone()
     }
 }
+
+/// Convert `Duration` to seconds for `Histogram`s
+#[inline]
+pub fn duration_to_seconds(d: std::time::Duration) -> f64 {
+    let nanos = f64::from(d.subsec_nanos()) / 1e9;
+    d.as_secs() as f64 + nanos
+}
diff --git a/src/telemetry.rs b/src/telemetry.rs
index c1c42fb..1cf0588 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -2,7 +2,12 @@ use crate::{Error, Result};
 
 ///  Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack
 pub fn get_trace_id() -> String {
-    use tracing_opentelemetry::OpenTelemetrySpanExt; // tracing::Span to opentelemetry::Context
-    use opentelemetry::trace::TraceContextExt; // opentelemetry::Context -> opentelemetry::trace::Span
-    tracing::Span::current().context().span().span_context().trace_id().to_hex()
+    use opentelemetry::trace::TraceContextExt;
+    use tracing_opentelemetry::OpenTelemetrySpanExt; // tracing::Span to opentelemetry::Context // opentelemetry::Context -> opentelemetry::trace::Span
+    tracing::Span::current()
+        .context()
+        .span()
+        .span_context()
+        .trace_id()
+        .to_hex()
 }

From 728900de7297befcffa42d3c9ad7b33cd179c827 Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Sat, 27 Mar 2021 20:34:05 +0000
Subject: [PATCH 15/16] error handling when crd is not installed

---
 Cargo.lock       | 18 +-----------------
 Cargo.toml       | 12 ++++++------
 src/manager.rs   |  9 ++++++---
 src/telemetry.rs |  2 +-
 4 files changed, 14 insertions(+), 27 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4355c2c..57d1f0b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,15 +1,5 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-[[package]]
-name = "Inflector"
-version = "0.11.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
-dependencies = [
- "lazy_static",
- "regex",
-]
-
 [[package]]
 name = "actix-codec"
 version = "0.4.0-beta.1"
@@ -1149,9 +1139,7 @@ dependencies = [
 [[package]]
 name = "kube"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
- "Inflector",
  "base64",
  "bytes",
  "chrono",
@@ -1185,9 +1173,7 @@ dependencies = [
 [[package]]
 name = "kube-derive"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
- "Inflector",
  "darling",
  "proc-macro2",
  "quote",
@@ -1198,7 +1184,6 @@ dependencies = [
 [[package]]
 name = "kube-runtime"
 version = "0.51.0"
-source = "git+https://github.com/clux/kube-rs.git?rev=70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522#70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522"
 dependencies = [
  "dashmap",
  "derivative",
@@ -1631,8 +1616,7 @@ dependencies = [
 
 [[package]]
 name = "prometheus"
-version = "0.11.0"
-source = "git+https://github.com/clux/rust-prometheus.git?branch=master#543e2389ef205e0daa044031358fb23338e0b694"
+version = "0.12.0"
 dependencies = [
  "cfg-if 1.0.0",
  "fnv",
diff --git a/Cargo.toml b/Cargo.toml
index e87b1fb..b758e99 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,9 +43,9 @@ opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "open
 #opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "async"] }
 
 # local dev
-#kube = { path = "../kube-rs/kube", features = ["derive"] }
-#kube-runtime = { path = "../kube-rs/kube-runtime" }
-kube = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522", features = ["derive"] }
-kube-runtime = { git = "https://github.com/clux/kube-rs.git", rev = "70a7ce1ec4ff2aeefe5a277cfd45ea0654c92522" }
-prometheus = { git = "https://github.com/clux/rust-prometheus.git", branch = "master" }
-#prometheus = { path = "../rust-prometheus" }
+kube = { path = "../kube-rs/kube", features = ["derive"] }
+kube-runtime = { path = "../kube-rs/kube-runtime" }
+#kube = { git = "https://github.com/clux/kube-rs.git", rev = "698f421652032aec5302eefa1593a4bee0d28b77", features = ["derive"] }
+#kube-runtime = { git = "https://github.com/clux/kube-rs.git", rev = "698f421652032aec5302eefa1593a4bee0d28b77" }
+#prometheus = { git = "https://github.com/clux/rust-prometheus.git", branch = "master" }
+prometheus = { path = "../rust-prometheus" }
diff --git a/src/manager.rs b/src/manager.rs
index 5b12504..a923b43 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -78,6 +78,7 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
     let mut exemplar_labels = HashMap::new();
     exemplar_labels.insert("traceID".into(), trace_id);
     let ex = Exemplar::new_with_labels(duration, exemplar_labels);
+    println!("exemplar: {:?}", ex);
     ctx.get_ref()
         .metrics
         .reconcile_duration
@@ -161,15 +162,17 @@ impl Manager {
         });
 
         let foos = Api::<Foo>::all(client);
-        //foos.get("testfoo").await.expect("please run: cargo run --bin crdgen | kubectl apply -f -");
+        // Ensure CRD is installed before loop-watching
+        let _r = foos.list(&ListParams::default().limit(1))
+            .await
+            .expect("is the crd installed? please run: cargo run --bin crdgen | kubectl apply -f -");
 
+        // All good. Start controller and return its future.
         let drainer = Controller::new(foos, ListParams::default())
             .run(reconcile, error_policy, context)
             .filter_map(|x| async move { std::result::Result::ok(x) })
             .for_each(|_| futures::future::ready(()))
             .boxed();
-        // what we do with the controller stream from .run() ^^ does not matter
-        // but we do need to consume it, hence general printing + return future
 
         (Self { state, metrics }, drainer)
     }
diff --git a/src/telemetry.rs b/src/telemetry.rs
index 1cf0588..eb629d3 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -3,7 +3,7 @@ use crate::{Error, Result};
 ///  Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack
 pub fn get_trace_id() -> String {
     use opentelemetry::trace::TraceContextExt;
-    use tracing_opentelemetry::OpenTelemetrySpanExt; // tracing::Span to opentelemetry::Context // opentelemetry::Context -> opentelemetry::trace::Span
+    use tracing_opentelemetry::OpenTelemetrySpanExt;
     tracing::Span::current()
         .context()
         .span()

From 43c3092228b407896f3cb1aa0deaf511b8994c1e Mon Sep 17 00:00:00 2001
From: clux <sszynrae@gmail.com>
Date: Wed, 31 Mar 2021 23:54:49 +0100
Subject: [PATCH 16/16] upgrade everything

also make exemplars work?
---
 Cargo.lock       | 576 +++++++++++++++++++++--------------------------
 Cargo.toml       |  19 +-
 Dockerfile       |  14 +-
 Makefile         |  12 +-
 src/main.rs      |   1 +
 src/manager.rs   |  16 +-
 src/telemetry.rs |   4 +-
 7 files changed, 288 insertions(+), 354 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 57d1f0b..f6d69cd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7,7 +7,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90673465c6187bd0829116b02be465dc0195a74d7719f76ffff0effef934a92e"
 dependencies = [
  "bitflags",
- "bytes",
+ "bytes 1.0.1",
  "futures-core",
  "futures-sink",
  "log",
@@ -31,7 +31,7 @@ dependencies = [
  "base64",
  "bitflags",
  "brotli2",
- "bytes",
+ "bytes 1.0.1",
  "bytestring",
  "cookie",
  "derive_more",
@@ -173,7 +173,7 @@ dependencies = [
  "actix-web-codegen",
  "ahash",
  "awc",
- "bytes",
+ "bytes 1.0.1",
  "derive_more",
  "either",
  "encoding_rs",
@@ -203,15 +203,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "addr2line"
-version = "0.14.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
-dependencies = [
- "gimli",
-]
-
 [[package]]
 name = "adler"
 version = "1.0.2"
@@ -226,7 +217,7 @@ checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957"
 dependencies = [
  "getrandom",
  "once_cell",
- "version_check 0.9.2",
+ "version_check",
 ]
 
 [[package]]
@@ -247,12 +238,39 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"
+
 [[package]]
 name = "array_tool"
 version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
 
+[[package]]
+name = "async-stream"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.47"
@@ -292,8 +310,8 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "base64",
- "bytes",
- "cfg-if 1.0.0",
+ "bytes 1.0.1",
+ "cfg-if",
  "derive_more",
  "futures-core",
  "log",
@@ -305,20 +323,6 @@ dependencies = [
  "serde_urlencoded",
 ]
 
-[[package]]
-name = "backtrace"
-version = "0.3.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
-dependencies = [
- "addr2line",
- "cfg-if 1.0.0",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
 [[package]]
 name = "base-x"
 version = "0.2.8"
@@ -331,25 +335,6 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
-[[package]]
-name = "bindgen"
-version = "0.51.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75"
-dependencies = [
- "bitflags",
- "cexpr",
- "cfg-if 0.1.10",
- "clang-sys",
- "lazy_static",
- "peeking_take_while",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash",
- "shlex",
-]
-
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -365,15 +350,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "boringssl-src"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c212e3cd725f096e4de48d4ded0909cb62cb92c1a3420ff0e95edb46390a7736"
-dependencies = [
- "cmake",
-]
-
 [[package]]
 name = "brotli-sys"
 version = "0.3.2"
@@ -406,6 +382,12 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
+[[package]]
+name = "bytes"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
+
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -418,7 +400,7 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
 ]
 
 [[package]]
@@ -427,21 +409,6 @@ version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
 
-[[package]]
-name = "cexpr"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
-dependencies = [
- "nom",
-]
-
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -462,26 +429,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "clang-sys"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
-[[package]]
-name = "cmake"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "const_fn"
 version = "0.4.5"
@@ -522,7 +469,7 @@ checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951"
 dependencies = [
  "percent-encoding",
  "time 0.2.25",
- "version_check 0.9.2",
+ "version_check",
 ]
 
 [[package]]
@@ -553,7 +500,7 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -597,7 +544,7 @@ version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "num_cpus",
 ]
 
@@ -638,7 +585,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "dirs-sys-next",
 ]
 
@@ -689,7 +636,7 @@ version = "0.8.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -706,26 +653,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "failure"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
-dependencies = [
- "backtrace",
- "failure_derive",
-]
-
-[[package]]
-name = "failure_derive"
-version = "0.1.8"
+name = "fixedbitset"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "synstructure",
-]
+checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
 
 [[package]]
 name = "flate2"
@@ -733,7 +664,7 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "crc32fast",
  "libc",
  "miniz_oxide",
@@ -869,7 +800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
 dependencies = [
  "typenum",
- "version_check 0.9.2",
+ "version_check",
 ]
 
 [[package]]
@@ -878,69 +809,18 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "wasi",
 ]
 
-[[package]]
-name = "gimli"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
-
-[[package]]
-name = "glob"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
-
-[[package]]
-name = "grpcio"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dfdbacbc27f875402f878928c7e9dc22ee26d01a56a9faf655f7090e7e25109"
-dependencies = [
- "futures",
- "grpcio-sys",
- "libc",
- "log",
- "parking_lot",
- "protobuf",
-]
-
-[[package]]
-name = "grpcio-compiler"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad76ae5de2e2d85fe8f45e072f7909e22fa9ffaf0d53b280825783720dab051c"
-dependencies = [
- "protobuf",
-]
-
-[[package]]
-name = "grpcio-sys"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01cf1bf66e791876cfb406952895d87912eab70e858a5474c3608e5297509eba"
-dependencies = [
- "bindgen",
- "boringssl-src",
- "cc",
- "cmake",
- "libc",
- "libz-sys",
- "pkg-config",
- "walkdir",
-]
-
 [[package]]
 name = "h2"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -959,6 +839,15 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
+[[package]]
+name = "heck"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -974,7 +863,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "fnv",
  "itoa",
 ]
@@ -985,7 +874,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "http",
 ]
 
@@ -1016,10 +905,11 @@ version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "futures-channel",
  "futures-core",
  "futures-util",
+ "h2",
  "http",
  "http-body",
  "httparse",
@@ -1051,7 +941,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "hyper",
  "native-tls",
  "tokio",
@@ -1091,7 +981,25 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
+]
+
+[[package]]
+name = "itertools"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
+dependencies = [
+ "either",
 ]
 
 [[package]]
@@ -1129,7 +1037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bcc1f973542059e6d5a6d63de6a9539d0ec784f82b2327f3c1915d33200bc6a4"
 dependencies = [
  "base64",
- "bytes",
+ "bytes 1.0.1",
  "chrono",
  "serde",
  "serde-value",
@@ -1138,10 +1046,12 @@ dependencies = [
 
 [[package]]
 name = "kube"
-version = "0.51.0"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65b2fbe85ad92f8435a0f52072f55bd7f0843e082c8e461e56d1ce29440554c8"
 dependencies = [
  "base64",
- "bytes",
+ "bytes 1.0.1",
  "chrono",
  "dirs-next",
  "either",
@@ -1172,7 +1082,9 @@ dependencies = [
 
 [[package]]
 name = "kube-derive"
-version = "0.51.0"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dca5161aebec14aa62448469b4c128d165e70da503d43813702a23deeff9612"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -1183,7 +1095,9 @@ dependencies = [
 
 [[package]]
 name = "kube-runtime"
-version = "0.51.0"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdbf7b51e6a4984e929dab0bd13b61592b750a3385e298aaa9100c4830205a12"
 dependencies = [
  "dashmap",
  "derivative",
@@ -1216,28 +1130,6 @@ version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a"
 
-[[package]]
-name = "libloading"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
-dependencies = [
- "cc",
- "winapi",
-]
-
-[[package]]
-name = "libz-sys"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "linked-hash-map"
 version = "0.5.4"
@@ -1259,7 +1151,7 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -1322,6 +1214,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "multimap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+
 [[package]]
 name = "native-tls"
 version = "0.2.7"
@@ -1340,16 +1238,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "nom"
-version = "4.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
-dependencies = [
- "memchr",
- "version_check 0.1.5",
-]
-
 [[package]]
 name = "ntapi"
 version = "0.3.6"
@@ -1388,12 +1276,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "object"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
-
 [[package]]
 name = "once_cell"
 version = "1.7.2"
@@ -1413,7 +1295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70"
 dependencies = [
  "bitflags",
- "cfg-if 1.0.0",
+ "cfg-if",
  "foreign-types",
  "lazy_static",
  "libc",
@@ -1453,6 +1335,8 @@ dependencies = [
  "pin-project 1.0.5",
  "rand",
  "thiserror",
+ "tokio",
+ "tokio-stream",
 ]
 
 [[package]]
@@ -1463,13 +1347,12 @@ checksum = "ff889e68a02000126fa635a311abc329ce128c576b7dddc4cd3270f3c240afd0"
 dependencies = [
  "async-trait",
  "futures",
- "grpcio",
  "opentelemetry",
- "protobuf",
- "protobuf-codegen",
- "protoc-grpcio",
+ "prost 0.7.0",
  "thiserror",
  "tokio",
+ "tonic",
+ "tonic-build",
 ]
 
 [[package]]
@@ -1498,7 +1381,7 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "instant",
  "libc",
  "redox_syscall",
@@ -1506,12 +1389,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "peeking_take_while"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
-
 [[package]]
 name = "pem"
 version = "0.8.3"
@@ -1529,6 +1406,16 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "petgraph"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.27"
@@ -1617,8 +1504,9 @@ dependencies = [
 [[package]]
 name = "prometheus"
 version = "0.12.0"
+source = "git+https://github.com/clux/rust-prometheus.git?branch=master#d31ee3ec483579dad27b119bcee3887deeed5802"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "fnv",
  "lazy_static",
  "memchr",
@@ -1628,44 +1516,85 @@ dependencies = [
 ]
 
 [[package]]
-name = "protobuf"
-version = "2.22.0"
+name = "prost"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86"
+checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212"
+dependencies = [
+ "bytes 0.5.6",
+ "prost-derive 0.6.1",
+]
 
 [[package]]
-name = "protobuf-codegen"
-version = "2.22.0"
+name = "prost"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8217a1652dbc91d19c509c558234145faed729191a966896414e5889f62d543"
+checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2"
 dependencies = [
- "protobuf",
+ "bytes 1.0.1",
+ "prost-derive 0.7.0",
 ]
 
 [[package]]
-name = "protoc"
-version = "2.22.0"
+name = "prost-build"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57628f7456fe5fff1591901fb8b594e1584750cb857da23b65c456b42670524d"
+checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26"
 dependencies = [
+ "bytes 0.5.6",
+ "heck",
+ "itertools 0.8.2",
  "log",
+ "multimap",
+ "petgraph",
+ "prost 0.6.1",
+ "prost-types",
+ "tempfile",
  "which",
 ]
 
 [[package]]
-name = "protoc-grpcio"
-version = "2.0.0"
+name = "prost-derive"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af5d484461b7f14e103742f21014cc519f6f3711b05d50b3e86d912467a3f8b7"
+checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72"
 dependencies = [
- "failure",
- "grpcio-compiler",
- "protobuf",
- "protobuf-codegen",
- "protoc",
- "tempfile",
+ "anyhow",
+ "itertools 0.8.2",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4"
+dependencies = [
+ "anyhow",
+ "itertools 0.9.0",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa"
+dependencies = [
+ "bytes 0.5.6",
+ "prost 0.6.1",
 ]
 
+[[package]]
+name = "protobuf"
+version = "2.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86"
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -1777,18 +1706,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "rustc-demangle"
-version = "0.1.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
-
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -1804,15 +1721,6 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
-[[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
-
 [[package]]
 name = "schannel"
 version = "0.1.19"
@@ -1976,7 +1884,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f"
 dependencies = [
  "block-buffer",
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpuid-bool",
  "digest",
  "opaque-debug",
@@ -1997,12 +1905,6 @@ dependencies = [
  "lazy_static",
 ]
 
-[[package]]
-name = "shlex"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
-
 [[package]]
 name = "signal-hook-registry"
 version = "1.3.0"
@@ -2053,7 +1955,7 @@ version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "winapi",
 ]
@@ -2064,7 +1966,7 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2beb4d1860a61f571530b3f855a1b538d0200f7871c63331ecd6f17b1f014f8"
 dependencies = [
- "version_check 0.9.2",
+ "version_check",
 ]
 
 [[package]]
@@ -2139,25 +2041,13 @@ dependencies = [
  "unicode-xid",
 ]
 
-[[package]]
-name = "synstructure"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "unicode-xid",
-]
-
 [[package]]
 name = "tempfile"
 version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "rand",
  "redox_syscall",
@@ -2225,7 +2115,7 @@ dependencies = [
  "standback",
  "stdweb",
  "time-macros",
- "version_check 0.9.2",
+ "version_check",
  "winapi",
 ]
 
@@ -2274,7 +2164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a"
 dependencies = [
  "autocfg",
- "bytes",
+ "bytes 1.0.1",
  "libc",
  "memchr",
  "mio",
@@ -2318,13 +2208,24 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e177a5d8c3bf36de9ebe6d58537d8879e964332f93fb3339e43f618c81361af0"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b"
 dependencies = [
- "bytes",
+ "bytes 1.0.1",
  "futures-core",
  "futures-sink",
  "log",
@@ -2333,6 +2234,47 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91491e5f15431f2189ec8c1f9dcbadac949450399c22c912ceae9570eb472f61"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "base64",
+ "bytes 1.0.1",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "percent-encoding",
+ "pin-project 1.0.5",
+ "prost 0.7.0",
+ "prost-derive 0.7.0",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19970cf58f3acc820962be74c4021b8bbc8e8a1c4e3a02095d0aa60cde5f3633"
+dependencies = [
+ "proc-macro2",
+ "prost-build",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tower"
 version = "0.4.6"
@@ -2341,8 +2283,12 @@ checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap",
  "pin-project 1.0.5",
+ "rand",
+ "slab",
  "tokio",
+ "tokio-stream",
  "tokio-util",
  "tower-layer",
  "tower-service",
@@ -2367,7 +2313,7 @@ version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -2490,6 +2436,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"
@@ -2514,29 +2466,12 @@ version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
 
-[[package]]
-name = "version_check"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
-
 [[package]]
 name = "version_check"
 version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
 
-[[package]]
-name = "walkdir"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
-dependencies = [
- "same-file",
- "winapi",
- "winapi-util",
-]
-
 [[package]]
 name = "want"
 version = "0.3.0"
@@ -2559,7 +2494,7 @@ version = "0.2.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "wasm-bindgen-macro",
 ]
 
@@ -2609,12 +2544,11 @@ checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1"
 
 [[package]]
 name = "which"
-version = "4.0.2"
+version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87c14ef7e1b8b8ecfc75d5eca37949410046e66f15d185c01d70824f1f8111ef"
+checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
 dependencies = [
  "libc",
- "thiserror",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index b758e99..1a7b9c1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,8 +25,8 @@ actix-web = "4.0.0-beta.1"
 futures = "0.3.13"
 tokio = { version = "1.1.0", features = ["macros"] }
 #prometheus = "0.12.0"
-#kube = { version = "0.51.0", features = ["derive"] }
-#kube-runtime = "0.51.0"
+kube = { version = "0.52.0", features = ["derive"] }
+kube-runtime = "0.52.0"
 k8s-openapi = { version = "0.11.0", features = ["v1_19"], default-features=false }
 serde = { version = "1.0.123", features = ["derive"] }
 serde_json = "1.0.64"
@@ -38,14 +38,15 @@ tracing = "0.1.25"
 tracing-subscriber = { version = "0.2.16", features = ["json"] }
 tracing-futures = "0.2.5"
 tracing-opentelemetry = "0.11.0"
-opentelemetry = { version = "0.12.0", features = ["trace"] }
-opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "grpc-sys", "openssl"], default-features=false }
-#opentelemetry-otlp = { version = "0.5.0", features = ["tokio", "async"] }
+opentelemetry = { version = "0.12.0", features = ["trace", "tokio-support"] }
+# NB: in 0.13 s/tokio-support/rt-tokio/
+opentelemetry-otlp = { version = "0.5.0", features = ["tokio"] }
+# NB: otlp 0.6 and otel 0.13 blocked by https://github.com/tokio-rs/tracing/pull/1322
 
 # local dev
-kube = { path = "../kube-rs/kube", features = ["derive"] }
-kube-runtime = { path = "../kube-rs/kube-runtime" }
+#kube = { path = "../kube-rs/kube", features = ["derive"] }
+#kube-runtime = { path = "../kube-rs/kube-runtime" }
 #kube = { git = "https://github.com/clux/kube-rs.git", rev = "698f421652032aec5302eefa1593a4bee0d28b77", features = ["derive"] }
 #kube-runtime = { git = "https://github.com/clux/kube-rs.git", rev = "698f421652032aec5302eefa1593a4bee0d28b77" }
-#prometheus = { git = "https://github.com/clux/rust-prometheus.git", branch = "master" }
-prometheus = { path = "../rust-prometheus" }
+prometheus = { git = "https://github.com/clux/rust-prometheus.git", branch = "master" }
+#prometheus = { path = "../rust-prometheus" }
diff --git a/Dockerfile b/Dockerfile
index 3fd9dca..38a57d3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,4 @@
-FROM rust:1.50 as builder
-WORKDIR /usr/src/controller
-RUN apt-get update && apt-get install cmake protobuf-compiler -y
-COPY . .
-RUN cargo install --locked --path .
-
-FROM debian:buster-slim
-RUN apt-get update && apt-get install -y openssl && rm -rf /var/lib/apt/lists/*
-COPY --from=builder /usr/local/cargo/bin/controller /usr/local/bin/controller
-CMD ["controller"]
+FROM gcr.io/distroless/static:nonroot
+COPY --chown=nonroot:nonroot ./controller /app/
+EXPOSE 8080
+ENTRYPOINT ["/app/controller"]
diff --git a/Makefile b/Makefile
index 526d1ca..6af3c24 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,17 @@ forward-tempo:
 	kubectl port-forward -n monitoring service/grafana-agent-traces 55680:55680
 
 run:
-	OPENTELEMETRY_ENDPOINT_URL=0.0.0.0:55680 RUST_LOG=debug,kube=trace,hyper=info,tower=info cargo run
+	OPENTELEMETRY_ENDPOINT_URL=https://0.0.0.0:55680 RUST_LOG=info,kube=trace,controller=debug cargo run
+
+compile:
+	docker run --rm \
+		-v cargo-cache:/root/.cargo \
+		-v $$PWD:/volume \
+		-w /volume \
+		-it clux/muslrust:stable \
+		cargo build --release
+	sudo chown $$USER:$$USER -R target
+	mv target/x86_64-unknown-linux-musl/release/controller .
 
 build:
 	docker build -t $(REPO)/$(NAME):$(VERSION) .
diff --git a/src/main.rs b/src/main.rs
index a975023..4f5bf5b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -37,6 +37,7 @@ async fn main() -> Result<()> {
 
     let (tracer, _uninstall) = opentelemetry_otlp::new_pipeline()
         .with_endpoint(&otlp_endpoint)
+        // TODO: opentelemetry_otlp::new_pipeline().with_tonic().install_batch() in 0.6
         .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
             opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
                 "service.name",
diff --git a/src/manager.rs b/src/manager.rs
index a923b43..2358a44 100644
--- a/src/manager.rs
+++ b/src/manager.rs
@@ -2,7 +2,7 @@ use crate::{telemetry, Error, Result};
 use chrono::prelude::*;
 use futures::{future::BoxFuture, FutureExt, StreamExt};
 use kube::{
-    api::{Api, ListParams, Meta, Patch, PatchParams},
+    api::{Api, ListParams, Patch, PatchParams, Resource},
     client::Client,
     CustomResource,
 };
@@ -55,8 +55,8 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
 
     let client = ctx.get_ref().client.clone();
     ctx.get_ref().state.write().await.last_event = Utc::now();
-    let name = Meta::name(&foo);
-    let ns = Meta::namespace(&foo).expect("foo is namespaced");
+    let name = Resource::name(&foo);
+    let ns = Resource::namespace(&foo).expect("foo is namespaced");
     let foos: Api<Foo> = Api::namespaced(client, &ns);
 
     let new_status = Patch::Apply(json!({
@@ -73,7 +73,8 @@ async fn reconcile(foo: Foo, ctx: Context<Data>) -> Result<ReconcilerAction, Err
         .await
         .map_err(Error::KubeError)?;
 
-    let duration = duration_to_seconds(start.elapsed());
+    let duration = start.elapsed().as_millis() as f64;
+    println!("duration= {}", duration);
 
     let mut exemplar_labels = HashMap::new();
     exemplar_labels.insert("traceID".into(), trace_id);
@@ -187,10 +188,3 @@ impl Manager {
         self.state.read().await.clone()
     }
 }
-
-/// Convert `Duration` to seconds for `Histogram`s
-#[inline]
-pub fn duration_to_seconds(d: std::time::Duration) -> f64 {
-    let nanos = f64::from(d.subsec_nanos()) / 1e9;
-    d.as_secs() as f64 + nanos
-}
diff --git a/src/telemetry.rs b/src/telemetry.rs
index eb629d3..5929787 100644
--- a/src/telemetry.rs
+++ b/src/telemetry.rs
@@ -2,8 +2,8 @@ use crate::{Error, Result};
 
 ///  Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack
 pub fn get_trace_id() -> String {
-    use opentelemetry::trace::TraceContextExt;
-    use tracing_opentelemetry::OpenTelemetrySpanExt;
+    use opentelemetry::trace::TraceContextExt; // opentelemetry::Context -> opentelemetry::trace::Span
+    use tracing_opentelemetry::OpenTelemetrySpanExt; // tracing::Span to opentelemetry::Context
     tracing::Span::current()
         .context()
         .span()