From d59c4806014c6e643bc9e7068eec2148ed2e2ea2 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Wed, 9 Apr 2025 19:13:14 -0400 Subject: [PATCH 1/3] Support xgress flow control from SDK. Move xgress impl to SDK. Fixes #702 --- CHANGELOG.md | 65 +++++++ edgexg/impl.go | 16 ++ example/go.mod | 7 +- example/go.sum | 12 +- example/influxdb-client-go/go.mod | 7 +- example/influxdb-client-go/go.sum | 12 +- go.mod | 8 +- go.sum | 12 +- inspect/inspect.go | 7 + pb/edge_client_pb/edge_client.pb.go | 259 ++++++++++++++++------------ pb/edge_client_pb/edge_client.proto | 151 ++++++++-------- version | 2 +- xgress/circuit_inspections.go | 43 +++-- xgress/link_send_buffer.go | 13 +- xgress/messages.go | 16 ++ xgress/payload_ingester.go | 2 +- xgress/retransmitter.go | 1 + xgress/xgress.go | 27 ++- ziti/config.go | 23 ++- ziti/contexts.go | 21 ++- ziti/edge/channel.go | 8 +- ziti/edge/conn.go | 23 ++- ziti/edge/messages.go | 92 ++++++---- ziti/edge/msg_mux.go | 65 +++++++ ziti/edge/network/conn.go | 241 +++++++++++++++++++++++--- ziti/edge/network/conn_test.go | 30 +++- ziti/edge/network/factory.go | 22 ++- ziti/edge/network/listener.go | 2 + ziti/edge/network/msg_timer.go | 127 ++++++++++++++ ziti/edge/network/seq.go | 13 +- ziti/edge/network/seq_test.go | 24 ++- ziti/edge/network/xg_adapter.go | 145 ++++++++++++++++ ziti/options.go | 26 +++ ziti/sdkinfo/build_info.go | 2 +- ziti/xg_env.go | 39 +++++ ziti/ziti.go | 34 +++- 36 files changed, 1249 insertions(+), 348 deletions(-) create mode 100644 edgexg/impl.go create mode 100644 inspect/inspect.go create mode 100644 ziti/edge/network/msg_timer.go create mode 100644 ziti/edge/network/xg_adapter.go create mode 100644 ziti/xg_env.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 820c899e..ad7a7df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,68 @@ +# Release notes 1.1.0 + +## What's New + +* Experimental support for sdk based flow-control +* Config change for multiple underlays + +## SDK Flow Control + +If the router being connected to supports it, the sdk can now manage flow control +instead of delegating that to the router. This is mostly importing when running +multiple simultaneous circuits throught the SDK. When running multiple circuits, +a slow circuit can get stalled at the router because of flow control back-pressure. +This then back-pressures all circuits from the SDK to that router. + +By moving the flow-control to the SDK, a slow circuit will not negatively other +circuits to the same router. This is currently enabled in the `DialOptions`. + +``` +t := true +dialOptions := &ziti.DialOptions{ + ConnectTimeout: wf.ConnectTimeout, + SdkFlowControl: &t, +} +``` + +As this is an experimental feature, the configuration may change or be removed +in the future. + +## Config Changes + +The multi-underlay configuration has changed. There are now two settings. + +``` +// If set to a number greater than one, the sdk will attempt to create multiple connections to edge routers. +// This configuration value should not be considered part of the stable API yet. It currently defaults to one, +// but it may default to a larger number at some point in the future or be removed. If set to zero, it will +// be reset to one. +MaxDefaultConnections uint32 `json:"-"` + +// If set to a number greater than zero, the sdk will attempt to create one or more separate connection to +// each edge routers for control plane data, such as dials. This configuration value should not be considered +// part of the stable API yet. It currently defaults to zero, but it may default to 1 at some point in the future +// or be removed. +MaxControlConnections uint32 `json:"-"` +``` + +The old `EnableSeparateControlPlaneConnection` setting is gone. Set `MaxControlConnections` to 1 to enable +separation of control plane data. + +Note that while present, the `MaxDefaultConnections` should not be used yet. + +## Issues Fixed and Dependency Updates + +* github.com/openziti/sdk-golang: [v1.0.2 -> v1.1.0](https://github.com/openziti/sdk-golang/compare/v1.0.2...v1.1.0) + * [Issue #702](https://github.com/openziti/sdk-golang/issues/702) - [Go SDK] Support xgress flow control from the SDK + +* github.com/openziti/channel/v4: [v4.0.4 -> v4.0.6](https://github.com/openziti/channel/compare/v4.0.4...v4.0.6) + * [Issue #182](https://github.com/openziti/channel/issues/182) - MultiListener can deadlock + * [Issue #180](https://github.com/openziti/channel/issues/180) - Add GetUserData to Channel interface + +* github.com/openziti/identity: [v1.0.100 -> v1.0.101](https://github.com/openziti/identity/compare/v1.0.100...v1.0.101) + * [Issue #64](https://github.com/openziti/identity/issues/64) - Support a way to check if a cert/serverCert can be saved + + # Release notes 1.0.2 ## Issues Fixed and Dependency Updates diff --git a/edgexg/impl.go b/edgexg/impl.go new file mode 100644 index 00000000..9ae82355 --- /dev/null +++ b/edgexg/impl.go @@ -0,0 +1,16 @@ +package edgexg + +import "github.com/openziti/sdk-golang/ziti/edge" + +const ( + PayloadFlagsHeader uint8 = 0x10 +) + +// headers to pass through fabric to the other side +var HeadersToFabric = map[int32]uint8{ + edge.FlagsHeader: PayloadFlagsHeader, +} + +var HeadersFromFabric = map[uint8]int32{ + PayloadFlagsHeader: edge.FlagsHeader, +} diff --git a/example/go.mod b/example/go.mod index e8ad899c..ad69a987 100644 --- a/example/go.mod +++ b/example/go.mod @@ -37,6 +37,7 @@ require ( github.com/disintegration/imaging v1.6.2 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect @@ -79,10 +80,10 @@ require ( github.com/muhlemmer/gu v0.3.1 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/openziti/channel/v4 v4.0.4 // indirect + github.com/openziti/channel/v4 v4.0.6 // indirect github.com/openziti/edge-api v0.26.42 // indirect - github.com/openziti/identity v1.0.100 // indirect - github.com/openziti/metrics v1.4.0 // indirect + github.com/openziti/identity v1.0.101 // indirect + github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 // indirect github.com/openziti/secretstream v0.1.32 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect diff --git a/example/go.sum b/example/go.sum index 116e6ea8..a093425d 100644 --- a/example/go.sum +++ b/example/go.sum @@ -358,16 +358,16 @@ github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openziti/channel/v4 v4.0.4 h1:t/hMkKuZ2kqgenztcKjhdgrO/a5Kg2xZNNWB217UMvc= -github.com/openziti/channel/v4 v4.0.4/go.mod h1:ekqQwL27nKufMN8nB0jO+bjj9QfOCiPM0GCPllUSK60= +github.com/openziti/channel/v4 v4.0.6 h1:pcIn9xNnQXnenHoaOt347nG0dmnsisomnEIPq4F5YHI= +github.com/openziti/channel/v4 v4.0.6/go.mod h1:sU3S/MToQHES1V22O+xtQbX0QSteoMo6RzQiEMfir+w= github.com/openziti/edge-api v0.26.42 h1:Wi/BUttSUvedT9XGht7vi/zI/TNGc3ApvjkAviWhauA= github.com/openziti/edge-api v0.26.42/go.mod h1:sYHVpm26Jr1u7VooNJzTb2b2nGSlmCHMnbGC8XfWSng= github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8prYnehm8o= github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= -github.com/openziti/identity v1.0.100 h1:FTkbhykDCMw1z/wxEeDfmq1aBp2tLRjZ3ggioRT4pg8= -github.com/openziti/identity v1.0.100/go.mod h1:E4SHqfXaZldDCo/GIdSD/Xg61obuolWYg9Qe8lqGUrQ= -github.com/openziti/metrics v1.4.0 h1:uZALaZINoTFqRE3XcVZ/xGuvXJpDhn/a0kxaX586lzE= -github.com/openziti/metrics v1.4.0/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= +github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/runzmd v1.0.33 h1:tOyjRoUuVXIo1z1pNU32jALWkMmhzsSaDrhLtuOn3Ts= github.com/openziti/runzmd v1.0.33/go.mod h1:8c/uvZR/XWXQNllTq6LuTpfKL2DTNxfI2X2wYhgRwik= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= diff --git a/example/influxdb-client-go/go.mod b/example/influxdb-client-go/go.mod index 3e6b5a8d..f0edf8bf 100644 --- a/example/influxdb-client-go/go.mod +++ b/example/influxdb-client-go/go.mod @@ -27,6 +27,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/deepmap/oapi-codegen v1.15.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect @@ -93,11 +94,11 @@ require ( github.com/muhlemmer/gu v0.3.1 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/openziti/channel/v4 v4.0.4 // indirect + github.com/openziti/channel/v4 v4.0.6 // indirect github.com/openziti/edge-api v0.26.42 // indirect github.com/openziti/foundation/v2 v2.0.59 // indirect - github.com/openziti/identity v1.0.100 // indirect - github.com/openziti/metrics v1.4.0 // indirect + github.com/openziti/identity v1.0.101 // indirect + github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 // indirect github.com/openziti/secretstream v0.1.32 // indirect github.com/openziti/transport/v2 v2.0.168 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect diff --git a/example/influxdb-client-go/go.sum b/example/influxdb-client-go/go.sum index 0eef5127..585dca72 100644 --- a/example/influxdb-client-go/go.sum +++ b/example/influxdb-client-go/go.sum @@ -413,16 +413,16 @@ github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openziti/channel/v4 v4.0.4 h1:t/hMkKuZ2kqgenztcKjhdgrO/a5Kg2xZNNWB217UMvc= -github.com/openziti/channel/v4 v4.0.4/go.mod h1:ekqQwL27nKufMN8nB0jO+bjj9QfOCiPM0GCPllUSK60= +github.com/openziti/channel/v4 v4.0.6 h1:pcIn9xNnQXnenHoaOt347nG0dmnsisomnEIPq4F5YHI= +github.com/openziti/channel/v4 v4.0.6/go.mod h1:sU3S/MToQHES1V22O+xtQbX0QSteoMo6RzQiEMfir+w= github.com/openziti/edge-api v0.26.42 h1:Wi/BUttSUvedT9XGht7vi/zI/TNGc3ApvjkAviWhauA= github.com/openziti/edge-api v0.26.42/go.mod h1:sYHVpm26Jr1u7VooNJzTb2b2nGSlmCHMnbGC8XfWSng= github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8prYnehm8o= github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= -github.com/openziti/identity v1.0.100 h1:FTkbhykDCMw1z/wxEeDfmq1aBp2tLRjZ3ggioRT4pg8= -github.com/openziti/identity v1.0.100/go.mod h1:E4SHqfXaZldDCo/GIdSD/Xg61obuolWYg9Qe8lqGUrQ= -github.com/openziti/metrics v1.4.0 h1:uZALaZINoTFqRE3XcVZ/xGuvXJpDhn/a0kxaX586lzE= -github.com/openziti/metrics v1.4.0/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= +github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= github.com/openziti/secretstream v0.1.32/go.mod h1:8YaIbjyMwBeKQ7eOYcoVPKHT10u+4OVPXpnZAeDzC6o= github.com/openziti/transport/v2 v2.0.168 h1:1Anf7X+4xmSKQ12GdPJFhoMZi04QxgD4MJu3agFc1R4= diff --git a/go.mod b/go.mod index ee6c7a06..aedc8a08 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.23.1 require ( github.com/Jeffail/gabs v1.4.0 github.com/cenkalti/backoff/v4 v4.3.0 + github.com/emirpasic/gods v1.18.1 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa github.com/go-openapi/runtime v0.28.0 github.com/go-openapi/strfmt v0.23.0 @@ -17,11 +18,11 @@ require ( github.com/michaelquigley/pfxlog v0.6.10 github.com/mitchellh/go-ps v1.0.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openziti/channel/v4 v4.0.4 + github.com/openziti/channel/v4 v4.0.6 github.com/openziti/edge-api v0.26.42 github.com/openziti/foundation/v2 v2.0.59 - github.com/openziti/identity v1.0.100 - github.com/openziti/metrics v1.4.0 + github.com/openziti/identity v1.0.101 + github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 github.com/openziti/secretstream v0.1.32 github.com/openziti/transport/v2 v2.0.168 github.com/orcaman/concurrent-map/v2 v2.0.1 @@ -40,7 +41,6 @@ require ( require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 82488524..55758ec5 100644 --- a/go.sum +++ b/go.sum @@ -299,16 +299,16 @@ github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openziti/channel/v4 v4.0.4 h1:t/hMkKuZ2kqgenztcKjhdgrO/a5Kg2xZNNWB217UMvc= -github.com/openziti/channel/v4 v4.0.4/go.mod h1:ekqQwL27nKufMN8nB0jO+bjj9QfOCiPM0GCPllUSK60= +github.com/openziti/channel/v4 v4.0.6 h1:pcIn9xNnQXnenHoaOt347nG0dmnsisomnEIPq4F5YHI= +github.com/openziti/channel/v4 v4.0.6/go.mod h1:sU3S/MToQHES1V22O+xtQbX0QSteoMo6RzQiEMfir+w= github.com/openziti/edge-api v0.26.42 h1:Wi/BUttSUvedT9XGht7vi/zI/TNGc3ApvjkAviWhauA= github.com/openziti/edge-api v0.26.42/go.mod h1:sYHVpm26Jr1u7VooNJzTb2b2nGSlmCHMnbGC8XfWSng= github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8prYnehm8o= github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= -github.com/openziti/identity v1.0.100 h1:FTkbhykDCMw1z/wxEeDfmq1aBp2tLRjZ3ggioRT4pg8= -github.com/openziti/identity v1.0.100/go.mod h1:E4SHqfXaZldDCo/GIdSD/Xg61obuolWYg9Qe8lqGUrQ= -github.com/openziti/metrics v1.4.0 h1:uZALaZINoTFqRE3XcVZ/xGuvXJpDhn/a0kxaX586lzE= -github.com/openziti/metrics v1.4.0/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= +github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= +github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= github.com/openziti/secretstream v0.1.32/go.mod h1:8YaIbjyMwBeKQ7eOYcoVPKHT10u+4OVPXpnZAeDzC6o= github.com/openziti/transport/v2 v2.0.168 h1:1Anf7X+4xmSKQ12GdPJFhoMZi04QxgD4MJu3agFc1R4= diff --git a/inspect/inspect.go b/inspect/inspect.go new file mode 100644 index 00000000..608a143e --- /dev/null +++ b/inspect/inspect.go @@ -0,0 +1,7 @@ +package inspect + +type SdkInspectResponse struct { + Errors []string `json:"errors"` + Success bool `json:"success"` + Values map[string]any `json:"values"` +} diff --git a/pb/edge_client_pb/edge_client.pb.go b/pb/edge_client_pb/edge_client.pb.go index 07c8f77c..3d37c1a3 100644 --- a/pb/edge_client_pb/edge_client.pb.go +++ b/pb/edge_client_pb/edge_client.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.31.0 // protoc v3.21.12 // source: edge_client.proto @@ -24,10 +24,15 @@ const ( type ContentType int32 const ( - ContentType_Hello ContentType = 0 - ContentType_Ping ContentType = 1 - ContentType_Result ContentType = 2 - ContentType_Latency ContentType = 3 + ContentType_Hello ContentType = 0 + ContentType_Ping ContentType = 1 + ContentType_Result ContentType = 2 + ContentType_Latency ContentType = 3 + // use the same types as xgress uses on links + ContentType_XgPayloadType ContentType = 1100 + ContentType_XgAcknowledgementType ContentType = 1101 + ContentType_XgControlType ContentType = 1102 + ContentType_XgCloseType ContentType = 1103 ContentType_ConnectType ContentType = 60783 ContentType_StateConnectedType ContentType = 60784 ContentType_StateClosedType ContentType = 60785 @@ -49,6 +54,8 @@ const ( ContentType_UpdateTokenSuccessType ContentType = 60801 ContentType_UpdateTokenFailureType ContentType = 60802 ContentType_UpdateTokenType ContentType = 60803 + ContentType_InspectRequest ContentType = 60804 + ContentType_InspectResponse ContentType = 60805 ContentType_PostureResponseType ContentType = 10800 ContentType_PostureResponseSuccessType ContentType = 10801 ) @@ -60,6 +67,10 @@ var ( 1: "Ping", 2: "Result", 3: "Latency", + 1100: "XgPayloadType", + 1101: "XgAcknowledgementType", + 1102: "XgControlType", + 1103: "XgCloseType", 60783: "ConnectType", 60784: "StateConnectedType", 60785: "StateClosedType", @@ -81,6 +92,8 @@ var ( 60801: "UpdateTokenSuccessType", 60802: "UpdateTokenFailureType", 60803: "UpdateTokenType", + 60804: "InspectRequest", + 60805: "InspectResponse", 10800: "PostureResponseType", 10801: "PostureResponseSuccessType", } @@ -89,6 +102,10 @@ var ( "Ping": 1, "Result": 2, "Latency": 3, + "XgPayloadType": 1100, + "XgAcknowledgementType": 1101, + "XgControlType": 1102, + "XgCloseType": 1103, "ConnectType": 60783, "StateConnectedType": 60784, "StateClosedType": 60785, @@ -110,6 +127,8 @@ var ( "UpdateTokenSuccessType": 60801, "UpdateTokenFailureType": 60802, "UpdateTokenType": 60803, + "InspectRequest": 60804, + "InspectResponse": 60805, "PostureResponseType": 10800, "PostureResponseSuccessType": 10801, } @@ -175,6 +194,10 @@ const ( HeaderId_ConnectionMarker HeaderId = 1025 HeaderId_CircuitId HeaderId = 1026 HeaderId_StickinessToken HeaderId = 1027 + HeaderId_UseXgressToSdk HeaderId = 1028 + HeaderId_XgressCtrlId HeaderId = 1029 + HeaderId_XgressAddress HeaderId = 1030 + HeaderId_InspectRequestedValues HeaderId = 1031 ) // Enum value maps for HeaderId. @@ -210,6 +233,10 @@ var ( 1025: "ConnectionMarker", 1026: "CircuitId", 1027: "StickinessToken", + 1028: "UseXgressToSdk", + 1029: "XgressCtrlId", + 1030: "XgressAddress", + 1031: "InspectRequestedValues", } HeaderId_value = map[string]int32{ "ZER0": 0, @@ -242,6 +269,10 @@ var ( "ConnectionMarker": 1025, "CircuitId": 1026, "StickinessToken": 1027, + "UseXgressToSdk": 1028, + "XgressCtrlId": 1029, + "XgressAddress": 1030, + "InspectRequestedValues": 1031, } ) @@ -1202,110 +1233,124 @@ var file_edge_client_proto_rawDesc = []byte{ 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x2a, 0xd3, - 0x04, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x2a, 0xd6, + 0x05, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x02, 0x12, - 0x0b, 0x0a, 0x07, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0b, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xef, 0xda, 0x03, 0x12, - 0x18, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf0, 0xda, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf1, 0xda, 0x03, - 0x12, 0x0e, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf2, 0xda, 0x03, - 0x12, 0x0e, 0x0a, 0x08, 0x44, 0x69, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf3, 0xda, 0x03, - 0x12, 0x15, 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x6c, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, - 0x79, 0x70, 0x65, 0x10, 0xf4, 0xda, 0x03, 0x12, 0x14, 0x0a, 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x46, - 0x61, 0x69, 0x6c, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf5, 0xda, 0x03, 0x12, 0x0e, 0x0a, - 0x08, 0x42, 0x69, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf6, 0xda, 0x03, 0x12, 0x10, 0x0a, - 0x0a, 0x55, 0x6e, 0x62, 0x69, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf7, 0xda, 0x03, 0x12, - 0x1b, 0x0a, 0x15, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, - 0x6e, 0x64, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf8, 0xda, 0x03, 0x12, 0x0f, 0x0a, 0x09, - 0x50, 0x72, 0x6f, 0x62, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf9, 0xda, 0x03, 0x12, 0x14, 0x0a, - 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, - 0xfa, 0xda, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfb, 0xda, 0x03, 0x12, 0x14, 0x0a, 0x0e, 0x54, 0x72, - 0x61, 0x63, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfc, 0xda, 0x03, - 0x12, 0x1c, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfd, 0xda, 0x03, 0x12, 0x18, - 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x10, 0xfe, 0xda, 0x03, 0x12, 0x19, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x6e, - 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x10, - 0xff, 0xda, 0x03, 0x12, 0x11, 0x0a, 0x0b, 0x42, 0x69, 0x6e, 0x64, 0x53, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x10, 0x80, 0xdb, 0x03, 0x12, 0x1c, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, - 0x10, 0x81, 0xdb, 0x03, 0x12, 0x1c, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, 0x82, - 0xdb, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, 0x83, 0xdb, 0x03, 0x12, 0x18, 0x0a, 0x13, 0x50, 0x6f, 0x73, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x10, 0xb0, 0x54, 0x12, 0x1f, 0x0a, 0x1a, 0x50, 0x6f, 0x73, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, - 0x65, 0x10, 0xb1, 0x54, 0x2a, 0xa8, 0x04, 0x0a, 0x08, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x49, - 0x64, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x45, 0x52, 0x30, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x04, 0x55, - 0x55, 0x49, 0x44, 0x10, 0x80, 0x01, 0x12, 0x0b, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x64, - 0x10, 0xe8, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x53, 0x65, 0x71, 0x10, 0xe9, 0x07, 0x12, 0x11, 0x0a, - 0x0c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xea, 0x07, - 0x12, 0x0e, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x10, 0xeb, 0x07, - 0x12, 0x09, 0x0a, 0x04, 0x43, 0x6f, 0x73, 0x74, 0x10, 0xec, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x50, - 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x10, 0xed, 0x07, 0x12, 0x17, 0x0a, 0x12, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x10, 0xee, 0x07, 0x12, 0x1d, 0x0a, 0x18, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, - 0x74, 0x6f, 0x72, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x10, 0xef, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, - 0x10, 0xf0, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4d, 0x65, 0x74, - 0x68, 0x6f, 0x64, 0x10, 0xf1, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x10, - 0xf2, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x41, 0x70, 0x70, 0x44, 0x61, 0x74, 0x61, 0x10, 0xf3, 0x07, - 0x12, 0x19, 0x0a, 0x14, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x64, 0x10, 0xf4, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x10, 0xf5, 0x07, 0x12, 0x0e, - 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x10, 0xf6, 0x07, 0x12, 0x0e, - 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x10, 0xf7, 0x07, 0x12, 0x12, - 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x6f, 0x70, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x10, - 0xf8, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x6f, 0x70, 0x54, 0x79, - 0x70, 0x65, 0x10, 0xf9, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x6f, - 0x70, 0x49, 0x64, 0x10, 0xfa, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x61, 0x63, 0x65, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x10, 0xfb, - 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, - 0xfc, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x10, 0xfd, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x43, 0x6f, 0x6e, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, - 0xfe, 0x07, 0x12, 0x14, 0x0a, 0x0f, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x49, 0x6e, - 0x73, 0x70, 0x65, 0x63, 0x74, 0x10, 0xff, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x53, 0x75, 0x70, 0x70, - 0x6f, 0x72, 0x74, 0x73, 0x42, 0x69, 0x6e, 0x64, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x10, - 0x80, 0x08, 0x12, 0x15, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x10, 0x81, 0x08, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x69, 0x72, - 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x10, 0x82, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x53, 0x74, 0x69, - 0x63, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x83, 0x08, 0x2a, - 0x86, 0x02, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x4b, 0x10, - 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x10, 0x01, 0x12, - 0x15, 0x0a, 0x11, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x41, 0x70, 0x69, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, - 0x64, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x57, 0x72, - 0x6f, 0x6e, 0x67, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, - 0x12, 0x1f, 0x0a, 0x1b, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x64, 0x67, 0x65, 0x52, - 0x6f, 0x75, 0x74, 0x65, 0x72, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, - 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x10, 0x06, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x69, - 0x6e, 0x67, 0x4e, 0x6f, 0x74, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x10, 0x07, 0x12, 0x15, - 0x0a, 0x11, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, - 0x74, 0x6f, 0x72, 0x10, 0x08, 0x12, 0x15, 0x0a, 0x11, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, - 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x10, 0x09, 0x12, 0x0f, 0x0a, 0x0b, - 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x43, 0x6f, 0x73, 0x74, 0x10, 0x0a, 0x12, 0x19, 0x0a, - 0x15, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x4d, - 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x10, 0x0b, 0x2a, 0x38, 0x0a, 0x0f, 0x50, 0x72, 0x65, 0x63, - 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x44, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x65, 0x71, 0x75, - 0x69, 0x72, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x10, 0x02, 0x2a, 0x57, 0x0a, 0x04, 0x46, 0x6c, 0x61, 0x67, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x45, - 0x52, 0x4f, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0e, 0x0a, - 0x0a, 0x54, 0x52, 0x41, 0x43, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x02, 0x12, 0x0d, 0x0a, - 0x09, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x50, 0x41, 0x52, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, - 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x10, 0x08, 0x12, 0x11, 0x0a, 0x0d, 0x4d, 0x55, 0x4c, 0x54, - 0x49, 0x50, 0x41, 0x52, 0x54, 0x5f, 0x4d, 0x53, 0x47, 0x10, 0x10, 0x42, 0x4b, 0x0a, 0x17, 0x6f, - 0x72, 0x67, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f, 0x73, 0x64, 0x6b, 0x2d, - 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x5f, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0b, 0x0a, 0x07, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0d, + 0x58, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xcc, 0x08, + 0x12, 0x1a, 0x0a, 0x15, 0x58, 0x67, 0x41, 0x63, 0x6b, 0x6e, 0x6f, 0x77, 0x6c, 0x65, 0x64, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xcd, 0x08, 0x12, 0x12, 0x0a, 0x0d, + 0x58, 0x67, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x10, 0xce, 0x08, + 0x12, 0x10, 0x0a, 0x0b, 0x58, 0x67, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, + 0xcf, 0x08, 0x12, 0x11, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x10, 0xef, 0xda, 0x03, 0x12, 0x18, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf0, 0xda, 0x03, 0x12, + 0x15, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xf1, 0xda, 0x03, 0x12, 0x0e, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xf2, 0xda, 0x03, 0x12, 0x0e, 0x0a, 0x08, 0x44, 0x69, 0x61, 0x6c, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xf3, 0xda, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x44, 0x69, 0x61, 0x6c, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf4, 0xda, 0x03, 0x12, 0x14, 0x0a, + 0x0e, 0x44, 0x69, 0x61, 0x6c, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, + 0xf5, 0xda, 0x03, 0x12, 0x0e, 0x0a, 0x08, 0x42, 0x69, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, + 0xf6, 0xda, 0x03, 0x12, 0x10, 0x0a, 0x0a, 0x55, 0x6e, 0x62, 0x69, 0x6e, 0x64, 0x54, 0x79, 0x70, + 0x65, 0x10, 0xf7, 0xda, 0x03, 0x12, 0x1b, 0x0a, 0x15, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf8, + 0xda, 0x03, 0x12, 0x0f, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x54, 0x79, 0x70, 0x65, 0x10, + 0xf9, 0xda, 0x03, 0x12, 0x14, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x69, 0x6e, + 0x64, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfa, 0xda, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x10, 0xfb, 0xda, 0x03, + 0x12, 0x14, 0x0a, 0x0e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xfc, 0xda, 0x03, 0x12, 0x1c, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x10, 0xfd, 0xda, 0x03, 0x12, 0x18, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x73, 0x70, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x10, 0xfe, 0xda, 0x03, 0x12, 0x19, + 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x10, 0xff, 0xda, 0x03, 0x12, 0x11, 0x0a, 0x0b, 0x42, 0x69, 0x6e, + 0x64, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x10, 0x80, 0xdb, 0x03, 0x12, 0x1c, 0x0a, 0x16, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x10, 0x81, 0xdb, 0x03, 0x12, 0x1c, 0x0a, 0x16, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x10, 0x82, 0xdb, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, 0x83, 0xdb, 0x03, 0x12, + 0x14, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x10, 0x84, 0xdb, 0x03, 0x12, 0x15, 0x0a, 0x0f, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x10, 0x85, 0xdb, 0x03, 0x12, 0x18, 0x0a, 0x13, + 0x50, 0x6f, 0x73, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x10, 0xb0, 0x54, 0x12, 0x1f, 0x0a, 0x1a, 0x50, 0x6f, 0x73, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x54, 0x79, 0x70, 0x65, 0x10, 0xb1, 0x54, 0x2a, 0x81, 0x05, 0x0a, 0x08, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x49, 0x64, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x45, 0x52, 0x30, 0x10, 0x00, 0x12, 0x09, + 0x0a, 0x04, 0x55, 0x55, 0x49, 0x44, 0x10, 0x80, 0x01, 0x12, 0x0b, 0x0a, 0x06, 0x43, 0x6f, 0x6e, + 0x6e, 0x49, 0x64, 0x10, 0xe8, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x53, 0x65, 0x71, 0x10, 0xe9, 0x07, + 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x10, 0xea, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, + 0x10, 0xeb, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x43, 0x6f, 0x73, 0x74, 0x10, 0xec, 0x07, 0x12, 0x0f, + 0x0a, 0x0a, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x10, 0xed, 0x07, 0x12, + 0x17, 0x0a, 0x12, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x10, 0xee, 0x07, 0x12, 0x1d, 0x0a, 0x18, 0x54, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x10, 0xef, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x43, 0x61, 0x6c, 0x6c, 0x65, + 0x72, 0x49, 0x64, 0x10, 0xf0, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x10, 0xf1, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x46, 0x6c, 0x61, + 0x67, 0x73, 0x10, 0xf2, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x41, 0x70, 0x70, 0x44, 0x61, 0x74, 0x61, + 0x10, 0xf3, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x6e, 0x49, 0x64, 0x10, 0xf4, 0x07, 0x12, 0x11, + 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x10, 0xf5, + 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x10, 0xf6, + 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x10, 0xf7, + 0x07, 0x12, 0x12, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x6f, 0x70, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x10, 0xf8, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x6f, + 0x70, 0x54, 0x79, 0x70, 0x65, 0x10, 0xf9, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, + 0x65, 0x48, 0x6f, 0x70, 0x49, 0x64, 0x10, 0xfa, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x61, + 0x63, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, + 0x64, 0x10, 0xfb, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x10, 0xfc, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x65, + 0x72, 0x49, 0x64, 0x10, 0xfd, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x43, 0x6f, 0x6e, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x10, 0xfe, 0x07, 0x12, 0x14, 0x0a, 0x0f, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, + 0x73, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x10, 0xff, 0x07, 0x12, 0x18, 0x0a, 0x13, 0x53, + 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x42, 0x69, 0x6e, 0x64, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x10, 0x80, 0x08, 0x12, 0x15, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x10, 0x81, 0x08, 0x12, 0x0e, 0x0a, 0x09, + 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x49, 0x64, 0x10, 0x82, 0x08, 0x12, 0x14, 0x0a, 0x0f, + 0x53, 0x74, 0x69, 0x63, 0x6b, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, + 0x83, 0x08, 0x12, 0x13, 0x0a, 0x0e, 0x55, 0x73, 0x65, 0x58, 0x67, 0x72, 0x65, 0x73, 0x73, 0x54, + 0x6f, 0x53, 0x64, 0x6b, 0x10, 0x84, 0x08, 0x12, 0x11, 0x0a, 0x0c, 0x58, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x43, 0x74, 0x72, 0x6c, 0x49, 0x64, 0x10, 0x85, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x58, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x10, 0x86, 0x08, 0x12, 0x1b, + 0x0a, 0x16, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x10, 0x87, 0x08, 0x2a, 0x86, 0x02, 0x0a, 0x05, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x4b, 0x10, 0x00, 0x12, 0x0c, 0x0a, + 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x49, + 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x41, 0x70, 0x69, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x57, 0x72, 0x6f, 0x6e, 0x67, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, + 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x45, 0x64, 0x67, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x72, 0x46, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x10, 0x05, 0x12, 0x12, 0x0a, + 0x0e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x10, + 0x06, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x69, 0x6e, 0x67, 0x4e, 0x6f, + 0x74, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x10, 0x07, 0x12, 0x15, 0x0a, 0x11, 0x49, 0x6e, + 0x76, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x10, + 0x08, 0x12, 0x15, 0x0a, 0x11, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x50, 0x72, 0x65, 0x63, + 0x65, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x10, 0x09, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x6e, 0x76, 0x61, + 0x6c, 0x69, 0x64, 0x43, 0x6f, 0x73, 0x74, 0x10, 0x0a, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x4d, 0x69, 0x73, 0x73, 0x69, + 0x6e, 0x67, 0x10, 0x0b, 0x2a, 0x38, 0x0a, 0x0f, 0x50, 0x72, 0x65, 0x63, 0x65, 0x64, 0x65, 0x6e, + 0x63, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, + 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x02, 0x2a, 0x57, + 0x0a, 0x04, 0x46, 0x6c, 0x61, 0x67, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x45, 0x52, 0x4f, 0x10, 0x00, + 0x12, 0x07, 0x0a, 0x03, 0x46, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x52, 0x41, + 0x43, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4d, 0x55, 0x4c, + 0x54, 0x49, 0x50, 0x41, 0x52, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x45, + 0x41, 0x4d, 0x10, 0x08, 0x12, 0x11, 0x0a, 0x0d, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x50, 0x41, 0x52, + 0x54, 0x5f, 0x4d, 0x53, 0x47, 0x10, 0x10, 0x42, 0x4b, 0x0a, 0x17, 0x6f, 0x72, 0x67, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2e, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, + 0x70, 0x65, 0x6e, 0x7a, 0x69, 0x74, 0x69, 0x2f, 0x73, 0x64, 0x6b, 0x2d, 0x67, 0x6f, 0x6c, 0x61, + 0x6e, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pb/edge_client_pb/edge_client.proto b/pb/edge_client_pb/edge_client.proto index 9b1ffb2a..4925d561 100644 --- a/pb/edge_client_pb/edge_client.proto +++ b/pb/edge_client_pb/edge_client.proto @@ -8,83 +8,96 @@ option go_package = "github.com/openziti/sdk-golang/pb/edge_client_pb"; option java_package = "org.openziti.edge.proto"; enum ContentType { - Hello = 0; - Ping = 1; - Result = 2; - Latency = 3; - - ConnectType = 60783; - StateConnectedType = 60784; - StateClosedType = 60785; - DataType = 60786; - DialType = 60787; - DialSuccessType = 60788; - DialFailedType = 60789; - BindType = 60790; - UnbindType = 60791; - StateSessionEndedType = 60792; - ProbeType = 60793; - UpdateBindType = 60794; - HealthEventType = 60795; - TraceRouteType = 60796; + Hello = 0; + Ping = 1; + Result = 2; + Latency = 3; + + // use the same types as xgress uses on links + XgPayloadType = 1100; + XgAcknowledgementType = 1101; + XgControlType = 1102; + XgCloseType = 1103; + + ConnectType = 60783; + StateConnectedType = 60784; + StateClosedType = 60785; + DataType = 60786; + DialType = 60787; + DialSuccessType = 60788; + DialFailedType = 60789; + BindType = 60790; + UnbindType = 60791; + StateSessionEndedType = 60792; + ProbeType = 60793; + UpdateBindType = 60794; + HealthEventType = 60795; + TraceRouteType = 60796; TraceRouteResponseType = 60797; - ConnInspectRequest = 60798; - ConnInspectResponse = 60799; - BindSuccess = 60800; + ConnInspectRequest = 60798; + ConnInspectResponse = 60799; + BindSuccess = 60800; UpdateTokenSuccessType = 60801; UpdateTokenFailureType = 60802; - UpdateTokenType = 60803; - + UpdateTokenType = 60803; + + InspectRequest = 60804; + InspectResponse = 60805; + PostureResponseType = 10800; PostureResponseSuccessType = 10801; } enum HeaderId { ZER0 = 0; - UUID = 128; - ConnId = 1000; - Seq = 1001; - SessionToken = 1002; - PublicKey = 1003; - Cost = 1004; - Precedence = 1005; - TerminatorIdentity = 1006; + UUID = 128; + ConnId = 1000; + Seq = 1001; + SessionToken = 1002; + PublicKey = 1003; + Cost = 1004; + Precedence = 1005; + TerminatorIdentity = 1006; TerminatorIdentitySecret = 1007; - CallerId = 1008; - CryptoMethod = 1009; - Flags = 1010; - AppData = 1011; - RouterProvidedConnId = 1012; - HealthStatus = 1013; - ErrorCode = 1014; - Timestamp = 1015; - TraceHopCount = 1016; - TraceHopType = 1017; - TraceHopId = 1018; - TraceSourceRequestId = 1019; - TraceError = 1020; - ListenerId = 1021; - ConnType = 1022; - SupportsInspect = 1023; - SupportsBindSuccess = 1024; - ConnectionMarker = 1025; - CircuitId = 1026; - StickinessToken = 1027; + CallerId = 1008; + CryptoMethod = 1009; + Flags = 1010; + AppData = 1011; + RouterProvidedConnId = 1012; + HealthStatus = 1013; + ErrorCode = 1014; + Timestamp = 1015; + TraceHopCount = 1016; + TraceHopType = 1017; + TraceHopId = 1018; + TraceSourceRequestId = 1019; + TraceError = 1020; + ListenerId = 1021; + ConnType = 1022; + SupportsInspect = 1023; + SupportsBindSuccess = 1024; + ConnectionMarker = 1025; + CircuitId = 1026; + StickinessToken = 1027; + UseXgressToSdk = 1028; + XgressCtrlId = 1029; + XgressAddress = 1030; + InspectRequestedValues = 1031; } enum Error { - OK = 0; - Internal = 1; - InvalidApiSession = 2; - InvalidSession = 3; - WrongSessionType = 4; + OK = 0; + Internal = 1; + InvalidApiSession = 2; + InvalidSession = 3; + WrongSessionType = 4; InvalidEdgeRouterForSession = 5; - InvalidService = 6; - TunnelingNotEnabled = 7; - InvalidTerminator = 8; - InvalidPrecedence = 9; - InvalidCost = 10; - EncryptionDataMissing = 11; + InvalidService = 6; + TunnelingNotEnabled = 7; + InvalidTerminator = 8; + InvalidPrecedence = 9; + InvalidCost = 10; + EncryptionDataMissing = 11; } enum PrecedenceValue { @@ -130,7 +143,7 @@ message PostureResponse { } message OperatingSystem { - string type = 1; + string type = 1; string version = 2; string build = 3; } @@ -159,11 +172,11 @@ message PostureResponse { } message SdkInfo { - string appId = 1; + string appId = 1; string appVersion = 2; - string branch = 3; - string revision = 4; - string type = 5; - string version = 6; + string branch = 3; + string revision = 4; + string type = 5; + string version = 6; } } diff --git a/version b/version index d3827e75..9459d4ba 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.0 +1.1 diff --git a/xgress/circuit_inspections.go b/xgress/circuit_inspections.go index 09b99c3f..530e68ca 100644 --- a/xgress/circuit_inspections.go +++ b/xgress/circuit_inspections.go @@ -17,10 +17,11 @@ package xgress type CircuitInspectDetail struct { - CircuitId string `json:"circuitId"` - Forwards map[string]string `json:"forwards"` - XgressDetails map[string]*InspectDetail `json:"xgressDetails"` - LinkDetails map[string]*LinkInspectDetail `json:"linkDetails"` + CircuitId string `json:"circuitId"` + Forwards map[string]string `json:"forwards"` + XgressDetails map[string]*InspectDetail `json:"xgressDetails"` + RelatedEntities map[string]map[string]any `json:"relatedEntities"` + Errors []string `json:"errors"` includeGoroutines bool } @@ -36,8 +37,18 @@ func (self *CircuitInspectDetail) AddXgressDetail(xgressDetail *InspectDetail) { self.XgressDetails[xgressDetail.Address] = xgressDetail } -func (self *CircuitInspectDetail) AddLinkDetail(linkDetail *LinkInspectDetail) { - self.LinkDetails[linkDetail.Id] = linkDetail +func (self *CircuitInspectDetail) AddRelatedEntity(entityType string, id string, detail any) { + if self.RelatedEntities == nil { + self.RelatedEntities = make(map[string]map[string]any) + } + if self.RelatedEntities[entityType] == nil { + self.RelatedEntities[entityType] = make(map[string]any) + } + self.RelatedEntities[entityType][id] = detail +} + +func (self *CircuitInspectDetail) AddError(err error) { + self.Errors = append(self.Errors, err.Error()) } type InspectDetail struct { @@ -81,14 +92,14 @@ type RecvBufferDetail struct { AcquiredSafely bool `json:"acquiredSafely"` } -type LinkInspectDetail struct { - Id string `json:"id"` - Iteration uint32 `json:"iteration"` - Key string `json:"key"` - Split bool `json:"split"` - Protocol string `json:"protocol"` - DialAddress string `json:"dialAddress"` - Dest string `json:"dest"` - DestVersion string `json:"destVersion"` - Dialed bool `json:"dialed"` +type CircuitsDetail struct { + Circuits map[string]*CircuitDetail `json:"circuits"` +} + +type CircuitDetail struct { + CircuitId string `json:"circuitId"` + ConnId uint32 `json:"connId"` + Address string `json:"address"` + Originator string `json:"originator"` + IsXgress bool `json:"isXgress"` } diff --git a/xgress/link_send_buffer.go b/xgress/link_send_buffer.go index ed145d81..56730c2d 100644 --- a/xgress/link_send_buffer.go +++ b/xgress/link_send_buffer.go @@ -103,7 +103,7 @@ func NewLinkSendBuffer(x *Xgress) *LinkSendBuffer { x: x, buffer: make(map[int32]*txPayload), newlyBuffered: make(chan *txPayload), - newlyReceivedAcks: make(chan *Acknowledgement, 2), + newlyReceivedAcks: make(chan *Acknowledgement, 4), closeNotify: make(chan struct{}), windowsSize: x.Options.TxPortalStartSize, retxThreshold: x.Options.RetxStartMs, @@ -245,9 +245,7 @@ func (buffer *LinkSendBuffer) run() { case ack := <-buffer.newlyReceivedAcks: buffer.receiveAcknowledgement(ack) buffer.retransmit() - if buffer.closeWhenEmpty.Load() && len(buffer.buffer) == 0 && !buffer.x.Closed() && buffer.x.IsEndOfCircuitSent() { - go buffer.x.Close() - } + buffer.checkForCloseOnEmpty() case txPayload := <-buffered: buffer.buffer[txPayload.payload.GetSequence()] = txPayload @@ -259,6 +257,7 @@ func (buffer *LinkSendBuffer) run() { case <-retransmitTicker.C: buffer.retransmit() + buffer.checkForCloseOnEmpty() case <-buffer.closeNotify: buffer.close() @@ -267,6 +266,12 @@ func (buffer *LinkSendBuffer) run() { } } +func (buffer *LinkSendBuffer) checkForCloseOnEmpty() { + if buffer.closeWhenEmpty.Load() && len(buffer.buffer) == 0 && !buffer.x.Closed() && buffer.x.IsEndOfCircuitSent() { + go buffer.x.Close() + } +} + func (buffer *LinkSendBuffer) close() { if buffer.blockedByLocalWindow { buffer.metrics().BufferUnblockedByLocalWindow() diff --git a/xgress/messages.go b/xgress/messages.go index 01b1cded..a61b014c 100644 --- a/xgress/messages.go +++ b/xgress/messages.go @@ -63,6 +63,13 @@ func (o Originator) String() string { return "Terminator" } +func (o Originator) Invert() Originator { + if o == Initiator { + return Terminator + } + return Initiator +} + type Flag uint32 const ( @@ -70,6 +77,7 @@ const ( PayloadFlagOriginator Flag = 2 PayloadFlagCircuitStart Flag = 4 PayloadFlagChunk Flag = 8 + PayloadFlagRetransmit Flag = 16 ) func NewAcknowledgement(circuitId string, originator Originator) *Acknowledgement { @@ -304,6 +312,14 @@ func (payload *Payload) IsCircuitStartFlagSet() bool { return isFlagSet(payload.Flags, PayloadFlagCircuitStart) } +func (payload *Payload) IsRetransmitFlagSet() bool { + return isFlagSet(payload.Flags, PayloadFlagRetransmit) +} + +func (payload *Payload) MarkAsRetransmit() { + payload.Flags = setPayloadFlag(payload.Flags, PayloadFlagRetransmit) +} + func (payload *Payload) GetOriginator() Originator { if isFlagSet(payload.Flags, PayloadFlagOriginator) { return Terminator diff --git a/xgress/payload_ingester.go b/xgress/payload_ingester.go index cfc852fd..78361d8d 100644 --- a/xgress/payload_ingester.go +++ b/xgress/payload_ingester.go @@ -48,7 +48,7 @@ func (self *PayloadIngester) run() { for { select { case payloadEntry := <-self.payloadIngest: - payloadEntry.x.payloadIngester(payloadEntry.payload) + payloadEntry.x.acceptPayload(payloadEntry.payload) case x := <-self.payloadSendReq: x.queueSends() case evt := <-self.receiveBufferInspects: diff --git a/xgress/retransmitter.go b/xgress/retransmitter.go index fa308951..c4ff926e 100644 --- a/xgress/retransmitter.go +++ b/xgress/retransmitter.go @@ -143,6 +143,7 @@ func (self *Retransmitter) retransmitSender() { select { case retransmit := <-self.retransmitSend: if !retransmit.isAcked() { + retransmit.payload.MarkAsRetransmit() if err := retransmit.x.dataPlane.RetransmitPayload(retransmit.x.address, retransmit.payload); err != nil { // if xgress is closed, don't log the error. We still want to try retransmitting in case we're re-sending end of circuit if !retransmit.x.Closed() { diff --git a/xgress/xgress.go b/xgress/xgress.go index 050247ad..cf3c1df1 100644 --- a/xgress/xgress.go +++ b/xgress/xgress.go @@ -64,6 +64,12 @@ type ControlReceiver interface { HandleControlReceive(controlType ControlType, headers channel.Headers) } +type Env interface { + GetRetransmitter() *Retransmitter + GetPayloadIngester() *PayloadIngester + GetMetrics() Metrics +} + // DataPlaneAdapter is invoked by an xgress whenever messages need to be sent to the data plane. Generally a DataPlaneAdapter // is implemented to connect the xgress to a data plane data transmission system. type DataPlaneAdapter interface { @@ -78,9 +84,8 @@ type DataPlaneAdapter interface { // ForwardAcknowledgement is used to forward acks onto the data-plane from an xgress ForwardAcknowledgement(ack *Acknowledgement, address Address) - GetRetransmitter() *Retransmitter - GetPayloadIngester() *PayloadIngester - GetMetrics() Metrics + + Env } // CloseHandler is invoked by an xgress when the connected peer terminates the communication. @@ -260,7 +265,7 @@ func (self *Xgress) GetEndCircuit() *Payload { func (self *Xgress) ForwardEndOfCircuit(sendF func(payload *Payload) bool) { // for now always send end of circuit. too many is better than not enough - if !self.IsEndOfCircuitSent() { + if self.flags.CompareAndSet(endOfCircuitSentFlag, false, true) { sendF(self.GetEndCircuit()) self.flags.Set(endOfCircuitSentFlag, true) } @@ -321,6 +326,10 @@ func (self *Xgress) Close() { } } +func (self *Xgress) CloseSendBuffer() { + self.payloadBuffer.Close() +} + func (self *Xgress) Closed() bool { return self.flags.IsSet(closedFlag) } @@ -358,7 +367,7 @@ func (self *Xgress) HandleControlReceive(controlType ControlType, headers channe self.dataPlane.ForwardControlMessage(control, self) } -func (self *Xgress) payloadIngester(payload *Payload) { +func (self *Xgress) acceptPayload(payload *Payload) { if payload.IsCircuitStartFlagSet() && self.firstCircuitStartReceived() { go self.rx() } @@ -838,6 +847,10 @@ func (self *Xgress) GetSequence() uint64 { } func (self *Xgress) InspectCircuit(detail *CircuitInspectDetail) { + detail.AddXgressDetail(self.GetInspectDetail(detail.includeGoroutines)) +} + +func (self *Xgress) GetInspectDetail(includeGoroutines bool) *InspectDetail { timeSinceLastRxFromLink := time.Duration(time.Now().UnixMilli()-atomic.LoadInt64(&self.timeOfLastRxFromLink)) * time.Millisecond xgressDetail := &InspectDetail{ Address: string(self.address), @@ -851,11 +864,11 @@ func (self *Xgress) InspectCircuit(detail *CircuitInspectDetail) { Flags: strconv.FormatUint(uint64(self.flags.Load()), 2), } - if detail.IncludeGoroutines() { + if includeGoroutines { xgressDetail.Goroutines = self.getRelatedGoroutines(xgressDetail.XgressPointer, xgressDetail.LinkSendBufferPointer) } - detail.AddXgressDetail(xgressDetail) + return xgressDetail } func (self *Xgress) getRelatedGoroutines(contains ...string) []string { diff --git a/ziti/config.go b/ziti/config.go index e0f0fd72..4c6a499b 100644 --- a/ziti/config.go +++ b/ziti/config.go @@ -61,14 +61,25 @@ type Config struct { //Allows providing a function which controls how/where connections to a router are proxied. RouterProxy func(addr string) *transport.ProxyConfiguration `json:"-"` - // If set to true, the sdk will attempt to create a separate connection to edge routers for control plane data, - // such as dials. This flag should not be considered part of the stable API yet. It may default to true at - // some point in the future or be removed. - EnableSeparateControlPlaneConnection bool `json:"-"` + // If set to a number greater than one, the sdk will attempt to create multiple connections to edge routers. + // This configuration value should not be considered part of the stable API yet. It currently defaults to one, + // but it may default to a larger number at some point in the future or be removed. If set to zero, it will + // be reset to one. + MaxDefaultConnections uint32 `json:"-"` + + // If set to a number greater than zero, the sdk will attempt to create one or more separate connection to + // each edge routers for control plane data, such as dials. This configuration value should not be considered + // part of the stable API yet. It currently defaults to zero, but it may default to 1 at some point in the future + // or be removed. + MaxControlConnections uint32 `json:"-"` } -func (cfg *Config) SetSeparateControlPlaneConnectionEnabled(enabled bool) { - cfg.EnableSeparateControlPlaneConnection = enabled +func (cfg *Config) SetMaxControlConnections(val uint32) { + cfg.MaxControlConnections = val +} + +func (cfg *Config) SetMaxDefaultConnections(val uint32) { + cfg.MaxDefaultConnections = val } // NewConfig will create a new Config object from a provided Ziti Edge Client API URL and identity configuration. diff --git a/ziti/contexts.go b/ziti/contexts.go index 6d749d74..a780e51c 100644 --- a/ziti/contexts.go +++ b/ziti/contexts.go @@ -82,14 +82,19 @@ func NewContextWithOpts(cfg *Config, options *Options) (Context, error) { } newContext := &ContextImpl{ - Id: NewId(), - routerConnections: cmap.New[edge.RouterConn](), - options: options, - authQueryHandlers: map[string]func(query *rest_model.AuthQueryDetail, response MfaCodeResponse) error{}, - closeNotify: make(chan struct{}), - EventEmmiter: events.New(), - routerProxy: cfg.RouterProxy, - enableCtrlPlaneConnection: cfg.EnableSeparateControlPlaneConnection, + Id: NewId(), + routerConnections: cmap.New[edge.RouterConn](), + options: options, + authQueryHandlers: map[string]func(query *rest_model.AuthQueryDetail, response MfaCodeResponse) error{}, + closeNotify: make(chan struct{}), + EventEmmiter: events.New(), + routerProxy: cfg.RouterProxy, + maxDefaultConnections: int(cfg.MaxDefaultConnections), + maxControlConnections: int(cfg.MaxControlConnections), + } + + if newContext.maxDefaultConnections < 1 { + newContext.maxDefaultConnections = 1 } if cfg.ID.Cert != "" && cfg.ID.Key != "" { diff --git a/ziti/edge/channel.go b/ziti/edge/channel.go index 66a28a8c..2e982aea 100644 --- a/ziti/edge/channel.go +++ b/ziti/edge/channel.go @@ -16,7 +16,7 @@ const ( func NewBaseSdkChannel(underlay channel.Underlay) *BaseSdkChannel { senderContext := channel.NewSenderContext() - defaultMsgChan := make(chan channel.Sendable, 4) + defaultMsgChan := make(chan channel.Sendable, 64) controlMsgChan := make(chan channel.Sendable, 4) retryMsgChan := make(chan channel.Sendable, 4) @@ -134,14 +134,14 @@ func (self *BaseSdkChannel) UpdateCtrlChannelAvailable(ch channel.MultiChannel) self.controlChannelAvailable.Store(ch.GetUnderlayCountsByType()[ChannelTypeControl] > 0) } -func NewDialSdkChannel(dialer channel.DialUnderlayFactory, underlay channel.Underlay) UnderlayHandlerSdkChannel { +func NewDialSdkChannel(dialer channel.DialUnderlayFactory, underlay channel.Underlay, maxDefaultChannels, maxControlChannel int) UnderlayHandlerSdkChannel { result := &DialSdkChannel{ BaseSdkChannel: *NewBaseSdkChannel(underlay), dialer: dialer, } - result.constraints.AddConstraint(ChannelTypeDefault, 1, 1) - result.constraints.AddConstraint(ChannelTypeControl, 1, 0) + result.constraints.AddConstraint(ChannelTypeDefault, maxDefaultChannels, 1) + result.constraints.AddConstraint(ChannelTypeControl, maxControlChannel, 0) return result } diff --git a/ziti/edge/conn.go b/ziti/edge/conn.go index f7d23d14..8d96a958 100644 --- a/ziti/edge/conn.go +++ b/ziti/edge/conn.go @@ -19,6 +19,8 @@ package edge import ( "fmt" "github.com/openziti/edge-api/rest_model" + "github.com/openziti/foundation/v2/concurrenz" + "github.com/openziti/sdk-golang/xgress" "github.com/openziti/secretstream/kx" "io" "net" @@ -32,13 +34,17 @@ import ( "github.com/openziti/foundation/v2/sequence" ) +const ( + ConnFlagIdxFirstMsgSent = 0 +) + func init() { AddAddressParsers() } type RouterClient interface { - Connect(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *DialOptions) (Conn, error) - Listen(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *ListenOptions) (Listener, error) + Connect(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *DialOptions, envF func() xgress.Env) (Conn, error) + Listen(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *ListenOptions, envF func() xgress.Env) (Listener, error) //UpdateToken will attempt to send token updates to the connected router. A success/failure response is expected //within the timeout period. @@ -108,6 +114,7 @@ type MsgChannel struct { msgIdSeq *sequence.Sequence writeDeadline time.Time trace bool + flags concurrenz.AtomicBitSet } type TraceRouteResult struct { @@ -150,11 +157,7 @@ func (ec *MsgChannel) Write(data []byte) (n int, err error) { } func (ec *MsgChannel) WriteTraced(data []byte, msgUUID []byte, hdrs map[int32][]byte) (int, error) { - copyBuf := make([]byte, len(data)) - copy(copyBuf, data) - - seq := ec.msgIdSeq.Next() - msg := NewDataMsg(ec.id, seq, copyBuf) + msg := NewDataMsg(ec.id, data) if msgUUID != nil { msg.Headers[UUIDHeader] = msgUUID } @@ -165,13 +168,13 @@ func (ec *MsgChannel) WriteTraced(data []byte, msgUUID []byte, hdrs map[int32][] // indicate that we can accept multipart messages // with the first message - if seq == 1 { + if ec.flags.CompareAndSet(ConnFlagIdxFirstMsgSent, false, true) { flags, _ := msg.GetUint32Header(FlagsHeader) flags = flags | MULTIPART msg.PutUint32Header(FlagsHeader, flags) } ec.TraceMsg("write", msg) - pfxlog.Logger().WithFields(GetLoggerFields(msg)).Debugf("writing %v bytes", len(copyBuf)) + pfxlog.Logger().WithFields(GetLoggerFields(msg)).Debugf("writing %v bytes", len(data)) // NOTE: We need to wait for the buffer to be on the wire before returning. The Writer contract // states that buffers are not allowed be retained, and if we have it queued asynchronously @@ -223,6 +226,7 @@ type DialOptions struct { CallerId string AppData []byte StickinessToken []byte + SdkFlowControl bool } func (d DialOptions) GetConnectTimeout() time.Duration { @@ -244,6 +248,7 @@ type ListenOptions struct { IdentitySecret string BindUsingEdgeIdentity bool ManualStart bool + SdkFlowControl bool ListenerId string KeyPair *kx.KeyPair eventC chan *ListenerEvent diff --git a/ziti/edge/messages.go b/ziti/edge/messages.go index 8fca7684..5dcbc505 100644 --- a/ziti/edge/messages.go +++ b/ziti/edge/messages.go @@ -18,24 +18,28 @@ package edge import ( "encoding/binary" + "encoding/json" "github.com/openziti/channel/v4" "github.com/openziti/foundation/v2/uuidz" + "github.com/openziti/sdk-golang/inspect" "github.com/openziti/sdk-golang/pb/edge_client_pb" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) const ( - ContentTypeConnect = int32(edge_client_pb.ContentType_ConnectType) - ContentTypeStateConnected = int32(edge_client_pb.ContentType_StateConnectedType) - ContentTypeStateClosed = int32(edge_client_pb.ContentType_StateClosedType) - ContentTypeData = int32(edge_client_pb.ContentType_DataType) - ContentTypeDial = int32(edge_client_pb.ContentType_DialType) - ContentTypeDialSuccess = int32(edge_client_pb.ContentType_DialSuccessType) - ContentTypeDialFailed = int32(edge_client_pb.ContentType_DialFailedType) - ContentTypeBind = int32(edge_client_pb.ContentType_BindType) - ContentTypeUnbind = int32(edge_client_pb.ContentType_UnbindType) - ContentTypeStateSessionEnded = int32(edge_client_pb.ContentType_StateSessionEndedType) + ContentTypeConnect = int32(edge_client_pb.ContentType_ConnectType) + ContentTypeStateConnected = int32(edge_client_pb.ContentType_StateConnectedType) + ContentTypeStateClosed = int32(edge_client_pb.ContentType_StateClosedType) + ContentTypeData = int32(edge_client_pb.ContentType_DataType) + ContentTypeDial = int32(edge_client_pb.ContentType_DialType) + ContentTypeDialSuccess = int32(edge_client_pb.ContentType_DialSuccessType) + ContentTypeDialFailed = int32(edge_client_pb.ContentType_DialFailedType) + ContentTypeBind = int32(edge_client_pb.ContentType_BindType) + ContentTypeUnbind = int32(edge_client_pb.ContentType_UnbindType) + + // ContentTypeStateSessionEnded = int32(edge_client_pb.ContentType_StateSessionEndedType) + ContentTypeProbe = int32(edge_client_pb.ContentType_ProbeType) ContentTypeUpdateBind = int32(edge_client_pb.ContentType_UpdateBindType) ContentTypeHealthEvent = int32(edge_client_pb.ContentType_HealthEventType) @@ -43,14 +47,22 @@ const ( ContentTypeTraceRouteResponse = int32(edge_client_pb.ContentType_TraceRouteResponseType) ContentTypeConnInspectRequest = int32(edge_client_pb.ContentType_ConnInspectRequest) - ContentTypeConnInspectResponse = int32(edge_client_pb.ContentType_ConnInspectResponse) - ContentTypeBindSuccess = int32(edge_client_pb.ContentType_BindSuccess) + ContentTypeConnInspectResponse = int32(edge_client_pb.ContentType_InspectResponse) + ContentTypeInspectRequest = int32(edge_client_pb.ContentType_InspectRequest) + ContentTypeInspectResponse = int32(edge_client_pb.ContentType_ConnInspectResponse) + + ContentTypeBindSuccess = int32(edge_client_pb.ContentType_BindSuccess) ContentTypeUpdateToken = int32(edge_client_pb.ContentType_UpdateTokenType) ContentTypeUpdateTokenSuccess = int32(edge_client_pb.ContentType_UpdateTokenSuccessType) ContentTypeUpdateTokenFailure = int32(edge_client_pb.ContentType_UpdateTokenFailureType) ContentTypePostureResponse = int32(edge_client_pb.ContentType_PostureResponseType) + + ContentTypeXgPayload = int32(edge_client_pb.ContentType_XgPayloadType) + ContentTypeXgAcknowledgement = int32(edge_client_pb.ContentType_XgAcknowledgementType) + ContentTypeXgControl = int32(edge_client_pb.ContentType_XgControlType) + ContentTypeXgClose = int32(edge_client_pb.ContentType_XgCloseType) ) const ( @@ -85,6 +97,10 @@ const ( ConnectionMarkerHeader = int32(edge_client_pb.HeaderId_ConnectionMarker) CircuitIdHeader = int32(edge_client_pb.HeaderId_CircuitId) StickinessTokenHeader = int32(edge_client_pb.HeaderId_StickinessToken) + UseXgressToSdkHeader = int32(edge_client_pb.HeaderId_UseXgressToSdk) + XgressCtrlIdHeader = int32(edge_client_pb.HeaderId_XgressCtrlId) + XgressAddressHeader = int32(edge_client_pb.HeaderId_XgressAddress) + InspectRequestValuesHeader = int32(edge_client_pb.HeaderId_InspectRequestedValues) ) const ( @@ -170,15 +186,14 @@ type MsgEvent struct { Msg *channel.Message } -func newMsg(contentType int32, connId uint32, seq uint32, data []byte) *channel.Message { +func newMsg(contentType int32, connId uint32, data []byte) *channel.Message { msg := channel.NewMessage(contentType, data) msg.PutUint32Header(ConnIdHeader, connId) - msg.PutUint32Header(SeqHeader, seq) return msg } -func NewDataMsg(connId uint32, seq uint32, data []byte) *channel.Message { - return newMsg(ContentTypeData, connId, seq, data) +func NewDataMsg(connId uint32, data []byte) *channel.Message { + return newMsg(ContentTypeData, connId, data) } func NewProbeMsg() *channel.Message { @@ -211,8 +226,28 @@ func NewConnInspectResponse(connId uint32, connType ConnType, state string) *cha return msg } +func NewInspectRequest(connId *uint32, requestedValues ...string) *channel.Message { + msg := channel.NewMessage(ContentTypeInspectRequest, nil) + if connId != nil { + msg.PutUint32Header(ConnIdHeader, *connId) + } + msg.PutStringSliceHeader(InspectRequestValuesHeader, requestedValues) + return msg +} + +func NewInspectResponse(connId uint32, resp *inspect.SdkInspectResponse) (*channel.Message, error) { + b, err := json.Marshal(resp) + if err != nil { + return nil, err + } + + msg := channel.NewMessage(ContentTypeInspectResponse, b) + msg.PutUint32Header(ConnIdHeader, connId) + return msg, nil +} + func NewConnectMsg(connId uint32, token string, pubKey []byte, options *DialOptions) *channel.Message { - msg := newMsg(ContentTypeConnect, connId, 0, []byte(token)) + msg := newMsg(ContentTypeConnect, connId, []byte(token)) if pubKey != nil { msg.Headers[PublicKeyHeader] = pubKey msg.PutByteHeader(CryptoMethodHeader, byte(CryptoMethodLibsodium)) @@ -234,23 +269,24 @@ func NewConnectMsg(connId uint32, token string, pubKey []byte, options *DialOpti } func NewStateConnectedMsg(connId uint32) *channel.Message { - return newMsg(ContentTypeStateConnected, connId, 0, nil) + return newMsg(ContentTypeStateConnected, connId, nil) } func NewStateClosedMsg(connId uint32, message string) *channel.Message { - return newMsg(ContentTypeStateClosed, connId, 0, []byte(message)) + return newMsg(ContentTypeStateClosed, connId, []byte(message)) } func NewDialMsg(connId uint32, token string, callerId string) *channel.Message { - msg := newMsg(ContentTypeDial, connId, 0, []byte(token)) + msg := newMsg(ContentTypeDial, connId, []byte(token)) msg.Headers[CallerIdHeader] = []byte(callerId) return msg } func NewBindMsg(connId uint32, token string, pubKey []byte, options *ListenOptions) *channel.Message { - msg := newMsg(ContentTypeBind, connId, 0, []byte(token)) + msg := newMsg(ContentTypeBind, connId, []byte(token)) msg.PutBoolHeader(SupportsInspectHeader, true) msg.PutBoolHeader(SupportsBindSuccessHeader, true) + msg.PutBoolHeader(UseXgressToSdkHeader, options.SdkFlowControl) if pubKey != nil { msg.Headers[PublicKeyHeader] = pubKey @@ -283,11 +319,11 @@ func NewBindMsg(connId uint32, token string, pubKey []byte, options *ListenOptio } func NewUnbindMsg(connId uint32, token string) *channel.Message { - return newMsg(ContentTypeUnbind, connId, 0, []byte(token)) + return newMsg(ContentTypeUnbind, connId, []byte(token)) } func NewUpdateBindMsg(connId uint32, token string, cost *uint16, precedence *Precedence) *channel.Message { - msg := newMsg(ContentTypeUpdateBind, connId, 0, []byte(token)) + msg := newMsg(ContentTypeUpdateBind, connId, []byte(token)) if cost != nil { msg.PutUint16Header(CostHeader, *cost) } @@ -298,7 +334,7 @@ func NewUpdateBindMsg(connId uint32, token string, cost *uint16, precedence *Pre } func NewHealthEventMsg(connId uint32, token string, pass bool) *channel.Message { - msg := newMsg(ContentTypeHealthEvent, connId, 0, []byte(token)) + msg := newMsg(ContentTypeHealthEvent, connId, []byte(token)) msg.PutBoolHeader(HealthStatusHeader, pass) return msg } @@ -306,16 +342,12 @@ func NewHealthEventMsg(connId uint32, token string, pass bool) *channel.Message func NewDialSuccessMsg(connId uint32, newConnId uint32) *channel.Message { newConnIdBytes := make([]byte, 4) binary.LittleEndian.PutUint32(newConnIdBytes, newConnId) - msg := newMsg(ContentTypeDialSuccess, connId, 0, newConnIdBytes) + msg := newMsg(ContentTypeDialSuccess, connId, newConnIdBytes) return msg } func NewDialFailedMsg(connId uint32, message string) *channel.Message { - return newMsg(ContentTypeDialFailed, connId, 0, []byte(message)) -} - -func NewStateSessionEndedMsg(reason string) *channel.Message { - return newMsg(ContentTypeStateSessionEnded, 0, 0, []byte(reason)) + return newMsg(ContentTypeDialFailed, connId, []byte(message)) } // NewUpdateTokenMsg creates a message sent to edge routers to update the token that diff --git a/ziti/edge/msg_mux.go b/ziti/edge/msg_mux.go index 6e67ba17..67b6b6f7 100644 --- a/ziti/edge/msg_mux.go +++ b/ziti/edge/msg_mux.go @@ -20,11 +20,15 @@ import ( "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/channel/v4" + "github.com/openziti/sdk-golang/inspect" + "github.com/openziti/sdk-golang/xgress" "github.com/pkg/errors" "github.com/sirupsen/logrus" "math" + "strings" "sync" "sync/atomic" + "time" ) type MsgSink interface { @@ -85,6 +89,10 @@ func (mux *CowMapMsgMux) ContentType() int32 { func (mux *CowMapMsgMux) HandleReceive(msg *channel.Message, ch channel.Channel) { connId, found := msg.GetUint32Header(ConnIdHeader) if !found { + if msg.ContentType == ContentTypeInspectRequest { + mux.HandleInspect(msg, ch) + return + } pfxlog.Logger().Errorf("received edge message with no connId header. content type: %v", msg.ContentType) return } @@ -104,6 +112,63 @@ func (mux *CowMapMsgMux) HandleReceive(msg *channel.Message, ch channel.Channel) } } +func (mux *CowMapMsgMux) HandleInspect(msg *channel.Message, ch channel.Channel) { + resp := &inspect.SdkInspectResponse{ + Success: true, + Values: make(map[string]any), + } + requestedValues, _, err := msg.GetStringSliceHeader(InspectRequestValuesHeader) + if err != nil { + resp.Errors = append(resp.Errors, err.Error()) + resp.Success = false + mux.returnInspectResponse(msg, ch, resp) + return + } + + for _, requested := range requestedValues { + lc := strings.ToLower(requested) + if lc == "circuits" { + circuitsDetail := &xgress.CircuitsDetail{ + Circuits: make(map[string]*xgress.CircuitDetail), + } + + for _, sink := range mux.getSinks() { + if circuitInfoSrc, ok := sink.(interface { + GetCircuitDetail() *xgress.CircuitDetail + }); ok { + circuitDetail := circuitInfoSrc.GetCircuitDetail() + if circuitDetail != nil { + circuitsDetail.Circuits[circuitDetail.CircuitId] = circuitDetail + } + } + } + resp.Values[requested] = circuitsDetail + } + } + + mux.returnInspectResponse(msg, ch, resp) +} + +func (mux *CowMapMsgMux) returnInspectResponse(msg *channel.Message, ch channel.Channel, resp *inspect.SdkInspectResponse) { + var sender channel.Sender = ch + if mc, ok := ch.(channel.MultiChannel); ok { + if sdkChan, ok := mc.GetUnderlayHandler().(SdkChannel); ok { + sender = sdkChan.GetControlSender() + } + } + + reply, err := NewInspectResponse(0, resp) + if err != nil { + pfxlog.Logger().WithError(err).Error("failed to create inspect response") + return + } + reply.ReplyTo(msg) + + if err = reply.WithTimeout(5 * time.Second).Send(sender); err != nil { + pfxlog.Logger().WithError(err).Error("failed to send inspect response") + } +} + func (mux *CowMapMsgMux) HandleClose(channel.Channel) { mux.Close() } diff --git a/ziti/edge/network/conn.go b/ziti/edge/network/conn.go index 6eee7a09..2a0ebc8f 100644 --- a/ziti/edge/network/conn.go +++ b/ziti/edge/network/conn.go @@ -19,8 +19,11 @@ package network import ( "encoding/json" "fmt" + "github.com/openziti/sdk-golang/inspect" + "github.com/openziti/sdk-golang/xgress" "io" "net" + "strings" "sync" "sync/atomic" "time" @@ -59,6 +62,7 @@ type edgeConn struct { hosting cmap.ConcurrentMap[string, *edgeListener] flags uint32 closed atomic.Bool + closeNotify chan struct{} readFIN atomic.Bool sentFIN atomic.Bool serviceName string @@ -76,6 +80,9 @@ type edgeConn struct { sender secretstream.Encryptor appData []byte sync.Mutex + + dataSink io.Writer + xgCircuit *XgAdapter } func (conn *edgeConn) Write(data []byte) (int, error) { @@ -92,10 +99,13 @@ func (conn *edgeConn) Write(data []byte) (int, error) { return 0, err } - _, err = conn.MsgChannel.Write(cipherData) + _, err = conn.dataSink.Write(cipherData) return len(data), err } else { - return conn.MsgChannel.Write(data) + copyBuf := make([]byte, len(data)) + copy(copyBuf, data) + + return conn.dataSink.Write(copyBuf) } } @@ -157,11 +167,26 @@ func (conn *edgeConn) Accept(msg *channel.Message) { switch conn.connType { case ConnTypeDial: - if msg.ContentType == edge.ContentTypeStateClosed { + switch msg.ContentType { + case edge.ContentTypeXgPayload: + conn.HandleXgPayload(msg) + return + + case edge.ContentTypeXgAcknowledgement: + conn.HandleXgAcknowledgement(msg) + return + + case edge.ContentTypeStateClosed: + if conn.IsClosed() { + return + } conn.sentFIN.Store(true) // if we're not closing until all reads are done, at least prevent more writes - } - if msg.ContentType == edge.ContentTypeTraceRoute { + case edge.ContentTypeInspectRequest: + conn.HandleInspect(msg) + return + + case edge.ContentTypeTraceRoute: hops, _ := msg.GetUint32Header(edge.TraceHopCountHeader) if hops > 0 { hops-- @@ -218,6 +243,108 @@ func (conn *edgeConn) Accept(msg *channel.Message) { } } +func (conn *edgeConn) HandleXgPayload(msg *channel.Message) { + adapter := conn.xgCircuit + if adapter == nil { + // TODO: handle + return + } + + payload, err := xgress.UnmarshallPayload(msg) + if err != nil { + adapter.xg.Close() + return + } + + if err = adapter.xg.SendPayload(payload, 0, 0); err != nil { + adapter.xg.Close() + } +} + +func (conn *edgeConn) HandleXgAcknowledgement(msg *channel.Message) { + adapter := conn.xgCircuit + if adapter == nil { + // TODO: handle + return + } + + ack, err := xgress.UnmarshallAcknowledgement(msg) + if err != nil { + adapter.xg.Close() + return + } + + if err = adapter.xg.SendAcknowledgement(ack); err != nil { + adapter.xg.Close() + } + // adapter.env.GetAckIngester().Ingest(msg, adapter.xg) +} + +func (conn *edgeConn) HandleInspect(msg *channel.Message) { + resp := &inspect.SdkInspectResponse{ + Success: true, + Values: make(map[string]any), + } + requestedValues, _, err := msg.GetStringSliceHeader(edge.InspectRequestValuesHeader) + if err != nil { + resp.Errors = append(resp.Errors, err.Error()) + resp.Success = false + conn.returnInspectResponse(msg, resp) + return + } + + for _, requested := range requestedValues { + lc := strings.ToLower(requested) + if strings.HasPrefix(lc, "circuit:") { + circuitId := requested[len("circuit:"):] + if conn.xgCircuit != nil && conn.circuitId == circuitId { + detail := conn.xgCircuit.xg.GetInspectDetail(false) + resp.Values[requested] = detail + } + } else if strings.HasPrefix(lc, "circuitandstacks:") { + circuitId := requested[len("circuitAndStacks:"):] + if conn.xgCircuit != nil && conn.circuitId == circuitId { + detail := conn.xgCircuit.xg.GetInspectDetail(true) + resp.Values[requested] = detail + } + } + } + + conn.returnInspectResponse(msg, resp) +} + +func (conn *edgeConn) GetCircuitDetail() *xgress.CircuitDetail { + if conn.connType == ConnTypeDial { + detail := &xgress.CircuitDetail{ + CircuitId: conn.circuitId, + ConnId: conn.Id(), + } + + if conn.xgCircuit != nil { + detail.IsXgress = true + detail.Originator = conn.xgCircuit.xg.Originator().String() + detail.Address = string(conn.xgCircuit.xg.Address()) + } + + return detail + } + + return nil +} + +func (conn *edgeConn) returnInspectResponse(msg *channel.Message, resp *inspect.SdkInspectResponse) { + reply, err := edge.NewInspectResponse(conn.Id(), resp) + if err != nil { + pfxlog.Logger().WithError(err).Error("failed to create inspect response") + return + } + reply.ReplyTo(msg) + + if err = reply.WithTimeout(5 * time.Second).Send(conn.GetControlSender()); err != nil { + pfxlog.Logger().WithError(err).Error("failed to send inspect response") + } +} + func (conn *edgeConn) IsClosed() bool { return conn.closed.Load() } @@ -256,6 +383,11 @@ func (conn *edgeConn) SetReadDeadline(t time.Time) error { func (conn *edgeConn) HandleMuxClose() error { conn.close(true) + + // If the channel is closed, stop the send buffer as we can't rtx anything anyway + if xgCircuit := conn.xgCircuit; xgCircuit != nil { + xgCircuit.xg.Close() + } return nil } @@ -270,13 +402,14 @@ func (conn *edgeConn) GetStickinessToken() []byte { func (conn *edgeConn) HandleClose(channel.Channel) { logger := pfxlog.Logger().WithField("connId", conn.Id()).WithField("marker", conn.marker) defer logger.Debug("received HandleClose from underlying channel, marking conn closed") - conn.readQ.Close() - conn.closed.Store(true) + if conn.closed.CompareAndSwap(false, true) { + close(conn.closeNotify) + } conn.sentFIN.Store(true) conn.readFIN.Store(true) } -func (conn *edgeConn) Connect(session *rest_model.SessionDetail, options *edge.DialOptions) (edge.Conn, error) { +func (conn *edgeConn) Connect(session *rest_model.SessionDetail, options *edge.DialOptions, envF func() xgress.Env) (edge.Conn, error) { logger := pfxlog.Logger(). WithField("marker", conn.marker). WithField("connId", conn.Id()). @@ -287,7 +420,9 @@ func (conn *edgeConn) Connect(session *rest_model.SessionDetail, options *edge.D pub = conn.keyPair.Public() } connectRequest := edge.NewConnectMsg(conn.Id(), *session.Token, pub, options) - connectRequest.Headers[edge.ConnectionMarkerHeader] = []byte(conn.marker) + connectRequest.PutStringHeader(edge.ConnectionMarkerHeader, conn.marker) + connectRequest.PutBoolHeader(edge.UseXgressToSdkHeader, options.SdkFlowControl) + conn.TraceMsg("connect", connectRequest) replyMsg, err := connectRequest.WithTimeout(options.ConnectTimeout).SendForReply(conn.GetControlSender()) if err != nil { @@ -303,6 +438,18 @@ func (conn *edgeConn) Connect(session *rest_model.SessionDetail, options *edge.D return nil, errors.Errorf("unexpected response to connect attempt: %v", replyMsg.ContentType) } + conn.circuitId, _ = replyMsg.GetStringHeader(edge.CircuitIdHeader) + if stickinessToken, ok := replyMsg.Headers[edge.StickinessTokenHeader]; ok { + if conn.customState == nil { + conn.customState = map[int32][]byte{} + } + conn.customState[edge.StickinessTokenHeader] = stickinessToken + } + + if err = conn.setupFlowControl(replyMsg, xgress.Initiator, envF); err != nil { + return nil, err + } + if conn.crypto { // There is no race condition where we can receive the other side crypto header // because the processing of the crypto header takes place in Conn.Read which @@ -322,18 +469,47 @@ func (conn *edgeConn) Connect(session *rest_model.SessionDetail, options *edge.D logger.Warn("connection is not end-to-end-encrypted") } } - conn.circuitId, _ = replyMsg.GetStringHeader(edge.CircuitIdHeader) - if stickinessToken, ok := replyMsg.Headers[edge.StickinessTokenHeader]; ok { - if conn.customState == nil { - conn.customState = map[int32][]byte{} - } - conn.customState[edge.StickinessTokenHeader] = stickinessToken - } + logger.Debug("connected") return conn, nil } +func (conn *edgeConn) setupFlowControl(msg *channel.Message, originator xgress.Originator, envF func() xgress.Env) error { + if useXg, _ := msg.GetBoolHeader(edge.UseXgressToSdkHeader); useXg { + ctrlId, ok := msg.GetStringHeader(edge.XgressCtrlIdHeader) + if !ok { + _ = conn.Close() + return fmt.Errorf("xgress conn id header not found for circuit %s", conn.circuitId) + } + addr, ok := msg.GetStringHeader(edge.XgressAddressHeader) + if !ok { + _ = conn.Close() + return fmt.Errorf("xgress address header not found for circuit %s", conn.circuitId) + } + + xgAdapter := &XgAdapter{ + conn: conn, + readC: make(chan []byte), + closeNotify: conn.closeNotify, + env: envF(), + } + conn.xgCircuit = xgAdapter + xg := xgress.NewXgress(conn.circuitId, ctrlId, xgress.Address(addr), xgAdapter, originator, xgress.DefaultOptions(), nil) + xgAdapter.xg = xg + conn.dataSink = xgAdapter + + xg.SetDataPlaneAdapter(xgAdapter) + xg.AddCloseHandler(xgAdapter) + + xg.Start() + } else { + conn.dataSink = &conn.MsgChannel + } + + return nil +} + func (conn *edgeConn) establishClientCrypto(keypair *kx.KeyPair, peerKey []byte, method edge.CryptoMethod) error { var err error var rx, tx []byte @@ -353,7 +529,7 @@ func (conn *edgeConn) establishClientCrypto(keypair *kx.KeyPair, peerKey []byte, conn.rxKey = rx - if _, err = conn.MsgChannel.Write(txHeader); err != nil { + if _, err = conn.dataSink.Write(txHeader); err != nil { return errors.Wrap(err, "failed to write crypto header") } @@ -385,7 +561,7 @@ func (conn *edgeConn) establishServerCrypto(keypair *kx.KeyPair, peerKey []byte, return txHeader, nil } -func (conn *edgeConn) listen(session *rest_model.SessionDetail, service *rest_model.ServiceDetail, options *edge.ListenOptions) (*edgeListener, error) { +func (conn *edgeConn) listen(session *rest_model.SessionDetail, service *rest_model.ServiceDetail, options *edge.ListenOptions, envF func() xgress.Env) (*edgeListener, error) { logger := pfxlog.ContextLogger(conn.GetChannel().Label()). WithField("connId", conn.Id()). WithField("serviceName", *service.Name). @@ -401,6 +577,7 @@ func (conn *edgeConn) listen(session *rest_model.SessionDetail, service *rest_mo edgeChan: conn, manualStart: options.ManualStart, eventC: options.GetEventChannel(), + envF: envF, } logger.Debug("adding listener for session") conn.hosting.Set(*session.Token, listener) @@ -578,6 +755,9 @@ func (conn *edgeConn) close(closedByRemote bool) { if !conn.closed.CompareAndSwap(false, true) { return } + + close(conn.closeNotify) + conn.readFIN.Store(true) conn.sentFIN.Store(true) @@ -592,8 +772,10 @@ func (conn *edgeConn) close(closedByRemote bool) { } } - conn.readQ.Close() - conn.msgMux.RemoveMsgSink(conn) // if we switch back to ChMsgMux will need to be done async again, otherwise we may deadlock + // if we're using xgress, wait to remove the conn from the mux until the xgress closes, otherwise it becomes unroutable. + if conn.xgCircuit == nil { + conn.msgMux.RemoveMsgSink(conn) // if we switch back to ChMsgMux will need to be done async again, otherwise we may deadlock + } if conn.connType == ConnTypeBind { for entry := range conn.hosting.IterBuffered() { @@ -643,9 +825,11 @@ func (conn *edgeConn) newChildConnection(message *channel.Message) { sourceIdentity, _ := message.GetStringHeader(edge.CallerIdHeader) marker, _ := message.GetStringHeader(edge.ConnectionMarkerHeader) + closeNotify := make(chan struct{}) edgeCh := &edgeConn{ + closeNotify: closeNotify, MsgChannel: *edge.NewEdgeMsgChannel(conn.SdkChannel, id), - readQ: NewNoopSequencer[*channel.Message](4), + readQ: NewNoopSequencer[*channel.Message](closeNotify, 4), msgMux: conn.msgMux, sourceIdentity: sourceIdentity, crypto: conn.crypto, @@ -664,6 +848,8 @@ func (conn *edgeConn) newChildConnection(message *channel.Message) { err := conn.msgMux.AddMsgSink(edgeCh) // duplicate errors only happen on the server side, since client controls ids if err != nil { + conn.close(true) + newConnLogger.WithError(err).Error("invalid conn id, already in use") reply := edge.NewDialFailedMsg(conn.Id(), err.Error()) reply.ReplyTo(message) @@ -673,6 +859,16 @@ func (conn *edgeConn) newChildConnection(message *channel.Message) { return } + if err = edgeCh.setupFlowControl(message, xgress.Terminator, listener.envF); err != nil { + logger.WithError(err).Error("failed to start flow control") + reply := edge.NewDialFailedMsg(conn.Id(), fmt.Sprintf("failed to start flow control (%s)", err.Error())) + reply.ReplyTo(message) + if err := reply.WithPriority(channel.Highest).WithTimeout(5 * time.Second).SendAndWaitForWire(conn.GetControlSender()); err != nil { + logger.WithError(err).Error("failed to send reply to dial request") + } + return + } + var txHeader []byte if edgeCh.crypto { newConnLogger.Debug("setting up crypto") @@ -689,6 +885,7 @@ func (conn *edgeConn) newChildConnection(message *channel.Message) { } if err != nil { + conn.close(true) newConnLogger.WithError(err).Error("failed to establish connection") reply := edge.NewDialFailedMsg(conn.Id(), err.Error()) reply.ReplyTo(message) @@ -824,7 +1021,7 @@ func (self *newConnHandler) dialSucceeded() error { if self.txHeader != nil { newConnLogger.Debug("sending crypto header") - if _, err := self.edgeCh.MsgChannel.Write(self.txHeader); err != nil { + if _, err := self.edgeCh.dataSink.Write(self.txHeader); err != nil { newConnLogger.WithError(err).Error("failed to write crypto header") return err } diff --git a/ziti/edge/network/conn_test.go b/ziti/edge/network/conn_test.go index dc5f8c9c..acaafe62 100644 --- a/ziti/edge/network/conn_test.go +++ b/ziti/edge/network/conn_test.go @@ -22,18 +22,21 @@ func BenchmarkConnWriteBaseLine(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - msg := edge.NewDataMsg(1, uint32(i), data) + msg := edge.NewDataMsg(1, data) err := testChannel.Send(msg) req.NoError(err) } } func BenchmarkConnWrite(b *testing.B) { + closeNotify := make(chan struct{}) + defer close(closeNotify) + mux := edge.NewCowMapMsgMux() testChannel := edge.NewSingleSdkChannel(&NoopTestChannel{}) conn := &edgeConn{ MsgChannel: *edge.NewEdgeMsgChannel(testChannel, 1), - readQ: NewNoopSequencer[*channel.Message](4), + readQ: NewNoopSequencer[*channel.Message](closeNotify, 4), msgMux: mux, serviceName: "test", } @@ -52,10 +55,13 @@ func BenchmarkConnWrite(b *testing.B) { } func BenchmarkConnRead(b *testing.B) { + closeNotify := make(chan struct{}) + defer close(closeNotify) + mux := edge.NewCowMapMsgMux() testChannel := edge.NewSingleSdkChannel(&NoopTestChannel{}) - readQ := NewNoopSequencer[*channel.Message](4) + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) conn := &edgeConn{ MsgChannel: *edge.NewEdgeMsgChannel(testChannel, 1), readQ: readQ, @@ -71,7 +77,7 @@ func BenchmarkConnRead(b *testing.B) { for !stop.Load() { counter += 1 data := make([]byte, 877) - msg := edge.NewDataMsg(1, counter, data) + msg := edge.NewDataMsg(1, data) err := readQ.PutSequenced(msg) if err != nil { panic(err) @@ -104,7 +110,7 @@ func BenchmarkSequencer(b *testing.B) { for !stop.Load() { counter += 1 data := make([]byte, 877) - msg := edge.NewDataMsg(1, counter, data) + msg := edge.NewDataMsg(1, data) event := &edge.MsgEvent{ ConnId: 1, Seq: counter, @@ -125,10 +131,14 @@ func BenchmarkSequencer(b *testing.B) { func TestReadMultipart(t *testing.T) { req := require.New(t) + + closeNotify := make(chan struct{}) + defer close(closeNotify) + mux := edge.NewCowMapMsgMux() testChannel := edge.NewSingleSdkChannel(&NoopTestChannel{}) - readQ := NewNoopSequencer[*channel.Message](4) + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) conn := &edgeConn{ MsgChannel: *edge.NewEdgeMsgChannel(testChannel, 1), readQ: readQ, @@ -145,10 +155,10 @@ func TestReadMultipart(t *testing.T) { multipart = binary.LittleEndian.AppendUint16(multipart, uint16(len(w))) multipart = append(multipart, []byte(w)...) } - msg := edge.NewDataMsg(1, uint32(0), multipart) + msg := edge.NewDataMsg(1, multipart) msg.Headers.PutUint32Header(edge.FlagsHeader, uint32(edge.MULTIPART_MSG)) _ = readQ.PutSequenced(msg) - msg = edge.NewDataMsg(1, uint32(0), nil) + msg = edge.NewDataMsg(1, nil) msg.Headers.PutUint32Header(edge.FlagsHeader, uint32(edge.FIN)) err := readQ.PutSequenced(msg) if err != nil { @@ -175,6 +185,10 @@ func TestReadMultipart(t *testing.T) { type NoopTestChannel struct { } +func (ch *NoopTestChannel) GetUserData() interface{} { + return nil +} + func (ch *NoopTestChannel) Headers() map[int32][]byte { return nil } diff --git a/ziti/edge/network/factory.go b/ziti/edge/network/factory.go index 0107de4a..3ebf2709 100644 --- a/ziti/edge/network/factory.go +++ b/ziti/edge/network/factory.go @@ -18,6 +18,7 @@ package network import ( "fmt" + "github.com/openziti/sdk-golang/xgress" "time" "github.com/michaelquigley/pfxlog" @@ -84,6 +85,10 @@ func (conn *routerConn) BindChannel(binding channel.Binding) error { binding.AddReceiveHandlerF(edge.ContentTypeTraceRoute, conn.msgMux.HandleReceive) binding.AddReceiveHandlerF(edge.ContentTypeConnInspectRequest, conn.msgMux.HandleReceive) binding.AddReceiveHandlerF(edge.ContentTypeBindSuccess, conn.msgMux.HandleReceive) + binding.AddReceiveHandlerF(edge.ContentTypeXgPayload, conn.msgMux.HandleReceive) + binding.AddReceiveHandlerF(edge.ContentTypeXgAcknowledgement, conn.msgMux.HandleReceive) + binding.AddReceiveHandlerF(edge.ContentTypeXgControl, conn.msgMux.HandleReceive) + binding.AddReceiveHandlerF(edge.ContentTypeInspectRequest, conn.msgMux.HandleReceive) // Since data is the common message type, it gets to be dispatched directly binding.AddTypedReceiveHandler(conn.msgMux) @@ -96,9 +101,11 @@ func (conn *routerConn) BindChannel(binding channel.Binding) error { func (conn *routerConn) NewDialConn(service *rest_model.ServiceDetail) *edgeConn { id := conn.msgMux.GetNextId() + closeNotify := make(chan struct{}) edgeCh := &edgeConn{ + closeNotify: closeNotify, MsgChannel: *edge.NewEdgeMsgChannel(conn.ch, id), - readQ: NewNoopSequencer[*channel.Message](4), + readQ: NewNoopSequencer[*channel.Message](closeNotify, 4), msgMux: conn.msgMux, serviceName: *service.Name, connType: ConnTypeDial, @@ -145,9 +152,11 @@ func (conn *routerConn) UpdateToken(token []byte, timeout time.Duration) error { func (conn *routerConn) NewListenConn(service *rest_model.ServiceDetail, keyPair *kx.KeyPair) *edgeConn { id := conn.msgMux.GetNextId() + closeNotify := make(chan struct{}) edgeCh := &edgeConn{ + closeNotify: closeNotify, MsgChannel: *edge.NewEdgeMsgChannel(conn.ch, id), - readQ: NewNoopSequencer[*channel.Message](4), + readQ: NewNoopSequencer[*channel.Message](closeNotify, 4), msgMux: conn.msgMux, serviceName: *service.Name, connType: ConnTypeBind, @@ -155,6 +164,7 @@ func (conn *routerConn) NewListenConn(service *rest_model.ServiceDetail, keyPair crypto: keyPair != nil, hosting: cmap.New[*edgeListener](), } + edgeCh.dataSink = &edgeCh.MsgChannel // duplicate errors only happen on the server side, since client controls ids if err := conn.msgMux.AddMsgSink(edgeCh); err != nil { @@ -168,9 +178,9 @@ func (conn *routerConn) NewListenConn(service *rest_model.ServiceDetail, keyPair return edgeCh } -func (conn *routerConn) Connect(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *edge.DialOptions) (edge.Conn, error) { +func (conn *routerConn) Connect(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *edge.DialOptions, envF func() xgress.Env) (edge.Conn, error) { ec := conn.NewDialConn(service) - dialConn, err := ec.Connect(session, options) + dialConn, err := ec.Connect(session, options, envF) if err != nil { if err2 := ec.Close(); err2 != nil { pfxlog.Logger().Errorf("failed to cleanup connection for service '%v' (%v)", service.Name, err2) @@ -179,7 +189,7 @@ func (conn *routerConn) Connect(service *rest_model.ServiceDetail, session *rest return dialConn, err } -func (conn *routerConn) Listen(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *edge.ListenOptions) (edge.Listener, error) { +func (conn *routerConn) Listen(service *rest_model.ServiceDetail, session *rest_model.SessionDetail, options *edge.ListenOptions, envF func() xgress.Env) (edge.Listener, error) { ec := conn.NewListenConn(service, options.KeyPair) log := pfxlog.Logger(). @@ -188,7 +198,7 @@ func (conn *routerConn) Listen(service *rest_model.ServiceDetail, session *rest_ WithField("serviceId", *service.ID). WithField("serviceName", *service.Name) - listener, err := ec.listen(session, service, options) + listener, err := ec.listen(session, service, options, envF) if err != nil { log.WithError(err).Error("failed to establish listener") diff --git a/ziti/edge/network/listener.go b/ziti/edge/network/listener.go index 28e605ed..ba96ef95 100644 --- a/ziti/edge/network/listener.go +++ b/ziti/edge/network/listener.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/michaelquigley/pfxlog" "github.com/openziti/edge-api/rest_model" + "github.com/openziti/sdk-golang/xgress" "github.com/openziti/sdk-golang/ziti/edge" "github.com/pkg/errors" "math" @@ -94,6 +95,7 @@ type edgeListener struct { manualStart bool established atomic.Bool eventC chan *edge.ListenerEvent + envF func() xgress.Env } func (listener *edgeListener) Id() uint32 { diff --git a/ziti/edge/network/msg_timer.go b/ziti/edge/network/msg_timer.go new file mode 100644 index 00000000..6755b53d --- /dev/null +++ b/ziti/edge/network/msg_timer.go @@ -0,0 +1,127 @@ +package network + +import ( + "fmt" + "github.com/openziti/channel/v4" + "github.com/openziti/metrics" + "sort" + "strings" + "sync/atomic" + "time" +) + +type timingReceiveHandler struct { + handler channel.ReceiveHandler + timer metrics.Histogram +} + +func (t *timingReceiveHandler) HandleReceive(m *channel.Message, ch channel.Channel) { + start := time.Now() + t.handler.HandleReceive(m, ch) + t.timer.Update(int64(time.Since(start))) +} + +func NewMessageTimingBinding(binding channel.Binding) channel.Binding { + registry := metrics.NewRegistry("", nil) + wrapper := &messageTimingBinding{ + binding: binding, + registry: registry, + } + closeNotify := make(chan struct{}) + closed := atomic.Bool{} + binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { + if closed.CompareAndSwap(false, true) { + close(closeNotify) + } + })) + reporter := metrics.NewDelegatingReporter(registry, wrapper, closeNotify) + go reporter.Start(5 * time.Second) + return wrapper +} + +type messageTimingBinding struct { + registry metrics.Registry + binding channel.Binding + output []string +} + +func (self *messageTimingBinding) StartReport(metrics.Registry) {} + +func (self *messageTimingBinding) EndReport(metrics.Registry) { + sort.Strings(self.output) + for _, output := range self.output { + fmt.Println(output) + } + self.output = nil +} + +func (self *messageTimingBinding) Printf(msg string, args ...interface{}) { + self.output = append(self.output, fmt.Sprintf(msg, args...)) +} + +func (self *messageTimingBinding) Filter(name string) bool { + return strings.HasSuffix(name, metrics.MetricNameCount) || + strings.HasSuffix(name, metrics.MetricNamePercentile) +} + +func (self *messageTimingBinding) AcceptIntMetric(name string, value int64) { + self.Printf("%s -> %d", name, value) +} + +func (self *messageTimingBinding) AcceptFloatMetric(name string, value float64) { + self.Printf("%s -> %f", name, value) +} + +func (self *messageTimingBinding) AcceptPercentileMetric(name string, value metrics.PercentileSource) { + self.Printf("%s.50p -> %s", name, time.Duration(value.Percentile(.5)).String()) + self.Printf("%s.75p -> %s", name, time.Duration(value.Percentile(.75)).String()) + self.Printf("%s.95p -> %s", name, time.Duration(value.Percentile(.95)).String()) +} + +func (self *messageTimingBinding) Bind(h channel.BindHandler) error { + return h.BindChannel(self) +} + +func (self *messageTimingBinding) AddPeekHandler(h channel.PeekHandler) { + self.binding.AddPeekHandler(h) +} + +func (self *messageTimingBinding) AddTransformHandler(h channel.TransformHandler) { + self.binding.AddTransformHandler(h) +} + +func (self *messageTimingBinding) AddReceiveHandler(contentType int32, h channel.ReceiveHandler) { + timer := self.registry.Histogram(fmt.Sprintf("msgs.%d.time", contentType)) + self.binding.AddReceiveHandler(contentType, &timingReceiveHandler{ + handler: h, + timer: timer, + }) +} + +func (self *messageTimingBinding) AddReceiveHandlerF(contentType int32, h channel.ReceiveHandlerF) { + self.AddReceiveHandler(contentType, h) +} + +func (self *messageTimingBinding) AddTypedReceiveHandler(h channel.TypedReceiveHandler) { + self.AddReceiveHandler(h.ContentType(), h) +} + +func (self *messageTimingBinding) AddErrorHandler(h channel.ErrorHandler) { + self.binding.AddErrorHandler(h) +} + +func (self *messageTimingBinding) AddCloseHandler(h channel.CloseHandler) { + self.binding.AddCloseHandler(h) +} + +func (self *messageTimingBinding) SetUserData(data interface{}) { + self.binding.SetUserData(data) +} + +func (self *messageTimingBinding) GetUserData() interface{} { + return self.binding.GetUserData() +} + +func (self *messageTimingBinding) GetChannel() channel.Channel { + return self.binding.GetChannel() +} diff --git a/ziti/edge/network/seq.go b/ziti/edge/network/seq.go index d0721121..a7ea3d28 100644 --- a/ziti/edge/network/seq.go +++ b/ziti/edge/network/seq.go @@ -23,20 +23,19 @@ func (r ReadTimout) Temporary() bool { return true } -func NewNoopSequencer[T any](channelDepth int) *noopSeq[T] { +func NewNoopSequencer[T any](closeNotify <-chan struct{}, channelDepth int) *noopSeq[T] { return &noopSeq[T]{ + closeNotify: closeNotify, ch: make(chan T, channelDepth), - closeNotify: make(chan struct{}), deadlineNotify: make(chan struct{}), } } type noopSeq[T any] struct { ch chan T - closeNotify chan struct{} + closeNotify <-chan struct{} deadlineNotify chan struct{} deadline concurrenz.AtomicValue[time.Time] - closed atomic.Bool readInProgress atomic.Bool } @@ -103,9 +102,3 @@ func (seq *noopSeq[T]) GetNext() (T, error) { } } } - -func (seq *noopSeq[T]) Close() { - if seq.closed.CompareAndSwap(false, true) { - close(seq.closeNotify) - } -} diff --git a/ziti/edge/network/seq_test.go b/ziti/edge/network/seq_test.go index dac87ab2..35705bee 100644 --- a/ziti/edge/network/seq_test.go +++ b/ziti/edge/network/seq_test.go @@ -9,7 +9,10 @@ import ( ) func Test_SeqNormalReadDeadline(t *testing.T) { - readQ := NewNoopSequencer[*channel.Message](4) + closeNotify := make(chan struct{}) + defer close(closeNotify) + + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) start := time.Now() readQ.SetReadDeadline(start.Add(10 * time.Millisecond)) @@ -28,14 +31,17 @@ func Test_SeqNormalReadDeadline(t *testing.T) { } func Test_SeqNormalReadWithDeadline(t *testing.T) { - readQ := NewNoopSequencer[*channel.Message](4) + closeNotify := make(chan struct{}) + defer close(closeNotify) + + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) start := time.Now() readQ.SetReadDeadline(start.Add(10 * time.Millisecond)) req := require.New(t) data := make([]byte, 877) - msg := edge.NewDataMsg(1, 1, data) + msg := edge.NewDataMsg(1, data) req.NoError(readQ.PutSequenced(msg)) val, err := readQ.GetNext() @@ -44,11 +50,14 @@ func Test_SeqNormalReadWithDeadline(t *testing.T) { } func Test_SeqNormalReadWithNoDeadline(t *testing.T) { - readQ := NewNoopSequencer[*channel.Message](4) + closeNotify := make(chan struct{}) + defer close(closeNotify) + + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) req := require.New(t) data := make([]byte, 877) - msg := edge.NewDataMsg(1, 1, data) + msg := edge.NewDataMsg(1, data) req.NoError(readQ.PutSequenced(msg)) val, err := readQ.GetNext() @@ -57,7 +66,10 @@ func Test_SeqNormalReadWithNoDeadline(t *testing.T) { } func Test_SeqReadWithInterrupt(t *testing.T) { - readQ := NewNoopSequencer[*channel.Message](4) + closeNotify := make(chan struct{}) + defer close(closeNotify) + + readQ := NewNoopSequencer[*channel.Message](closeNotify, 4) start := time.Now() req := require.New(t) diff --git a/ziti/edge/network/xg_adapter.go b/ziti/edge/network/xg_adapter.go new file mode 100644 index 00000000..317af003 --- /dev/null +++ b/ziti/edge/network/xg_adapter.go @@ -0,0 +1,145 @@ +package network + +import ( + "fmt" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/channel/v4" + "github.com/openziti/sdk-golang/edgexg" + "github.com/openziti/sdk-golang/xgress" + "github.com/openziti/sdk-golang/ziti/edge" + "github.com/sirupsen/logrus" + "io" + "time" +) + +type XgAdapter struct { + conn *edgeConn + readC chan []byte + closeNotify <-chan struct{} + env xgress.Env + xg *xgress.Xgress +} + +func (self *XgAdapter) HandleXgressClose(x *xgress.Xgress) { + self.xg.ForwardEndOfCircuit(func(payload *xgress.Payload) bool { + self.ForwardPayload(payload, x) + return true + }) + self.conn.close(true) + + // see note in close + self.conn.msgMux.RemoveMsgSink(self.conn) + + xgCloseMsg := channel.NewMessage(edge.ContentTypeXgClose, []byte(self.xg.CircuitId())) + if err := xgCloseMsg.WithTimeout(5 * time.Second).Send(self.conn.SdkChannel.GetControlSender()); err != nil { + pfxlog.Logger().WithError(err).Error("failed to send close xg close message") + } +} + +func (self *XgAdapter) ForwardPayload(payload *xgress.Payload, x *xgress.Xgress) { + msg := payload.Marshall() + msg.PutUint32Header(edge.ConnIdHeader, self.conn.Id()) + if err := self.conn.MsgChannel.GetDefaultSender().Send(msg); err != nil { + pfxlog.Logger().WithError(err).Error("failed to send payload") + } +} + +func (self *XgAdapter) RetransmitPayload(srcAddr xgress.Address, payload *xgress.Payload) error { + msg := payload.Marshall() + return self.conn.MsgChannel.GetDefaultSender().Send(msg) +} + +func (self *XgAdapter) ForwardControlMessage(control *xgress.Control, x *xgress.Xgress) { + msg := control.Marshall() + if err := self.conn.MsgChannel.GetDefaultSender().Send(msg); err != nil { + pfxlog.Logger().WithError(err).Error("failed to forward control message") + } +} + +func (self *XgAdapter) ForwardAcknowledgement(ack *xgress.Acknowledgement, address xgress.Address) { + msg := ack.Marshall() + if err := self.conn.MsgChannel.GetDefaultSender().Send(msg); err != nil { + pfxlog.Logger().WithError(err).Error("failed to send acknowledgement") + } +} + +func (self *XgAdapter) GetRetransmitter() *xgress.Retransmitter { + return self.env.GetRetransmitter() +} + +func (self *XgAdapter) GetPayloadIngester() *xgress.PayloadIngester { + return self.env.GetPayloadIngester() +} + +func (self *XgAdapter) GetMetrics() xgress.Metrics { + return self.env.GetMetrics() +} + +func (self *XgAdapter) Close() error { + return self.conn.Close() +} + +func (self *XgAdapter) LogContext() string { + return fmt.Sprintf("xg/%s", self.conn.GetCircuitId()) +} + +func (self *XgAdapter) Write(bytes []byte) (int, error) { + select { + case self.readC <- bytes: + return len(bytes), nil + case <-self.closeNotify: + return 0, io.EOF + } +} + +func (self *XgAdapter) ReadPayload() ([]byte, map[uint8][]byte, error) { + // log := pfxlog.ContextLogger(self.LogContext()).WithField("connId", self.conn.Id()) + + var data []byte + select { + case data = <-self.readC: + case <-self.closeNotify: + return nil, nil, io.EOF + } + + return data, nil, nil +} + +func (self *XgAdapter) WritePayload(bytes []byte, headers map[uint8][]byte) (int, error) { + var msgUUID []byte + var edgeHdrs map[int32][]byte + + if headers != nil { + msgUUID = headers[xgress.HeaderKeyUUID] + + edgeHdrs = make(map[int32][]byte) + for k, v := range headers { + if edgeHeader, found := edgexg.HeadersFromFabric[k]; found { + edgeHdrs[edgeHeader] = v + } + } + } + + msg := edge.NewDataMsg(self.conn.Id(), bytes) + if msgUUID != nil { + msg.Headers[edge.UUIDHeader] = msgUUID + } + + for k, v := range edgeHdrs { + msg.Headers[k] = v + } + + if err := self.conn.readQ.PutSequenced(msg); err != nil { + logrus.WithFields(edge.GetLoggerFields(msg)).WithError(err). + Error("error pushing edge message to sequencer") + return 0, err + } + + logrus.WithFields(edge.GetLoggerFields(msg)).Debugf("received %v bytes (msg type: %v)", len(msg.Body), msg.ContentType) + return len(msg.Body), nil +} + +func (self *XgAdapter) HandleControlMsg(controlType xgress.ControlType, headers channel.Headers, responder xgress.ControlReceiver) error { + //TODO implement me + panic("implement me") +} diff --git a/ziti/options.go b/ziti/options.go index fe4842d1..ffb48925 100644 --- a/ziti/options.go +++ b/ziti/options.go @@ -55,6 +55,21 @@ type DialOptions struct { Identity string AppData []byte StickinessToken []byte + + // WARNING: Experimental setting, may be removed and/or defaults may change + // + // If set to true, flow-control will be managed from the SDK instead of the router. This prevents + // multiplexed circuits from interfering with each other. If the embedding application is only + // opening one circuit at a time, then it should not make much difference. Setting this to true + // may affect memory use and network traffic. + // + // NOTES: + // 1. Currently defaults to false, but may change to true in the future. + // 2. Requires router side support + // 3. Note that if config.MaxDefaultConnects is set to a value greater than one, the SDK will + // always use sdk flow-control, as otherwise multiple data channels can lead to out-of-order + // data corruption. + SdkFlowControl *bool } func (d DialOptions) GetConnectTimeout() time.Duration { @@ -96,6 +111,17 @@ type ListenOptions struct { // If set to true, requires that AcceptEdge is called on the edge.Listener ManualStart bool + // NOTE: Experimental setting, may be removed and/or defaults may change + // + // If set to true, flow-control will be managed from the SDK instead of the router. This prevents + // multiplexed circuits from interfering with each other. If the embedding application is only + // opening one circuit at a time, then it should not make much difference. Setting this to true + // may affect memory use and network traffic. + // + // Currently defaults to false, but may change to true in the future. + // Requires router side support + SdkFlowControl *bool + // Wait for N listeners before returning from the Listen call. By default it will return // before any listeners have been established. WaitForNEstablishedListeners uint diff --git a/ziti/sdkinfo/build_info.go b/ziti/sdkinfo/build_info.go index 2f1a8806..c936e801 100644 --- a/ziti/sdkinfo/build_info.go +++ b/ziti/sdkinfo/build_info.go @@ -20,5 +20,5 @@ package sdkinfo const ( - Version = "v1.0.3" + Version = "v1.1.0" ) diff --git a/ziti/xg_env.go b/ziti/xg_env.go new file mode 100644 index 00000000..dbe0c0d0 --- /dev/null +++ b/ziti/xg_env.go @@ -0,0 +1,39 @@ +package ziti + +import ( + "github.com/openziti/metrics" + "github.com/openziti/sdk-golang/xgress" +) + +type xgEnv struct { + retransmitter *xgress.Retransmitter + payloadIngester *xgress.PayloadIngester + metrics xgress.Metrics +} + +func NewXgressEnv(closeNotify <-chan struct{}, registry metrics.Registry) xgress.Env { + return &xgEnv{ + retransmitter: xgress.NewRetransmitter(dummyRetransmitterFaultReporter{}, registry, closeNotify), + payloadIngester: xgress.NewPayloadIngester(closeNotify), + metrics: xgress.NewMetrics(registry), + } +} + +func (x xgEnv) GetRetransmitter() *xgress.Retransmitter { + return x.retransmitter +} + +func (x xgEnv) GetPayloadIngester() *xgress.PayloadIngester { + return x.payloadIngester +} + +func (x xgEnv) GetMetrics() xgress.Metrics { + return x.metrics +} + +type dummyRetransmitterFaultReporter struct{} + +func (d dummyRetransmitterFaultReporter) ReportForwardingFault(circuitId string, ctrlId string) { + // the only way to get a fault is if the connection goes down, in which case the circuit will + // get torn down anyway +} diff --git a/ziti/ziti.go b/ziti/ziti.go index b910efad..52012800 100644 --- a/ziti/ziti.go +++ b/ziti/ziti.go @@ -29,6 +29,7 @@ import ( "github.com/openziti/foundation/v2/concurrenz" "github.com/openziti/foundation/v2/stringz" apis "github.com/openziti/sdk-golang/edge-apis" + "github.com/openziti/sdk-golang/xgress" "github.com/openziti/secretstream/kx" "math" "math/rand" @@ -195,7 +196,11 @@ type ContextImpl struct { lastSuccessfulApiSessionRefresh time.Time routerProxy func(addr string) *transport.ProxyConfiguration - enableCtrlPlaneConnection bool + maxControlConnections int + maxDefaultConnections int + + lock sync.Mutex + xgressEnv xgress.Env } func (context *ContextImpl) AddServiceAddedListener(handler func(Context, *rest_model.ServiceDetail)) func() { @@ -1075,6 +1080,7 @@ func (context *ContextImpl) DialWithOptions(serviceName string, options *DialOpt Identity: options.Identity, AppData: options.AppData, StickinessToken: options.StickinessToken, + SdkFlowControl: (options.SdkFlowControl != nil && *options.SdkFlowControl) || context.maxDefaultConnections > 1, } if edgeDialOptions.GetConnectTimeout() == 0 { edgeDialOptions.ConnectTimeout = 15 * time.Second @@ -1208,7 +1214,7 @@ func (context *ContextImpl) dialSession(service *rest_model.ServiceDetail, sessi if err != nil { return nil, err } - return edgeConnFactory.Connect(service, session, options) + return edgeConnFactory.Connect(service, session, options, context.getXgressEnv) } func (context *ContextImpl) ensureApiSession() error { @@ -1257,6 +1263,10 @@ func (context *ContextImpl) listenSession(service *rest_model.ServiceDetail, opt edgeListenOptions.MaxTerminators = 1 } + // TODO: uncomment + //edgeListenOptions.SdkFlowControl = options.SdkFlowControl != nil && *options.SdkFlowControl + edgeListenOptions.SdkFlowControl = true + if listenerMgr, err := newListenerManager(service, context, edgeListenOptions, options.WaitForNEstablishedListeners); err != nil { return nil, err } else { @@ -1415,7 +1425,7 @@ func (context *ContextImpl) connectEdgeRouter(routerName, ingressUrl string) *ed options.ConnectTimeout = 15 * time.Second headers := channel.Headers{} - if context.enableCtrlPlaneConnection { + if context.maxControlConnections > 0 || context.maxDefaultConnections > 1 { headers.PutBoolHeader(channel.IsGroupedHeader, true) headers.PutStringHeader(channel.TypeHeader, edge.ChannelTypeDefault) } @@ -1432,7 +1442,7 @@ func (context *ContextImpl) connectEdgeRouter(routerName, ingressUrl string) *ed var ch channel.Channel if isGrouped, _ := channel.Headers(underlay.Headers()).GetBoolHeader(channel.IsGroupedHeader); isGrouped { - var dialSdkChannel = edge.NewDialSdkChannel(dialer, underlay) + var dialSdkChannel = edge.NewDialSdkChannel(dialer, underlay, context.maxDefaultConnections, context.maxControlConnections) multiChannelConfig := &channel.MultiChannelConfig{ LogicalName: fmt.Sprintf("ziti-sdk[router=%v]", ingressUrl), Options: options, @@ -1478,6 +1488,7 @@ func (context *ContextImpl) connectEdgeRouter(routerName, ingressUrl string) *ed func(exist bool, oldV edge.RouterConn, newV edge.RouterConn) edge.RouterConn { if exist { // use the routerConnection already in the map, close new one pfxlog.Logger().Infof("connection to %s already established, closing duplicate connection", ingressUrl) + go func() { if err := newV.Close(); err != nil { pfxlog.Logger().Errorf("unable to close router connection (%v)", err) @@ -1724,6 +1735,9 @@ func (context *ContextImpl) Close() { } func (context *ContextImpl) Metrics() metrics.Registry { + if context.metrics == nil { + _ = context.Authenticate() + } return context.metrics } @@ -1734,10 +1748,20 @@ func (context *ContextImpl) EnrollZitiMfa() (*rest_model.DetailMfa, error) { func (context *ContextImpl) VerifyZitiMfa(code string) error { return context.CtrlClt.VerifyMfa(code) } + func (context *ContextImpl) RemoveZitiMfa(code string) error { return context.CtrlClt.RemoveMfa(code) } +func (context *ContextImpl) getXgressEnv() xgress.Env { + context.lock.Lock() + defer context.lock.Unlock() + if context.xgressEnv == nil { + context.xgressEnv = NewXgressEnv(context.closeNotify, context.metrics) + } + return context.xgressEnv +} + type waitForNHelper struct { count uint mgr *listenerManager @@ -1998,7 +2022,7 @@ func (mgr *listenerManager) createListener(routerConnection edge.RouterConn, ses logger := pfxlog.Logger().WithField("serviceName", *mgr.service.Name). WithField("router", routerConnection.GetRouterName()) svc := mgr.listener.GetService() - listener, err := routerConnection.Listen(svc, session, mgr.options) + listener, err := routerConnection.Listen(svc, session, mgr.options, mgr.context.getXgressEnv) elapsed := time.Since(start) if err == nil { logger = logger.WithField("connId", listener.Id()) From 9db5a2edce22160410de8f6735a244199cabf3d2 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Tue, 29 Apr 2025 11:29:32 -0400 Subject: [PATCH 2/3] Fix verbiage --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad7a7df5..9cb300c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,13 @@ ## SDK Flow Control If the router being connected to supports it, the sdk can now manage flow control -instead of delegating that to the router. This is mostly importing when running +instead of delegating that to the router. This is mostly important when running multiple simultaneous circuits throught the SDK. When running multiple circuits, a slow circuit can get stalled at the router because of flow control back-pressure. This then back-pressures all circuits from the SDK to that router. -By moving the flow-control to the SDK, a slow circuit will not negatively other -circuits to the same router. This is currently enabled in the `DialOptions`. +By moving the flow-control to the SDK, a slow circuit will not negatively impact +other circuits to the same router. This is currently enabled in the `DialOptions`. ``` t := true From 091a7926e8f3d6fb52a90cc2a5fc14fc436dadc9 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Tue, 29 Apr 2025 11:37:09 -0400 Subject: [PATCH 3/3] Fix ListenOptions UseSdkFlowControl and update changelog --- CHANGELOG.md | 6 +++++- example/go.mod | 2 +- example/go.sum | 4 ++-- example/influxdb-client-go/go.mod | 2 +- example/influxdb-client-go/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- xgress/link_send_buffer.go | 11 +++-------- ziti/ziti.go | 4 +--- 9 files changed, 18 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb300c5..2edf746c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ a slow circuit can get stalled at the router because of flow control back-pressu This then back-pressures all circuits from the SDK to that router. By moving the flow-control to the SDK, a slow circuit will not negatively impact -other circuits to the same router. This is currently enabled in the `DialOptions`. +other circuits to the same router. This is currently enabled in the `DialOptions` +and `ListenOptions` for the dial and hosting sides respectively. ``` t := true @@ -22,6 +23,9 @@ dialOptions := &ziti.DialOptions{ ConnectTimeout: wf.ConnectTimeout, SdkFlowControl: &t, } + +listenOptions := ziti.DefaultListenOptions() +listenOptions.SdkFlowControl = &t ``` As this is an experimental feature, the configuration may change or be removed diff --git a/example/go.mod b/example/go.mod index ad69a987..a98af809 100644 --- a/example/go.mod +++ b/example/go.mod @@ -83,7 +83,7 @@ require ( github.com/openziti/channel/v4 v4.0.6 // indirect github.com/openziti/edge-api v0.26.42 // indirect github.com/openziti/identity v1.0.101 // indirect - github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 // indirect + github.com/openziti/metrics v1.4.1 // indirect github.com/openziti/secretstream v0.1.32 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/parallaxsecond/parsec-client-go v0.0.0-20221025095442-f0a77d263cf9 // indirect diff --git a/example/go.sum b/example/go.sum index a093425d..1455d8db 100644 --- a/example/go.sum +++ b/example/go.sum @@ -366,8 +366,8 @@ github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8p github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/metrics v1.4.1 h1:JMnfmW8liTzmqJpd7HDPCSvDDFg1UrE0WoLhW9XdIG8= +github.com/openziti/metrics v1.4.1/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/runzmd v1.0.33 h1:tOyjRoUuVXIo1z1pNU32jALWkMmhzsSaDrhLtuOn3Ts= github.com/openziti/runzmd v1.0.33/go.mod h1:8c/uvZR/XWXQNllTq6LuTpfKL2DTNxfI2X2wYhgRwik= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= diff --git a/example/influxdb-client-go/go.mod b/example/influxdb-client-go/go.mod index f0edf8bf..cd4e74fb 100644 --- a/example/influxdb-client-go/go.mod +++ b/example/influxdb-client-go/go.mod @@ -98,7 +98,7 @@ require ( github.com/openziti/edge-api v0.26.42 // indirect github.com/openziti/foundation/v2 v2.0.59 // indirect github.com/openziti/identity v1.0.101 // indirect - github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 // indirect + github.com/openziti/metrics v1.4.1 // indirect github.com/openziti/secretstream v0.1.32 // indirect github.com/openziti/transport/v2 v2.0.168 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect diff --git a/example/influxdb-client-go/go.sum b/example/influxdb-client-go/go.sum index 585dca72..35fc4fa2 100644 --- a/example/influxdb-client-go/go.sum +++ b/example/influxdb-client-go/go.sum @@ -421,8 +421,8 @@ github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8p github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/metrics v1.4.1 h1:JMnfmW8liTzmqJpd7HDPCSvDDFg1UrE0WoLhW9XdIG8= +github.com/openziti/metrics v1.4.1/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= github.com/openziti/secretstream v0.1.32/go.mod h1:8YaIbjyMwBeKQ7eOYcoVPKHT10u+4OVPXpnZAeDzC6o= github.com/openziti/transport/v2 v2.0.168 h1:1Anf7X+4xmSKQ12GdPJFhoMZi04QxgD4MJu3agFc1R4= diff --git a/go.mod b/go.mod index aedc8a08..1103338c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/openziti/edge-api v0.26.42 github.com/openziti/foundation/v2 v2.0.59 github.com/openziti/identity v1.0.101 - github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 + github.com/openziti/metrics v1.4.1 github.com/openziti/secretstream v0.1.32 github.com/openziti/transport/v2 v2.0.168 github.com/orcaman/concurrent-map/v2 v2.0.1 diff --git a/go.sum b/go.sum index 55758ec5..fdd1ee22 100644 --- a/go.sum +++ b/go.sum @@ -307,8 +307,8 @@ github.com/openziti/foundation/v2 v2.0.59 h1:PJwrcTq62x+cONBeKMlnsuphsTlOvTz8j8p github.com/openziti/foundation/v2 v2.0.59/go.mod h1:76gmsdIBHvv4O3I0TuFBfO58Rv7YN8FA0ojwYz27ZxE= github.com/openziti/identity v1.0.101 h1:XujPT6eCv3Gqsodh9846EoWfQZwDAFrjPLxm1cipveY= github.com/openziti/identity v1.0.101/go.mod h1:OZLcSf2wGqAa7/8SmbwLQxv0Gsv4i4pxTk83WPHIr+o= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83 h1:XJfBC7Yd6bQCUsGaCXqoOyBNTtdohGepxTveFv7Yn7E= -github.com/openziti/metrics v1.4.1-0.20250428140813-8f25360e7e83/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= +github.com/openziti/metrics v1.4.1 h1:JMnfmW8liTzmqJpd7HDPCSvDDFg1UrE0WoLhW9XdIG8= +github.com/openziti/metrics v1.4.1/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/secretstream v0.1.32 h1:89/ZVcwIQjdVmWDfVRfMEChJJXTLXJ59AYBw5j646M4= github.com/openziti/secretstream v0.1.32/go.mod h1:8YaIbjyMwBeKQ7eOYcoVPKHT10u+4OVPXpnZAeDzC6o= github.com/openziti/transport/v2 v2.0.168 h1:1Anf7X+4xmSKQ12GdPJFhoMZi04QxgD4MJu3agFc1R4= diff --git a/xgress/link_send_buffer.go b/xgress/link_send_buffer.go index 56730c2d..08049cab 100644 --- a/xgress/link_send_buffer.go +++ b/xgress/link_send_buffer.go @@ -245,7 +245,9 @@ func (buffer *LinkSendBuffer) run() { case ack := <-buffer.newlyReceivedAcks: buffer.receiveAcknowledgement(ack) buffer.retransmit() - buffer.checkForCloseOnEmpty() + if buffer.closeWhenEmpty.Load() && len(buffer.buffer) == 0 && !buffer.x.Closed() && buffer.x.IsEndOfCircuitSent() { + go buffer.x.Close() + } case txPayload := <-buffered: buffer.buffer[txPayload.payload.GetSequence()] = txPayload @@ -257,7 +259,6 @@ func (buffer *LinkSendBuffer) run() { case <-retransmitTicker.C: buffer.retransmit() - buffer.checkForCloseOnEmpty() case <-buffer.closeNotify: buffer.close() @@ -266,12 +267,6 @@ func (buffer *LinkSendBuffer) run() { } } -func (buffer *LinkSendBuffer) checkForCloseOnEmpty() { - if buffer.closeWhenEmpty.Load() && len(buffer.buffer) == 0 && !buffer.x.Closed() && buffer.x.IsEndOfCircuitSent() { - go buffer.x.Close() - } -} - func (buffer *LinkSendBuffer) close() { if buffer.blockedByLocalWindow { buffer.metrics().BufferUnblockedByLocalWindow() diff --git a/ziti/ziti.go b/ziti/ziti.go index 52012800..280672f2 100644 --- a/ziti/ziti.go +++ b/ziti/ziti.go @@ -1263,9 +1263,7 @@ func (context *ContextImpl) listenSession(service *rest_model.ServiceDetail, opt edgeListenOptions.MaxTerminators = 1 } - // TODO: uncomment - //edgeListenOptions.SdkFlowControl = options.SdkFlowControl != nil && *options.SdkFlowControl - edgeListenOptions.SdkFlowControl = true + edgeListenOptions.SdkFlowControl = options.SdkFlowControl != nil && *options.SdkFlowControl if listenerMgr, err := newListenerManager(service, context, edgeListenOptions, options.WaitForNEstablishedListeners); err != nil { return nil, err