From 5592bb1b453bf00be2c756487614700820a6c95f Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Wed, 29 Jun 2022 11:34:38 -0700 Subject: [PATCH] feat: expose pending messages and processing rate (#79) * feat: expose pending messages and processing rate Signed-off-by: Derek Wang --- cmd/commands/processor.go | 23 +- .../crds/numaflow.numaproj.io_pipelines.yaml | 8 +- .../crds/numaflow.numaproj.io_vertices.yaml | 8 +- config/install.yaml | 16 +- docs/APIs.md | 62 ++ pkg/apis/numaflow/v1alpha1/generated.pb.go | 839 ++++++++++++------ pkg/apis/numaflow/v1alpha1/generated.proto | 16 +- pkg/apis/numaflow/v1alpha1/vertex_instance.go | 8 + pkg/apis/numaflow/v1alpha1/vertex_types.go | 6 +- .../v1alpha1/zz_generated.deepcopy.go | 26 + pkg/isb/interfaces.go | 13 +- pkg/isb/jetstream/options.go | 13 +- pkg/isb/jetstream/reader.go | 2 +- pkg/isb/jetstream/writer.go | 6 +- pkg/isb/redis/read.go | 1 - pkg/metrics/metrics.go | 134 ++- pkg/metrics/metrics_test.go | 3 +- pkg/sinks/sink.go | 41 +- pkg/sources/generator/tickgen.go | 17 +- pkg/sources/generator/tickgen_test.go | 5 +- pkg/sources/generator/watermark.go | 14 +- pkg/sources/generator/watermark_test.go | 3 +- pkg/sources/http/http.go | 4 - pkg/sources/source.go | 54 +- pkg/sources/types/metadata.go | 9 - pkg/udf/udf.go | 57 +- test/fixtures/expect.go | 2 +- test/fixtures/util.go | 9 +- 28 files changed, 986 insertions(+), 413 deletions(-) create mode 100644 pkg/apis/numaflow/v1alpha1/vertex_instance.go delete mode 100644 pkg/sources/types/metadata.go diff --git a/cmd/commands/processor.go b/cmd/commands/processor.go index d10d10027..57efde689 100644 --- a/cmd/commands/processor.go +++ b/cmd/commands/processor.go @@ -52,30 +52,29 @@ func NewProcessorCommand() *cobra.Command { return fmt.Errorf("invalid replica %q", replicaStr) } log = log.With("vertex", vertex.Name) + vertexInstance := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: hostname, + Replica: int32(replica), + } ctx := logging.WithLogger(signals.SetupSignalHandler(), log) switch processorType { case "source": p := &sources.SourceProcessor{ - ISBSvcType: dfv1.ISBSvcType(isbSvcType), - Vertex: vertex, - Hostname: hostname, - Replica: replica, + ISBSvcType: dfv1.ISBSvcType(isbSvcType), + VertexInstance: vertexInstance, } return p.Start(ctx) case "sink": p := &sinks.SinkProcessor{ - ISBSvcType: dfv1.ISBSvcType(isbSvcType), - Vertex: vertex, - Hostname: hostname, - Replica: replica, + ISBSvcType: dfv1.ISBSvcType(isbSvcType), + VertexInstance: vertexInstance, } return p.Start(ctx) case "udf": p := &udf.UDFProcessor{ - ISBSvcType: dfv1.ISBSvcType(isbSvcType), - Vertex: vertex, - Hostname: hostname, - Replica: replica, + ISBSvcType: dfv1.ISBSvcType(isbSvcType), + VertexInstance: vertexInstance, } return p.Start(ctx) default: diff --git a/config/base/crds/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/numaflow.numaproj.io_pipelines.yaml index e5b90fe0e..d7b75fb36 100644 --- a/config/base/crds/numaflow.numaproj.io_pipelines.yaml +++ b/config/base/crds/numaflow.numaproj.io_pipelines.yaml @@ -1444,8 +1444,14 @@ spec: type: string scale: properties: + lookbackSeconds: + default: 180 + description: Lookback seconds to calculate the average pending + messages and processing rate + format: int32 + type: integer max: - default: 1 + default: 100 description: Maximum replicas format: int32 type: integer diff --git a/config/base/crds/numaflow.numaproj.io_vertices.yaml b/config/base/crds/numaflow.numaproj.io_vertices.yaml index e70a6727f..8d7f27145 100644 --- a/config/base/crds/numaflow.numaproj.io_vertices.yaml +++ b/config/base/crds/numaflow.numaproj.io_vertices.yaml @@ -1316,8 +1316,14 @@ spec: type: integer scale: properties: + lookbackSeconds: + default: 180 + description: Lookback seconds to calculate the average pending + messages and processing rate + format: int32 + type: integer max: - default: 1 + default: 100 description: Maximum replicas format: int32 type: integer diff --git a/config/install.yaml b/config/install.yaml index b45c607c2..391742a15 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -6170,8 +6170,14 @@ spec: type: string scale: properties: + lookbackSeconds: + default: 180 + description: Lookback seconds to calculate the average pending + messages and processing rate + format: int32 + type: integer max: - default: 1 + default: 100 description: Maximum replicas format: int32 type: integer @@ -10057,8 +10063,14 @@ spec: type: integer scale: properties: + lookbackSeconds: + default: 180 + description: Lookback seconds to calculate the average pending + messages and processing rate + format: int32 + type: integer max: - default: 1 + default: 100 description: Maximum replicas format: int32 type: integer diff --git a/docs/APIs.md b/docs/APIs.md index 822d7f993..70d9a2f77 100644 --- a/docs/APIs.md +++ b/docs/APIs.md @@ -2983,6 +2983,18 @@ Maximum replicas

+ + +lookbackSeconds
int32 + + +(Optional) +

+Lookback seconds to calculate the average pending messages and +processing rate +

+ +

@@ -3273,6 +3285,10 @@ Description Vertex

+(Appears on: +VertexInstance) +

+

@@ -3374,6 +3390,52 @@ Refer to the Kubernetes API documentation for the fields of the
+

+VertexInstance +

+

+

+VertexInstance is a wrapper of a vertex instance, which contains the +vertex spec and the instance information such as hostname and replica +index. +

+

+ + + + + + + + + + + + + + + + + + + + + +
+Field + +Description +
+vertex
+ Vertex +		 +
+hostname
string +		 +
+replica
int32 +		 +

VertexLimits

diff --git a/pkg/apis/numaflow/v1alpha1/generated.pb.go b/pkg/apis/numaflow/v1alpha1/generated.pb.go index f815666fd..9c954d295 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.pb.go +++ b/pkg/apis/numaflow/v1alpha1/generated.pb.go @@ -1392,10 +1392,38 @@ func (m *Vertex) XXX_DiscardUnknown() { var xxx_messageInfo_Vertex proto.InternalMessageInfo +func (m *VertexInstance) Reset() { *m = VertexInstance{} } +func (*VertexInstance) ProtoMessage() {} +func (*VertexInstance) Descriptor() ([]byte, []int) { + return fileDescriptor_9d0d1b17d3865563, []int{48} +} +func (m *VertexInstance) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VertexInstance) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *VertexInstance) XXX_Merge(src proto.Message) { + xxx_messageInfo_VertexInstance.Merge(m, src) +} +func (m *VertexInstance) XXX_Size() int { + return m.Size() +} +func (m *VertexInstance) XXX_DiscardUnknown() { + xxx_messageInfo_VertexInstance.DiscardUnknown(m) +} + +var xxx_messageInfo_VertexInstance proto.InternalMessageInfo + func (m *VertexLimits) Reset() { *m = VertexLimits{} } func (*VertexLimits) ProtoMessage() {} func (*VertexLimits) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{48} + return fileDescriptor_9d0d1b17d3865563, []int{49} } func (m *VertexLimits) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1423,7 +1451,7 @@ var xxx_messageInfo_VertexLimits proto.InternalMessageInfo func (m *VertexList) Reset() { *m = VertexList{} } func (*VertexList) ProtoMessage() {} func (*VertexList) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{49} + return fileDescriptor_9d0d1b17d3865563, []int{50} } func (m *VertexList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1451,7 +1479,7 @@ var xxx_messageInfo_VertexList proto.InternalMessageInfo func (m *VertexSpec) Reset() { *m = VertexSpec{} } func (*VertexSpec) ProtoMessage() {} func (*VertexSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{50} + return fileDescriptor_9d0d1b17d3865563, []int{51} } func (m *VertexSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1479,7 +1507,7 @@ var xxx_messageInfo_VertexSpec proto.InternalMessageInfo func (m *VertexStatus) Reset() { *m = VertexStatus{} } func (*VertexStatus) ProtoMessage() {} func (*VertexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{51} + return fileDescriptor_9d0d1b17d3865563, []int{52} } func (m *VertexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1507,7 +1535,7 @@ var xxx_messageInfo_VertexStatus proto.InternalMessageInfo func (m *Watermark) Reset() { *m = Watermark{} } func (*Watermark) ProtoMessage() {} func (*Watermark) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{52} + return fileDescriptor_9d0d1b17d3865563, []int{53} } func (m *Watermark) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1591,6 +1619,7 @@ func init() { proto.RegisterType((*UDF)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.UDF") proto.RegisterType((*UDSink)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.UDSink") proto.RegisterType((*Vertex)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.Vertex") + proto.RegisterType((*VertexInstance)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.VertexInstance") proto.RegisterType((*VertexLimits)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.VertexLimits") proto.RegisterType((*VertexList)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.VertexList") proto.RegisterType((*VertexSpec)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.VertexSpec") @@ -1603,286 +1632,291 @@ func init() { } var fileDescriptor_9d0d1b17d3865563 = []byte{ - // 4453 bytes of a gzipped FileDescriptorProto + // 4529 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5c, 0x4f, 0x6c, 0x1c, 0x59, - 0x5a, 0x4f, 0xf5, 0x1f, 0xbb, 0xfb, 0x6b, 0x3b, 0xb6, 0x5f, 0x32, 0xa1, 0x63, 0x32, 0xee, 0x50, - 0xa3, 0x19, 0x05, 0xd8, 0x6d, 0x33, 0x66, 0x96, 0xcd, 0xc2, 0xce, 0x66, 0xdc, 0x76, 0xec, 0x71, - 0x62, 0x67, 0xcc, 0x6b, 0x3b, 0x61, 0x76, 0x56, 0x84, 0xe7, 0xea, 0xd7, 0xed, 0x9a, 0xae, 0xae, - 0xea, 0xad, 0x7a, 0xe5, 0xc4, 0x23, 0x56, 0x20, 0x71, 0x18, 0x10, 0xa0, 0x5d, 0x89, 0x0b, 0xd2, - 0x0a, 0xc4, 0x61, 0x25, 0xc4, 0x81, 0x0b, 0x02, 0x09, 0x81, 0x56, 0xe2, 0x84, 0x86, 0xdb, 0x1c, - 0x90, 0x18, 0xa4, 0x95, 0xb5, 0x63, 0x24, 0x6e, 0x88, 0x45, 0x73, 0x8b, 0x90, 0x40, 0xef, 0x4f, - 0xfd, 0xed, 0x6a, 0xc7, 0xee, 0xb6, 0xc3, 0x81, 0xdc, 0xba, 0xde, 0xf7, 0xbd, 0xdf, 0xf7, 0xfe, - 0x7e, 0xef, 0xfb, 0xf3, 0x5e, 0xc3, 0x7a, 0xc7, 0x64, 0xfb, 0xfe, 0x5e, 0xdd, 0x70, 0x7a, 0x8b, - 0xb6, 0xdf, 0x23, 0x7d, 0xd7, 0xf9, 0x50, 0xfc, 0x68, 0x5b, 0xce, 0x93, 0xc5, 0x7e, 0xb7, 0xb3, - 0x48, 0xfa, 0xa6, 0x17, 0x95, 0x1c, 0xbc, 0x49, 0xac, 0xfe, 0x3e, 0x79, 0x73, 0xb1, 0x43, 0x6d, - 0xea, 0x12, 0x46, 0x5b, 0xf5, 0xbe, 0xeb, 0x30, 0x07, 0x7d, 0x35, 0x02, 0xaa, 0x07, 0x40, 0xf5, - 0xa0, 0x5a, 0xbd, 0xdf, 0xed, 0xd4, 0x39, 0x50, 0x54, 0x12, 0x00, 0xcd, 0x7f, 0x39, 0xd6, 0x82, - 0x8e, 0xd3, 0x71, 0x16, 0x05, 0xde, 0x9e, 0xdf, 0x16, 0x5f, 0xe2, 0x43, 0xfc, 0x92, 0x72, 0xe6, - 0xf5, 0xee, 0x6d, 0xaf, 0x6e, 0x3a, 0xbc, 0x59, 0x8b, 0x86, 0xe3, 0xd2, 0xc5, 0x83, 0x81, 0xb6, - 0xcc, 0xbf, 0x15, 0xf1, 0xf4, 0x88, 0xb1, 0x6f, 0xda, 0xd4, 0x3d, 0x0c, 0xfa, 0xb2, 0xe8, 0x52, - 0xcf, 0xf1, 0x5d, 0x83, 0x9e, 0xa9, 0x96, 0xb7, 0xd8, 0xa3, 0x8c, 0x64, 0xc9, 0x5a, 0x1c, 0x56, - 0xcb, 0xf5, 0x6d, 0x66, 0xf6, 0x06, 0xc5, 0xfc, 0xd2, 0xf3, 0x2a, 0x78, 0xc6, 0x3e, 0xed, 0x91, - 0x74, 0x3d, 0xfd, 0x3f, 0xa7, 0xe0, 0xf2, 0xf2, 0x9e, 0xc7, 0x5c, 0x62, 0xb0, 0x87, 0xd4, 0x65, - 0xf4, 0x29, 0xba, 0x09, 0x05, 0x9b, 0xf4, 0x68, 0x55, 0xbb, 0xa9, 0xdd, 0x2a, 0x37, 0xa6, 0x3e, - 0x39, 0xaa, 0x5d, 0x3a, 0x3e, 0xaa, 0x15, 0x1e, 0x90, 0x1e, 0xc5, 0x82, 0x82, 0x0c, 0x98, 0x90, - 0xbd, 0xad, 0xe6, 0x6f, 0x6a, 0xb7, 0x2a, 0x4b, 0x77, 0xea, 0x23, 0x4e, 0x53, 0xbd, 0x29, 0x60, - 0x1a, 0x70, 0x7c, 0x54, 0x9b, 0x90, 0xbf, 0xb1, 0x82, 0x46, 0x1f, 0x40, 0xc1, 0x33, 0xed, 0x6e, - 0xb5, 0x20, 0x44, 0xbc, 0x3d, 0xba, 0x08, 0xd3, 0xee, 0x36, 0x4a, 0xbc, 0x07, 0xfc, 0x17, 0x16, - 0xa0, 0xe8, 0xbb, 0x1a, 0xcc, 0x19, 0x8e, 0xcd, 0x08, 0x1f, 0xa8, 0x1d, 0xda, 0xeb, 0x5b, 0x84, - 0xd1, 0x6a, 0x51, 0x88, 0xba, 0x37, 0xb2, 0xa8, 0x95, 0x34, 0x62, 0xe3, 0x95, 0xe3, 0xa3, 0xda, - 0xdc, 0x40, 0x31, 0x1e, 0x94, 0x8d, 0x1e, 0x41, 0xde, 0x6f, 0xb5, 0xab, 0x13, 0xa2, 0x09, 0x5f, - 0x1f, 0xb9, 0x09, 0xbb, 0xab, 0x6b, 0x8d, 0xc9, 0xe3, 0xa3, 0x5a, 0x7e, 0x77, 0x75, 0x0d, 0x73, - 0x44, 0xd4, 0x85, 0x12, 0x5f, 0x65, 0x2d, 0xc2, 0x48, 0x75, 0x52, 0xa0, 0x2f, 0x8f, 0x8c, 0xbe, - 0xa5, 0x80, 0x1a, 0x53, 0xc7, 0x47, 0xb5, 0x52, 0xf0, 0x85, 0x43, 0x01, 0xe8, 0x8f, 0x34, 0x98, - 0xb2, 0x9d, 0x16, 0x6d, 0x52, 0x8b, 0x1a, 0xcc, 0x71, 0xab, 0xa5, 0x9b, 0xf9, 0x5b, 0x95, 0xa5, - 0xf7, 0x47, 0x96, 0x98, 0x5c, 0x9b, 0xf5, 0x07, 0x31, 0xec, 0xbb, 0x36, 0x73, 0x0f, 0x1b, 0x57, - 0xd5, 0xfa, 0x9c, 0x8a, 0x93, 0x70, 0xa2, 0x11, 0x68, 0x17, 0x2a, 0xcc, 0xb1, 0xf8, 0xba, 0x37, - 0x1d, 0xdb, 0xab, 0x96, 0x45, 0x9b, 0x16, 0xea, 0x72, 0xcb, 0x70, 0xc9, 0x75, 0xbe, 0xe7, 0xeb, - 0x07, 0x6f, 0xd6, 0x77, 0x42, 0xb6, 0xc6, 0x15, 0x05, 0x5c, 0x89, 0xca, 0x3c, 0x1c, 0xc7, 0x41, - 0x14, 0x66, 0x3c, 0x6a, 0xf8, 0xae, 0xc9, 0x0e, 0xf9, 0x14, 0xd3, 0xa7, 0xac, 0x0a, 0x62, 0x80, - 0xdf, 0xc8, 0x82, 0xde, 0x76, 0x5a, 0xcd, 0x24, 0x77, 0xe3, 0xca, 0xf1, 0x51, 0x6d, 0x26, 0x55, - 0x88, 0xd3, 0x98, 0xc8, 0x86, 0x59, 0xb3, 0x47, 0x3a, 0x74, 0xdb, 0xb7, 0xac, 0x26, 0x35, 0x5c, - 0xca, 0xbc, 0x6a, 0x45, 0x74, 0xe1, 0x56, 0x96, 0x9c, 0x4d, 0xc7, 0x20, 0xd6, 0x7b, 0x7b, 0x1f, - 0x52, 0x83, 0x61, 0xda, 0xa6, 0x2e, 0xb5, 0x0d, 0xda, 0xa8, 0xaa, 0xce, 0xcc, 0x6e, 0xa4, 0x90, - 0xf0, 0x00, 0x36, 0x5a, 0x87, 0xb9, 0xbe, 0x6b, 0x3a, 0xa2, 0x09, 0x16, 0xf1, 0x3c, 0xbe, 0xf1, - 0xab, 0x53, 0x42, 0x19, 0x5c, 0x57, 0x30, 0x73, 0xdb, 0x69, 0x06, 0x3c, 0x58, 0x07, 0xdd, 0x82, - 0x52, 0x50, 0x58, 0x9d, 0xbe, 0xa9, 0xdd, 0x2a, 0xca, 0x65, 0x13, 0xd4, 0xc5, 0x21, 0x15, 0xad, - 0x41, 0x89, 0xb4, 0xdb, 0xa6, 0xcd, 0x39, 0x2f, 0x8b, 0x21, 0xbc, 0x91, 0xd5, 0xb5, 0x65, 0xc5, - 0x23, 0x71, 0x82, 0x2f, 0x1c, 0xd6, 0x45, 0xf7, 0x00, 0x79, 0xd4, 0x3d, 0x30, 0x0d, 0xba, 0x6c, - 0x18, 0x8e, 0x6f, 0x33, 0xd1, 0xf6, 0x19, 0xd1, 0xf6, 0x79, 0xd5, 0x76, 0xd4, 0x1c, 0xe0, 0xc0, - 0x19, 0xb5, 0xd0, 0x5d, 0x98, 0x3c, 0x70, 0x2c, 0xbf, 0x47, 0xbd, 0xea, 0xac, 0x18, 0xed, 0xf9, - 0xac, 0x26, 0x3d, 0x14, 0x2c, 0x8d, 0x19, 0x05, 0x3e, 0x29, 0xbf, 0x3d, 0x1c, 0xd4, 0x45, 0x26, - 0x4c, 0x58, 0x66, 0xcf, 0x64, 0x5e, 0x75, 0x4e, 0x74, 0xec, 0xee, 0xc8, 0x5b, 0x41, 0x6e, 0x81, - 0x4d, 0x01, 0x26, 0x35, 0xa6, 0xfc, 0x8d, 0x95, 0x00, 0x64, 0x40, 0xd1, 0x33, 0x88, 0x45, 0xab, - 0x48, 0x48, 0xfa, 0xc6, 0xe8, 0x2a, 0x93, 0xa3, 0x34, 0xa6, 0x55, 0x9f, 0x8a, 0xe2, 0x13, 0x4b, - 0xec, 0xf9, 0x3b, 0x30, 0x37, 0xb0, 0x09, 0xd1, 0x2c, 0xe4, 0xbb, 0xf4, 0x50, 0x9e, 0x18, 0x98, - 0xff, 0x44, 0x57, 0xa1, 0x78, 0x40, 0x2c, 0x9f, 0x56, 0x73, 0xa2, 0x4c, 0x7e, 0xfc, 0x72, 0xee, - 0xb6, 0xa6, 0x3f, 0x82, 0xe9, 0x65, 0x9f, 0xed, 0x3b, 0xae, 0xf9, 0x91, 0xd8, 0x47, 0x68, 0x0d, - 0x8a, 0xcc, 0xe9, 0x52, 0x5b, 0x54, 0xaf, 0x2c, 0xbd, 0x9e, 0x35, 0xcc, 0x72, 0x6d, 0xde, 0xa7, - 0x87, 0x81, 0xdc, 0x46, 0x99, 0xb7, 0x6c, 0x87, 0xd7, 0xc3, 0xb2, 0xba, 0xfe, 0x4d, 0x98, 0x68, - 0xf8, 0xed, 0x36, 0x75, 0x4f, 0x71, 0x82, 0xd5, 0xa1, 0xc0, 0x0e, 0xfb, 0xaa, 0x75, 0xe1, 0xd2, - 0x28, 0xec, 0x1c, 0xf6, 0xe9, 0xb3, 0xa3, 0x1a, 0x48, 0x1c, 0xfe, 0x85, 0x05, 0x9f, 0xfe, 0x85, - 0x06, 0x57, 0x64, 0xa1, 0x5a, 0x3d, 0x2b, 0x8e, 0xdd, 0x36, 0x3b, 0x88, 0x42, 0xd1, 0xa5, 0x2d, - 0xd3, 0x53, 0x6d, 0x5f, 0x1d, 0x79, 0xc8, 0x31, 0x47, 0x91, 0xa0, 0xb2, 0x6b, 0xa2, 0x00, 0x4b, - 0x74, 0xe4, 0x43, 0xf9, 0x43, 0xca, 0x3c, 0xe6, 0x52, 0xd2, 0x13, 0x6d, 0xae, 0x2c, 0xbd, 0x3b, - 0xb2, 0xa8, 0x7b, 0x94, 0x35, 0x05, 0x92, 0x12, 0x37, 0x7d, 0x7c, 0x54, 0x2b, 0x87, 0x85, 0x38, - 0x92, 0xa4, 0xff, 0x7b, 0x0e, 0xca, 0xe1, 0xe1, 0x85, 0x5e, 0x83, 0xa2, 0xd0, 0x15, 0x6a, 0x58, - 0xc3, 0xe5, 0x21, 0x54, 0x0a, 0x96, 0x34, 0xf4, 0x3a, 0x4c, 0x1a, 0x4e, 0xaf, 0x47, 0xec, 0x56, - 0x35, 0x77, 0x33, 0x7f, 0xab, 0xdc, 0xa8, 0xf0, 0x5d, 0xb1, 0x22, 0x8b, 0x70, 0x40, 0x43, 0x37, - 0xa0, 0x40, 0xdc, 0x8e, 0x57, 0xcd, 0x0b, 0x1e, 0x71, 0x3a, 0x2f, 0xbb, 0x1d, 0x0f, 0x8b, 0x52, - 0xf4, 0x35, 0xc8, 0x53, 0xfb, 0xa0, 0x5a, 0x18, 0xbe, 0xed, 0xee, 0xda, 0x07, 0x0f, 0x89, 0xdb, - 0xa8, 0xa8, 0x36, 0xe4, 0xef, 0xda, 0x07, 0x98, 0xd7, 0x41, 0xef, 0xc3, 0x94, 0xdc, 0x79, 0x5b, - 0x7c, 0x23, 0x7b, 0xd5, 0xa2, 0xc0, 0xa8, 0x0d, 0xdf, 0xba, 0x82, 0x2f, 0x3a, 0x45, 0x62, 0x85, - 0x1e, 0x4e, 0x40, 0xa1, 0xf7, 0xa1, 0x1c, 0x58, 0x79, 0x9e, 0x3a, 0xa7, 0x33, 0x15, 0x30, 0x56, - 0x4c, 0x98, 0x7e, 0xdb, 0x37, 0x5d, 0xda, 0xa3, 0x36, 0xf3, 0x1a, 0x73, 0x4a, 0x40, 0x39, 0xa0, - 0x7a, 0x38, 0x42, 0xd3, 0xff, 0x2b, 0x07, 0x83, 0x56, 0x42, 0x52, 0xa0, 0x76, 0x9e, 0x02, 0xd1, - 0x1e, 0xcc, 0x84, 0x7a, 0x7f, 0xdb, 0xb1, 0x4c, 0xe3, 0x50, 0x6d, 0x85, 0xdb, 0xaa, 0xda, 0xcc, - 0x46, 0x92, 0xfc, 0xec, 0xa8, 0xf6, 0xea, 0xa0, 0x8d, 0x5c, 0x8f, 0x18, 0x70, 0x1a, 0x90, 0xcb, - 0x48, 0x1f, 0x8f, 0xd2, 0x5c, 0x7c, 0x6d, 0xc8, 0x0e, 0x1f, 0xe1, 0x6c, 0x1c, 0x7d, 0xa5, 0xe8, - 0x7f, 0x91, 0x83, 0xc2, 0xdd, 0x56, 0x87, 0x72, 0x6d, 0xd1, 0x76, 0x9d, 0x5e, 0x5a, 0x5b, 0xac, - 0xb9, 0x4e, 0x0f, 0x0b, 0x0a, 0x9a, 0x87, 0x1c, 0x73, 0xd4, 0x00, 0x81, 0xa2, 0xe7, 0x76, 0x1c, - 0x9c, 0x63, 0x0e, 0xfa, 0x08, 0xc0, 0x70, 0xec, 0x96, 0x29, 0x4d, 0x8b, 0xfc, 0x98, 0x16, 0xe4, - 0x9a, 0xe3, 0x3e, 0x21, 0x6e, 0x6b, 0x25, 0x44, 0x6c, 0x5c, 0x3e, 0x3e, 0xaa, 0x41, 0xf4, 0x8d, - 0x63, 0xd2, 0x50, 0x27, 0x3c, 0x5b, 0xa4, 0x91, 0xbc, 0x32, 0xb2, 0x5c, 0x3e, 0x10, 0xc3, 0x4f, - 0x16, 0xfd, 0x0f, 0x35, 0x80, 0x88, 0x05, 0xbd, 0x0d, 0x33, 0x7b, 0x42, 0x19, 0x6e, 0x91, 0xa7, - 0x9b, 0xd4, 0xee, 0xb0, 0x7d, 0x31, 0x78, 0x05, 0x39, 0x69, 0x8d, 0x24, 0x09, 0xa7, 0x79, 0xd1, - 0x3b, 0x30, 0x2b, 0x8b, 0x76, 0x3d, 0xa2, 0x30, 0xc5, 0xe0, 0x4e, 0x37, 0xae, 0x72, 0x13, 0xa5, - 0x91, 0xa2, 0xe1, 0x01, 0x6e, 0xfd, 0x2d, 0x98, 0x1b, 0x18, 0x29, 0x54, 0x83, 0x62, 0x97, 0x1e, - 0x6e, 0xf0, 0x73, 0x84, 0x2b, 0x15, 0xa1, 0x45, 0xef, 0xf3, 0x02, 0x2c, 0xcb, 0xf5, 0xff, 0xd6, - 0xa0, 0xb4, 0xe6, 0xdb, 0x86, 0x38, 0x75, 0x9e, 0x7f, 0x46, 0x04, 0x3a, 0x2a, 0x97, 0xa9, 0xa3, - 0x7c, 0x98, 0xe8, 0x3e, 0x09, 0x75, 0x58, 0x65, 0x69, 0x6b, 0xf4, 0x39, 0x57, 0x4d, 0xaa, 0xdf, - 0x17, 0x78, 0xd2, 0xac, 0xbd, 0xac, 0x1a, 0x34, 0x71, 0xff, 0x91, 0x10, 0xaa, 0x84, 0xcd, 0x7f, - 0x0d, 0x2a, 0x31, 0xb6, 0x33, 0x1d, 0xbc, 0x7f, 0xa9, 0xc1, 0xcc, 0xba, 0x74, 0xff, 0x1c, 0x57, - 0x3a, 0x5b, 0xe8, 0x3a, 0xe4, 0xdd, 0xbe, 0x2f, 0xea, 0xe7, 0xa5, 0xdf, 0x80, 0xb7, 0x77, 0x31, - 0x2f, 0x43, 0xbf, 0x06, 0xa5, 0x96, 0x2f, 0x4d, 0x5d, 0x75, 0xe4, 0xd4, 0x63, 0xfb, 0x2b, 0x74, - 0x32, 0xa3, 0x9e, 0x71, 0x27, 0x80, 0xef, 0xb8, 0x55, 0x55, 0x4b, 0x5a, 0x69, 0xc1, 0x17, 0x0e, - 0xd1, 0xf8, 0x19, 0xd1, 0xf3, 0x3a, 0x4d, 0xf3, 0x23, 0xe9, 0x3f, 0x16, 0xe5, 0x19, 0xb1, 0x25, - 0x8b, 0x70, 0x40, 0xd3, 0xbf, 0x9b, 0x83, 0x6b, 0xeb, 0x94, 0xad, 0x12, 0xda, 0x73, 0xec, 0x55, - 0xda, 0xb7, 0x9c, 0x43, 0xae, 0xda, 0x30, 0xfd, 0x36, 0x7a, 0x07, 0xc0, 0xf4, 0xf6, 0x9a, 0x07, - 0x06, 0x3f, 0xa2, 0xd5, 0x14, 0xde, 0x54, 0x23, 0x06, 0x1b, 0xcd, 0x86, 0xa2, 0x3c, 0x4b, 0x7c, - 0xe1, 0x58, 0x9d, 0xe8, 0x30, 0xcb, 0x9d, 0x70, 0x98, 0x35, 0x01, 0xfa, 0x91, 0x82, 0xcc, 0x0b, - 0xce, 0x5f, 0x0c, 0xc4, 0x9c, 0x45, 0x37, 0xc6, 0x60, 0xc6, 0x51, 0x59, 0x7f, 0x97, 0x87, 0xf9, - 0x75, 0xca, 0xc2, 0xb3, 0x5a, 0xd9, 0x22, 0xcd, 0x3e, 0x35, 0xf8, 0xa8, 0x7c, 0xac, 0xc1, 0x84, - 0x45, 0xf6, 0xa8, 0xe5, 0x89, 0x2d, 0x50, 0x59, 0x7a, 0x3c, 0xf2, 0x9a, 0x1c, 0x2e, 0xa5, 0xbe, - 0x29, 0x24, 0xa4, 0x56, 0xa9, 0x2c, 0xc4, 0x4a, 0x3c, 0xfa, 0x0a, 0x54, 0x0c, 0xcb, 0xf7, 0x18, - 0x75, 0xb7, 0x1d, 0x57, 0x6e, 0xee, 0x62, 0xe4, 0x50, 0xad, 0x44, 0x24, 0x1c, 0xe7, 0x43, 0x4b, - 0x00, 0x86, 0x65, 0x52, 0x9b, 0x89, 0x5a, 0x72, 0x6d, 0xa0, 0x60, 0xbc, 0x57, 0x42, 0x0a, 0x8e, - 0x71, 0x71, 0x51, 0x3d, 0xc7, 0x36, 0x99, 0x23, 0x45, 0x15, 0x92, 0xa2, 0xb6, 0x22, 0x12, 0x8e, - 0xf3, 0x89, 0x6a, 0x94, 0xb9, 0xa6, 0xe1, 0x89, 0x6a, 0xc5, 0x54, 0xb5, 0x88, 0x84, 0xe3, 0x7c, - 0x7c, 0xfb, 0xc5, 0xfa, 0x7f, 0xa6, 0xed, 0xf7, 0xf7, 0x25, 0x58, 0x48, 0x0c, 0x2b, 0x23, 0x8c, - 0xb6, 0x7d, 0xab, 0x49, 0x59, 0x30, 0x81, 0x5f, 0x81, 0x8a, 0x72, 0x44, 0x1e, 0x44, 0xaa, 0x29, - 0x6c, 0x54, 0x33, 0x22, 0xe1, 0x38, 0x1f, 0xfa, 0xfd, 0x68, 0xde, 0x73, 0x62, 0xde, 0x8d, 0xf3, - 0x99, 0xf7, 0x81, 0x06, 0x9e, 0x6a, 0xee, 0x17, 0xa1, 0x6c, 0x13, 0xe6, 0x89, 0x8d, 0xa4, 0xf6, - 0x4c, 0x68, 0x8b, 0x3c, 0x08, 0x08, 0x38, 0xe2, 0x41, 0xdb, 0x70, 0x55, 0x0d, 0xf1, 0xdd, 0xa7, - 0x7d, 0xc7, 0x65, 0xd4, 0x95, 0x75, 0x0b, 0xa2, 0xee, 0x0d, 0x55, 0xf7, 0xea, 0x56, 0x06, 0x0f, - 0xce, 0xac, 0x89, 0xb6, 0xe0, 0x8a, 0x21, 0x6c, 0x5b, 0x4c, 0x2d, 0x87, 0xb4, 0x02, 0xc0, 0xa2, - 0x00, 0xfc, 0x69, 0x05, 0x78, 0x65, 0x65, 0x90, 0x05, 0x67, 0xd5, 0x4b, 0xaf, 0xe6, 0x89, 0x91, - 0x56, 0xf3, 0xe4, 0x28, 0xab, 0xb9, 0x34, 0xda, 0x6a, 0x2e, 0x9f, 0x6e, 0x35, 0xf3, 0x91, 0xe7, - 0xeb, 0x88, 0xba, 0xdc, 0x21, 0x93, 0x2e, 0x96, 0x58, 0x78, 0x90, 0x1c, 0xf9, 0x66, 0x06, 0x0f, - 0xce, 0xac, 0x89, 0xf6, 0x60, 0x5e, 0x96, 0xdf, 0xb5, 0x0d, 0xf7, 0xb0, 0xcf, 0xd5, 0x7d, 0x0c, - 0xb7, 0x22, 0x70, 0x75, 0x85, 0x3b, 0xdf, 0x1c, 0xca, 0x89, 0x4f, 0x40, 0x41, 0xbf, 0x02, 0xd3, - 0x72, 0x96, 0xb6, 0x48, 0x3f, 0x16, 0x9b, 0x78, 0x45, 0xc1, 0x4e, 0xaf, 0xc4, 0x89, 0x38, 0xc9, - 0x8b, 0x96, 0x61, 0xa6, 0x7f, 0x60, 0xf0, 0x9f, 0x1b, 0xed, 0x07, 0x94, 0xb6, 0x68, 0x4b, 0x84, - 0x26, 0xca, 0x8d, 0x9f, 0x0a, 0x0c, 0xdf, 0xed, 0x24, 0x19, 0xa7, 0xf9, 0xd1, 0x6d, 0x98, 0xf2, - 0x18, 0x71, 0x99, 0x72, 0x6a, 0x44, 0xc0, 0xa2, 0x1c, 0x79, 0x10, 0xcd, 0x18, 0x0d, 0x27, 0x38, - 0xc7, 0xd1, 0x1e, 0xcf, 0xe4, 0x61, 0x28, 0xbc, 0xc2, 0x94, 0xda, 0xff, 0x9d, 0xb4, 0xda, 0xff, - 0x60, 0x9c, 0xed, 0x9f, 0x21, 0xe1, 0x54, 0xdb, 0xfe, 0x1e, 0x20, 0x57, 0xf9, 0xb0, 0xd2, 0x8d, - 0x89, 0x69, 0xfe, 0x30, 0xf4, 0x82, 0x07, 0x38, 0x70, 0x46, 0x2d, 0xd4, 0x84, 0x57, 0x3c, 0x6a, - 0x33, 0xd3, 0xa6, 0x56, 0x12, 0x4e, 0x1e, 0x09, 0xaf, 0x2a, 0xb8, 0x57, 0x9a, 0x59, 0x4c, 0x38, - 0xbb, 0xee, 0x38, 0x83, 0xff, 0xa3, 0xb2, 0x38, 0x77, 0xe5, 0xd0, 0x9c, 0x9b, 0xda, 0xfe, 0x38, - 0xad, 0xb6, 0x1f, 0x8f, 0x3f, 0x6f, 0xa3, 0xa9, 0xec, 0x25, 0x00, 0x31, 0x0b, 0x71, 0x9d, 0x1d, - 0x6a, 0x2a, 0x1c, 0x52, 0x70, 0x8c, 0x8b, 0xef, 0xc2, 0x60, 0x9c, 0xe3, 0xea, 0x3a, 0xdc, 0x85, - 0xcd, 0x38, 0x11, 0x27, 0x79, 0x87, 0xaa, 0xfc, 0xe2, 0xc8, 0x2a, 0xff, 0x1e, 0x20, 0xd3, 0x36, - 0x59, 0x38, 0xe5, 0x12, 0x6f, 0x22, 0x19, 0xf9, 0xdb, 0x18, 0xe0, 0xc0, 0x19, 0xb5, 0x86, 0x2c, - 0xe5, 0xc9, 0xf3, 0x5d, 0xca, 0xa5, 0xd1, 0x97, 0x32, 0x7a, 0x0c, 0xd7, 0x85, 0x28, 0x35, 0x3e, - 0x49, 0x60, 0xa9, 0xfc, 0x7f, 0x46, 0x01, 0x5f, 0xc7, 0xc3, 0x18, 0xf1, 0x70, 0x0c, 0x3e, 0x3f, - 0x86, 0x4b, 0x5b, 0x5c, 0x38, 0xb1, 0x86, 0x1f, 0x0c, 0x2b, 0x19, 0x3c, 0x38, 0xb3, 0x26, 0x5f, - 0x62, 0x8c, 0x2f, 0x43, 0xb2, 0x67, 0xd1, 0x96, 0x38, 0x08, 0x4a, 0xd1, 0x12, 0xdb, 0xd9, 0x6c, - 0x2a, 0x0a, 0x8e, 0x71, 0x65, 0xe9, 0xea, 0xa9, 0x33, 0xea, 0xea, 0x75, 0x91, 0xe6, 0x69, 0x27, - 0x8e, 0x04, 0xa5, 0xf0, 0xc3, 0x58, 0xf6, 0x4a, 0x9a, 0x01, 0x0f, 0xd6, 0x11, 0x47, 0xa5, 0xe1, - 0x9a, 0x7d, 0xe6, 0x25, 0xb1, 0x2e, 0xa7, 0x8e, 0xca, 0x0c, 0x1e, 0x9c, 0x59, 0x93, 0x1b, 0x29, - 0xfb, 0x94, 0x58, 0x6c, 0x3f, 0x09, 0x38, 0x93, 0x34, 0x52, 0xde, 0x1d, 0x64, 0xc1, 0x59, 0xf5, - 0xc6, 0x51, 0x6f, 0x7f, 0x90, 0x83, 0x2b, 0xeb, 0x54, 0xa5, 0x58, 0xb6, 0x9d, 0x56, 0xa0, 0xd7, - 0xfe, 0x9f, 0x7a, 0x59, 0x7f, 0xa2, 0x01, 0xbc, 0xbb, 0xb3, 0xb3, 0xad, 0x5c, 0xe4, 0x16, 0x14, - 0x88, 0xaf, 0x22, 0x1c, 0x95, 0xa5, 0xb5, 0xd1, 0x33, 0x59, 0xf1, 0xa0, 0xb7, 0x0a, 0x27, 0xf8, - 0x6c, 0x1f, 0x0b, 0x74, 0xf4, 0xb3, 0x30, 0xa9, 0xce, 0x06, 0x31, 0x56, 0xa5, 0x28, 0xa3, 0xa0, - 0xce, 0x0f, 0x1c, 0xd0, 0xf5, 0x9f, 0xe4, 0xe0, 0xda, 0x86, 0xcd, 0xa8, 0xdb, 0x64, 0xb4, 0x9f, - 0x08, 0x4a, 0xa3, 0xdf, 0x88, 0xe5, 0xfa, 0x64, 0x7b, 0x7f, 0xe1, 0x74, 0x3e, 0xbb, 0xcc, 0x17, - 0x6d, 0x51, 0x46, 0xa2, 0x5d, 0x19, 0x95, 0xc5, 0x12, 0x7c, 0x3e, 0x14, 0xbc, 0x3e, 0x35, 0x54, - 0x44, 0xa0, 0x39, 0xf2, 0x68, 0x64, 0x77, 0x80, 0xaf, 0xbc, 0x28, 0x16, 0x23, 0xd6, 0xa1, 0x10, - 0x87, 0xbe, 0x03, 0x13, 0x1e, 0x23, 0xcc, 0x0f, 0x22, 0x6c, 0xbb, 0xe7, 0x2d, 0x58, 0x80, 0x47, - 0x07, 0xa4, 0xfc, 0xc6, 0x4a, 0xa8, 0xfe, 0x13, 0x0d, 0xe6, 0xb3, 0x2b, 0x6e, 0x9a, 0x1e, 0x43, - 0xdf, 0x1a, 0x18, 0xf6, 0x53, 0x86, 0x4a, 0x78, 0x6d, 0x31, 0xe8, 0xb3, 0x4a, 0x70, 0x29, 0x28, - 0x89, 0x0d, 0x39, 0x83, 0xa2, 0xc9, 0x68, 0x2f, 0xb0, 0x12, 0xde, 0x3b, 0xe7, 0xae, 0xc7, 0x76, - 0x25, 0x97, 0x82, 0xa5, 0x30, 0xfd, 0xe3, 0xdc, 0xb0, 0x2e, 0xf3, 0x69, 0x41, 0xdd, 0x64, 0xe2, - 0xe3, 0xde, 0x78, 0x89, 0x8f, 0x86, 0x1f, 0x6b, 0xcf, 0x60, 0xfa, 0xe3, 0x37, 0x07, 0xd3, 0x1f, - 0xef, 0x8d, 0x9f, 0xfe, 0x48, 0x8d, 0xc2, 0xd0, 0x2c, 0xc8, 0x8f, 0x72, 0x70, 0xe3, 0xa4, 0x55, - 0x83, 0x3a, 0xe1, 0xe2, 0xd4, 0xc6, 0xbd, 0x0e, 0x71, 0xe2, 0x32, 0x44, 0x4b, 0x50, 0xec, 0xef, - 0x13, 0x2f, 0x50, 0xa7, 0xc1, 0xa9, 0x53, 0xdc, 0xe6, 0x85, 0xcf, 0x8e, 0x6a, 0x15, 0xa9, 0x86, - 0xc5, 0x27, 0x96, 0xac, 0x5c, 0xb1, 0xf4, 0xa8, 0xe7, 0x45, 0x86, 0x5d, 0xa8, 0x58, 0xb6, 0x64, - 0x31, 0x0e, 0xe8, 0x88, 0xc1, 0x84, 0x74, 0x96, 0x54, 0x38, 0x79, 0x73, 0xe4, 0x7e, 0x64, 0xa4, - 0xca, 0xa2, 0x4e, 0x29, 0xbf, 0x5b, 0xc9, 0xd2, 0xff, 0xea, 0x32, 0x5c, 0xcb, 0x9e, 0x13, 0xde, - 0xf6, 0x03, 0xea, 0x7a, 0xa6, 0x63, 0xab, 0xd3, 0x27, 0x4a, 0xb3, 0xca, 0x62, 0x1c, 0xd0, 0xd1, - 0x2d, 0x28, 0xb9, 0xb4, 0x6f, 0x99, 0x06, 0xf1, 0x94, 0xd3, 0x21, 0xa2, 0x8f, 0x58, 0x95, 0xe1, - 0x90, 0x3a, 0xe4, 0xea, 0x47, 0xfe, 0xff, 0xf0, 0xea, 0xc7, 0x9f, 0x6b, 0xdc, 0x9e, 0x93, 0x11, - 0x87, 0x81, 0x0a, 0x6a, 0x2e, 0xce, 0xb3, 0x65, 0xaf, 0x4a, 0xbb, 0x70, 0x88, 0x40, 0x3c, 0xbc, - 0x2d, 0xe8, 0x07, 0x1a, 0x54, 0x7b, 0x29, 0x83, 0xf1, 0x02, 0x6f, 0xcf, 0xdc, 0x38, 0x3e, 0xaa, - 0x55, 0xb7, 0x86, 0xc8, 0xc3, 0x43, 0x5b, 0x82, 0x7e, 0x0b, 0x2a, 0x7d, 0xbe, 0x2e, 0x3c, 0x46, - 0x6d, 0x83, 0xaa, 0x5c, 0xdd, 0xe8, 0xab, 0x79, 0x3b, 0xc2, 0x6a, 0x32, 0x97, 0x30, 0xda, 0x39, - 0x6c, 0xcc, 0x70, 0xd7, 0x2e, 0x46, 0xc0, 0x71, 0x89, 0x89, 0x3b, 0x37, 0x5b, 0x17, 0x7d, 0xe7, - 0xe6, 0xfb, 0xd9, 0x77, 0x6e, 0xc8, 0x39, 0x6b, 0xc8, 0x97, 0x77, 0x6f, 0x5e, 0xde, 0xbd, 0x79, - 0x51, 0x77, 0x6f, 0x6e, 0x41, 0xc9, 0xa3, 0x8c, 0x99, 0x76, 0xc7, 0xab, 0xce, 0xca, 0x04, 0x1d, - 0x97, 0xda, 0x54, 0x65, 0x38, 0xa4, 0xa2, 0x9f, 0x87, 0xb2, 0x08, 0xb1, 0x2d, 0xbb, 0x1d, 0xaf, - 0x3a, 0x27, 0x32, 0x75, 0xe2, 0x24, 0x6f, 0x06, 0x85, 0x38, 0xa2, 0xa3, 0xb7, 0x60, 0x4a, 0xa6, - 0x12, 0xe5, 0x11, 0x24, 0xee, 0xc9, 0x94, 0x1b, 0xb3, 0x7c, 0x05, 0x37, 0x62, 0xe5, 0x38, 0xc1, - 0xc5, 0x5d, 0x57, 0x1a, 0xc6, 0x21, 0xab, 0x57, 0x92, 0xae, 0x6b, 0x14, 0xa1, 0xc4, 0x31, 0x2e, - 0xf4, 0x2a, 0xe4, 0x99, 0xe5, 0x55, 0xaf, 0x0a, 0xe6, 0xd0, 0xc5, 0xd8, 0xd9, 0x6c, 0x62, 0x5e, - 0x3e, 0xfe, 0x25, 0x9a, 0xff, 0xd1, 0x60, 0x26, 0x75, 0x8f, 0x83, 0xcb, 0xf4, 0x5d, 0x4b, 0x9d, - 0x94, 0xa1, 0xcc, 0x5d, 0xbc, 0x89, 0x79, 0x39, 0x7a, 0xac, 0xfc, 0x98, 0xdc, 0x98, 0xfa, 0xe8, - 0xc1, 0xf2, 0x4e, 0x93, 0x3b, 0x2e, 0x03, 0x2e, 0xcc, 0xed, 0xd4, 0xe8, 0xe6, 0x93, 0x71, 0xd1, - 0x93, 0x47, 0x38, 0x16, 0x1c, 0x28, 0x9c, 0x26, 0x38, 0xa0, 0x7f, 0xa6, 0x41, 0xf9, 0x3e, 0x69, - 0x77, 0x49, 0xd3, 0xb4, 0xbb, 0xe8, 0x75, 0x98, 0xdc, 0x73, 0x9d, 0x2e, 0x75, 0x3d, 0x95, 0xfd, - 0x15, 0x29, 0xc5, 0x86, 0x2c, 0xc2, 0x01, 0x8d, 0xfb, 0xa3, 0xcc, 0xe9, 0x9b, 0x46, 0xda, 0x1f, - 0xdd, 0xe1, 0x85, 0x58, 0xd2, 0xd0, 0x23, 0x39, 0x77, 0xf9, 0x31, 0x6f, 0x62, 0xee, 0x6c, 0x36, - 0x65, 0x46, 0x35, 0x98, 0x75, 0xf4, 0x46, 0xc2, 0xbe, 0x2a, 0x0f, 0xb5, 0x88, 0xbe, 0x9f, 0x83, - 0x8a, 0xec, 0x9a, 0xf4, 0x40, 0xcf, 0xb3, 0x73, 0x77, 0x44, 0xf0, 0xdc, 0xf3, 0x7b, 0xd4, 0x5d, - 0x77, 0x1d, 0xbf, 0xaf, 0x66, 0x29, 0x1e, 0x0c, 0x89, 0x88, 0x61, 0x00, 0x3d, 0x2a, 0x0a, 0x46, - 0xa7, 0x70, 0x81, 0xa3, 0x53, 0x3c, 0x71, 0x74, 0xfe, 0x5a, 0x83, 0xf2, 0xa6, 0xd9, 0xa6, 0xc6, - 0xa1, 0x61, 0x51, 0xf4, 0x2d, 0xa8, 0xb6, 0xa8, 0x45, 0x19, 0x5d, 0x77, 0x89, 0x41, 0xb7, 0xa9, - 0x6b, 0x0a, 0x55, 0xef, 0xd8, 0x2d, 0x69, 0x8d, 0x17, 0xc3, 0x88, 0x45, 0x75, 0x75, 0x08, 0x1f, - 0x1e, 0x8a, 0x80, 0x36, 0x60, 0xaa, 0x45, 0x3d, 0xd3, 0xa5, 0xad, 0xed, 0x98, 0xdd, 0xfd, 0x7a, - 0xb0, 0xa4, 0x57, 0x63, 0xb4, 0x67, 0x47, 0xb5, 0xe9, 0x6d, 0xb3, 0x4f, 0x2d, 0xd3, 0xa6, 0xd2, - 0x00, 0x4f, 0x54, 0xd5, 0x8b, 0x90, 0xdf, 0x74, 0x3a, 0xfa, 0xef, 0xe6, 0x21, 0x3c, 0xc3, 0xd1, - 0xef, 0x69, 0x50, 0x21, 0xb6, 0xed, 0x30, 0x75, 0x38, 0xca, 0xf0, 0x3d, 0x1e, 0xdb, 0x54, 0xa8, - 0x2f, 0x47, 0xa0, 0xf2, 0xa4, 0x0e, 0x0f, 0xd4, 0x18, 0x05, 0xc7, 0x65, 0x23, 0x3f, 0x15, 0x8c, - 0xde, 0x1a, 0xbf, 0x15, 0xa7, 0x08, 0x3d, 0xcf, 0x7f, 0x03, 0x66, 0xd3, 0x8d, 0x3d, 0x8b, 0x22, - 0x1c, 0x27, 0xec, 0xf5, 0x67, 0x1a, 0x94, 0x02, 0x65, 0x86, 0x56, 0xa0, 0xe0, 0x7b, 0xd4, 0x3d, - 0xdb, 0x1d, 0x44, 0xa1, 0x01, 0x77, 0x3d, 0xea, 0x62, 0x51, 0x19, 0xbd, 0x07, 0xa5, 0x3e, 0xf1, - 0xbc, 0x27, 0x8e, 0xdb, 0x52, 0x6a, 0xf6, 0x94, 0x40, 0xf2, 0x6c, 0x56, 0x55, 0x71, 0x08, 0xa2, - 0xff, 0x70, 0x1a, 0x2a, 0x0f, 0x08, 0x33, 0x0f, 0xa8, 0xf0, 0x87, 0x2f, 0xc6, 0x21, 0xfa, 0x53, - 0x0d, 0xae, 0x25, 0x23, 0xd7, 0x17, 0xe8, 0x15, 0xcd, 0x1f, 0x1f, 0xd5, 0xae, 0xe1, 0x4c, 0x69, - 0x78, 0x48, 0x2b, 0x84, 0x7f, 0x34, 0x10, 0x08, 0xbf, 0x68, 0xff, 0xa8, 0x39, 0x4c, 0x20, 0x1e, - 0xde, 0x96, 0x97, 0xfe, 0xd1, 0x08, 0xfe, 0xd1, 0x85, 0xbf, 0x49, 0xf8, 0x5e, 0xb6, 0x7f, 0xf4, - 0x70, 0x74, 0x0b, 0x28, 0xda, 0x91, 0x2f, 0x9d, 0xa2, 0x97, 0x4e, 0xd1, 0x8b, 0x72, 0x8a, 0xfa, - 0x29, 0xa7, 0x68, 0x9c, 0x64, 0x84, 0xca, 0xf2, 0x4b, 0xb4, 0x61, 0xce, 0xd5, 0xf8, 0x6e, 0xca, - 0x1f, 0xe7, 0xe0, 0x4a, 0x86, 0x76, 0x40, 0xef, 0xc0, 0xac, 0xc7, 0x1c, 0x97, 0x74, 0x68, 0x34, - 0xa1, 0xf2, 0x40, 0x13, 0x37, 0x40, 0x9b, 0x29, 0x1a, 0x1e, 0xe0, 0x46, 0x8f, 0x01, 0x88, 0x61, - 0x50, 0xcf, 0xdb, 0x72, 0x5a, 0x81, 0x5d, 0x76, 0x87, 0xbb, 0x0b, 0xcb, 0x61, 0xe9, 0xb3, 0xa3, - 0xda, 0x97, 0xb3, 0x12, 0x46, 0x41, 0x7b, 0x98, 0xbc, 0xeb, 0x1d, 0x55, 0xc0, 0x31, 0x48, 0xf4, - 0xeb, 0x00, 0xf2, 0xf6, 0x77, 0x78, 0x4f, 0xf1, 0x39, 0x51, 0xfd, 0x7a, 0x70, 0xbb, 0xba, 0xfe, - 0xab, 0x3e, 0xb1, 0x19, 0x5f, 0x15, 0xe2, 0xee, 0xee, 0xc3, 0x10, 0x05, 0xc7, 0x10, 0xf5, 0x7f, - 0xcc, 0x41, 0x29, 0xb0, 0x17, 0x5f, 0x40, 0xde, 0xa6, 0x93, 0xc8, 0xdb, 0x8c, 0xfe, 0x08, 0x25, - 0x68, 0xf2, 0xd0, 0x4c, 0x8d, 0x93, 0xca, 0xd4, 0xac, 0x8f, 0x2f, 0xea, 0xe4, 0xdc, 0xcc, 0x33, - 0x0d, 0x2e, 0x07, 0xac, 0xea, 0x7e, 0xf2, 0x57, 0x61, 0xda, 0xa5, 0xa4, 0xd5, 0x20, 0xcc, 0xd8, - 0x17, 0xd3, 0x27, 0x6f, 0x27, 0xcf, 0x1d, 0x1f, 0xd5, 0xa6, 0x71, 0x9c, 0x80, 0x93, 0x7c, 0xa8, - 0x0e, 0xe0, 0xb7, 0xda, 0x8f, 0x1c, 0x57, 0x38, 0x5b, 0xf2, 0x4e, 0xb2, 0x98, 0xc4, 0xdd, 0xd5, - 0x35, 0x55, 0x8a, 0x63, 0x1c, 0x59, 0x17, 0xa1, 0xf3, 0x63, 0x5e, 0x84, 0x2e, 0x9c, 0xe9, 0x22, - 0xf4, 0x3f, 0x6b, 0x30, 0x15, 0x75, 0xfe, 0xc2, 0x53, 0x51, 0xed, 0x64, 0x2a, 0x6a, 0x79, 0xec, - 0xb9, 0x1d, 0x92, 0x7c, 0xfa, 0x41, 0x31, 0xea, 0x96, 0x48, 0x37, 0xed, 0xc1, 0xbc, 0x99, 0x99, - 0x82, 0x89, 0xa9, 0x8e, 0xf0, 0x5e, 0xd9, 0xc6, 0x50, 0x4e, 0x7c, 0x02, 0x0a, 0xf2, 0xa1, 0x74, - 0x40, 0x5d, 0x66, 0x1a, 0x34, 0xe8, 0xdf, 0xfa, 0x39, 0x3d, 0x5b, 0x8c, 0xc6, 0xf4, 0xa1, 0x12, - 0x80, 0x43, 0x51, 0x68, 0x0f, 0x8a, 0xb4, 0xd5, 0xa1, 0xc1, 0x3d, 0xf2, 0xb7, 0xc7, 0xba, 0xc3, - 0x1f, 0x8d, 0x27, 0xff, 0xf2, 0xb0, 0x84, 0x46, 0x1e, 0x94, 0xad, 0xc0, 0x65, 0x56, 0x06, 0x73, - 0x63, 0x64, 0x39, 0xa1, 0xf3, 0x1d, 0xdd, 0xeb, 0x0c, 0x8b, 0x70, 0x24, 0x07, 0x75, 0xc3, 0xd7, - 0x09, 0xc5, 0x73, 0xd2, 0x04, 0x27, 0xbc, 0x7d, 0xf3, 0xa0, 0xfc, 0x84, 0x30, 0xea, 0xf6, 0x88, - 0xdb, 0x55, 0x06, 0xed, 0xe8, 0x3d, 0x7c, 0x14, 0x20, 0x45, 0x3d, 0x0c, 0x8b, 0x70, 0x24, 0x47, - 0xff, 0x61, 0x2e, 0x52, 0x3d, 0x2f, 0x3a, 0x17, 0xf8, 0x56, 0x32, 0x17, 0xb8, 0x90, 0xce, 0x05, - 0xa6, 0x82, 0x11, 0x67, 0xcf, 0x06, 0x12, 0xa8, 0x58, 0xc4, 0x63, 0xbb, 0xfd, 0x16, 0x61, 0x2a, - 0x2a, 0x57, 0x59, 0xfa, 0xb9, 0xd3, 0x29, 0x93, 0x1d, 0xb3, 0x47, 0x23, 0x7b, 0x75, 0x33, 0x82, - 0xc1, 0x71, 0x4c, 0xfd, 0x3f, 0x34, 0x98, 0x1b, 0xc8, 0xff, 0xa2, 0x7d, 0x98, 0xb0, 0x85, 0x85, - 0x3d, 0xf6, 0xa3, 0xba, 0x98, 0xa1, 0x2e, 0x17, 0x8d, 0x2a, 0x50, 0xf8, 0xc8, 0x86, 0x12, 0x7d, - 0xca, 0xa8, 0x6b, 0x13, 0x4b, 0x1d, 0x8c, 0xe7, 0xf3, 0x80, 0x4f, 0xd8, 0x53, 0x77, 0x15, 0x32, - 0x0e, 0x65, 0xe8, 0x5f, 0xe4, 0xa0, 0x12, 0xe3, 0x7b, 0x5e, 0xc4, 0x56, 0xdc, 0x2b, 0x94, 0xae, - 0xe6, 0xae, 0x6b, 0xa9, 0x89, 0x8e, 0xdd, 0x2b, 0x54, 0x24, 0xbc, 0x89, 0xe3, 0x7c, 0x68, 0x09, - 0xa0, 0x47, 0x3c, 0x46, 0x5d, 0xa1, 0x1b, 0x53, 0xb7, 0xf9, 0xb6, 0x42, 0x0a, 0x8e, 0x71, 0xa1, - 0x9b, 0x2a, 0xfc, 0x51, 0x48, 0xbe, 0x86, 0x19, 0x12, 0xdb, 0x28, 0x9e, 0x43, 0x6c, 0x03, 0x75, - 0x60, 0x36, 0x68, 0x75, 0x40, 0x55, 0x1b, 0xf7, 0x94, 0xc0, 0xd2, 0x54, 0x4c, 0x41, 0xe0, 0x01, - 0x50, 0xfd, 0x6f, 0x34, 0x98, 0x4e, 0xd8, 0xbb, 0xe8, 0xb5, 0xf8, 0xe5, 0x85, 0x58, 0xa4, 0x34, - 0x71, 0xe9, 0xe0, 0x0d, 0x98, 0x90, 0x03, 0xa4, 0x06, 0x3e, 0xdc, 0x88, 0x72, 0x08, 0xb1, 0xa2, - 0xf2, 0x2d, 0xa5, 0x42, 0x29, 0xe9, 0x2d, 0xa5, 0x62, 0x2d, 0x38, 0xa0, 0xa3, 0x2f, 0x71, 0x0b, - 0x5e, 0xb6, 0x4e, 0x8d, 0x74, 0x78, 0x30, 0x04, 0xfd, 0xc0, 0x21, 0x87, 0xfe, 0x36, 0xc8, 0x97, - 0xb7, 0xe8, 0x3a, 0xe4, 0x7b, 0xa6, 0xad, 0xc2, 0x99, 0x22, 0x68, 0xba, 0x65, 0xda, 0x98, 0x97, - 0x09, 0x12, 0x79, 0xaa, 0x02, 0x3c, 0x92, 0x44, 0x9e, 0x62, 0x5e, 0x26, 0xde, 0xb7, 0x89, 0xd8, - 0xf8, 0x23, 0xc8, 0x5b, 0x4e, 0x47, 0x6d, 0xa6, 0xd1, 0x23, 0xb6, 0x9b, 0x4e, 0x47, 0x4a, 0xd8, - 0x74, 0x3a, 0x98, 0x23, 0x22, 0x03, 0x8a, 0x5d, 0xd2, 0xee, 0x12, 0xb5, 0x77, 0x46, 0xd7, 0xb7, - 0x61, 0x1c, 0x5f, 0x3d, 0xda, 0xe2, 0x9f, 0x58, 0x62, 0x23, 0x03, 0x26, 0xfc, 0x96, 0xf8, 0x23, - 0x88, 0x71, 0xff, 0x6b, 0x62, 0x77, 0x55, 0x88, 0x10, 0x8a, 0x40, 0xfe, 0xc6, 0x0a, 0x5a, 0xff, - 0xa7, 0x1c, 0xa8, 0xbf, 0x9f, 0x40, 0x3e, 0x94, 0x3b, 0xc1, 0x23, 0x29, 0x35, 0x66, 0xef, 0x8e, - 0x71, 0x2f, 0x37, 0xf1, 0xdc, 0x4a, 0xa6, 0xa6, 0xc2, 0x42, 0x1c, 0x49, 0x42, 0x34, 0x39, 0x96, - 0xab, 0x63, 0x8e, 0xa5, 0x14, 0x37, 0x38, 0x9a, 0x04, 0x0a, 0xfb, 0x8c, 0xf5, 0xd5, 0x58, 0x8e, - 0xfe, 0x5e, 0x30, 0xba, 0x1f, 0x27, 0x83, 0xa0, 0xfc, 0x1b, 0x0b, 0x68, 0xbd, 0x07, 0xea, 0xa8, - 0x42, 0x46, 0xe2, 0x69, 0xa4, 0x0c, 0x6e, 0x2f, 0x9e, 0xee, 0x00, 0x09, 0x9f, 0xf5, 0xc5, 0x5e, - 0x4c, 0x64, 0xbe, 0x81, 0xd4, 0xff, 0x35, 0x07, 0xf9, 0x9d, 0xcd, 0xa6, 0xbc, 0x00, 0x2c, 0x22, - 0x15, 0xb4, 0xd9, 0x35, 0xfb, 0x0f, 0xa9, 0x6b, 0xb6, 0xa5, 0x97, 0x5a, 0x8a, 0x5f, 0x00, 0x4e, - 0x73, 0xe0, 0x8c, 0x5a, 0xe8, 0x03, 0x98, 0x32, 0xc8, 0x0a, 0x75, 0x99, 0x54, 0x3a, 0x67, 0x8b, - 0xe5, 0x8a, 0x74, 0xe2, 0xca, 0x72, 0x54, 0x1d, 0x27, 0xc0, 0xd0, 0x2e, 0x80, 0x11, 0x41, 0xe7, - 0xcf, 0x02, 0x2d, 0xdf, 0x82, 0x46, 0xc0, 0x31, 0x20, 0x84, 0xa1, 0xdc, 0xe5, 0xac, 0x02, 0xb5, - 0x70, 0x16, 0x54, 0xb1, 0x28, 0xef, 0x07, 0x75, 0x71, 0x04, 0xa3, 0xff, 0x58, 0x83, 0xfc, 0xee, - 0xea, 0x1a, 0x72, 0xa0, 0x1c, 0xde, 0x5a, 0x51, 0x7b, 0xa2, 0x31, 0x7e, 0x18, 0x53, 0x0a, 0x0e, - 0x3f, 0x71, 0x24, 0x03, 0xed, 0xc3, 0xe4, 0x9e, 0x6f, 0x5a, 0xcc, 0xb4, 0x45, 0x9c, 0x67, 0x1c, - 0x4f, 0x23, 0x78, 0x5d, 0xa9, 0x92, 0x66, 0x12, 0x15, 0x07, 0xf0, 0xfa, 0x77, 0x40, 0xe9, 0x02, - 0x6e, 0x41, 0x5e, 0x44, 0x27, 0x43, 0x0b, 0x32, 0xab, 0xa3, 0xfa, 0x3f, 0xe4, 0x60, 0x42, 0xfd, - 0xed, 0xce, 0xc5, 0xc7, 0x00, 0x68, 0x22, 0x06, 0xb0, 0x32, 0xe6, 0x1f, 0x51, 0x0c, 0x8d, 0x00, - 0xf4, 0x52, 0x11, 0x80, 0x71, 0xff, 0xf1, 0xe2, 0x39, 0xfe, 0xff, 0x13, 0x98, 0x8a, 0xff, 0x33, - 0xc6, 0x0b, 0x73, 0xfe, 0xf5, 0x4f, 0x35, 0x80, 0x40, 0xf2, 0x85, 0x7b, 0xde, 0xad, 0xa4, 0xe7, - 0x7d, 0x67, 0xcc, 0x31, 0x1d, 0xe2, 0x77, 0xff, 0x6d, 0x21, 0xe8, 0x92, 0xf0, 0xba, 0x3f, 0xd6, - 0xe0, 0x32, 0x49, 0x78, 0xb2, 0xaa, 0x67, 0xe7, 0xe6, 0x18, 0x5f, 0x53, 0xcd, 0x48, 0xfd, 0x07, - 0x15, 0x4e, 0x89, 0x45, 0xb7, 0x61, 0xaa, 0xaf, 0xfc, 0x19, 0x61, 0xd5, 0xe6, 0x92, 0x77, 0x0b, - 0xb6, 0x63, 0x34, 0x9c, 0xe0, 0x7c, 0x4e, 0xe4, 0x20, 0x7f, 0x2e, 0x91, 0x83, 0x78, 0xae, 0xad, - 0x70, 0x62, 0xae, 0xcd, 0x86, 0x72, 0xdb, 0x75, 0x7a, 0xc2, 0x39, 0x57, 0xff, 0x4d, 0x31, 0xa6, - 0xc3, 0x1f, 0xea, 0x97, 0xb5, 0x00, 0x17, 0x47, 0x22, 0xb8, 0x22, 0x65, 0x8e, 0x94, 0x36, 0x71, - 0x1e, 0xd2, 0x42, 0xdb, 0x76, 0x47, 0xa2, 0xe2, 0x00, 0x5e, 0xff, 0x97, 0x5c, 0xb0, 0x0f, 0x9b, - 0xa9, 0xcb, 0xaa, 0xda, 0x90, 0xcb, 0xaa, 0xea, 0xbd, 0x41, 0xdc, 0x3d, 0x7d, 0x03, 0x26, 0x5c, - 0x4a, 0x3c, 0xc7, 0x56, 0x2f, 0x77, 0xc2, 0x3d, 0x8f, 0x45, 0x29, 0x56, 0xd4, 0xb8, 0x1b, 0x9b, - 0x7b, 0x8e, 0x1b, 0xfb, 0xa5, 0xd8, 0xdc, 0xe4, 0xc5, 0x9e, 0x0e, 0xb7, 0x59, 0xc6, 0xfc, 0x08, - 0x0b, 0x5d, 0xa5, 0x89, 0x8a, 0x69, 0x0b, 0x5d, 0xa5, 0x72, 0x42, 0x0e, 0xd4, 0x82, 0x29, 0xee, - 0xce, 0x0a, 0x2b, 0xbd, 0xb5, 0xcc, 0x46, 0xf0, 0x91, 0xc3, 0x15, 0xbc, 0x19, 0xc3, 0xc1, 0x09, - 0x54, 0xfd, 0xeb, 0x10, 0x45, 0x1f, 0xd0, 0x22, 0x94, 0xfb, 0xae, 0xd3, 0x27, 0x1d, 0xc2, 0xa8, - 0xb2, 0x6e, 0xc2, 0x15, 0xb0, 0x1d, 0x10, 0x70, 0xc4, 0xd3, 0xa8, 0x7f, 0xf2, 0xf9, 0xc2, 0xa5, - 0x4f, 0x3f, 0x5f, 0xb8, 0xf4, 0xd9, 0xe7, 0x0b, 0x97, 0x7e, 0xfb, 0x78, 0x41, 0xfb, 0xe4, 0x78, - 0x41, 0xfb, 0xf4, 0x78, 0x41, 0xfb, 0xec, 0x78, 0x41, 0xfb, 0xf1, 0xf1, 0x82, 0xf6, 0xbd, 0x7f, - 0x5b, 0xb8, 0xf4, 0xcd, 0x52, 0x30, 0xcd, 0xff, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x03, 0xe6, 0xe9, - 0xab, 0xbb, 0x4f, 0x00, 0x00, + 0x5a, 0x4f, 0xf5, 0x1f, 0xbb, 0xfb, 0x6b, 0x3b, 0xb6, 0x5f, 0x32, 0xa1, 0x63, 0x32, 0x76, 0xa8, + 0xd1, 0x8c, 0x02, 0xec, 0xb6, 0x19, 0x33, 0xcb, 0x66, 0x61, 0x67, 0x67, 0xdc, 0x76, 0xec, 0x38, + 0x69, 0x67, 0xcc, 0x6b, 0x3b, 0x61, 0x76, 0x56, 0x84, 0xe7, 0xea, 0xd7, 0xed, 0x9a, 0xae, 0xae, + 0xea, 0xad, 0x7a, 0xe5, 0xc4, 0xa3, 0x5d, 0x81, 0xc4, 0x61, 0x40, 0x80, 0x76, 0x25, 0x2e, 0x48, + 0x2b, 0x10, 0x87, 0x95, 0x10, 0x07, 0x2e, 0x08, 0x24, 0x04, 0x5a, 0x89, 0x13, 0x1a, 0x2e, 0x68, + 0x0e, 0x48, 0x0c, 0xd2, 0xca, 0xda, 0x31, 0x12, 0x37, 0xc4, 0xa2, 0xbd, 0x45, 0x48, 0xa0, 0xf7, + 0xa7, 0xfe, 0x76, 0xb5, 0x63, 0x77, 0xdb, 0xe1, 0x40, 0x6e, 0x5d, 0xef, 0xfb, 0xde, 0xef, 0x7b, + 0x7f, 0xbf, 0xf7, 0xfd, 0x79, 0xaf, 0x61, 0xa3, 0x63, 0xb2, 0x7d, 0x7f, 0xaf, 0x66, 0x38, 0xbd, + 0x25, 0xdb, 0xef, 0x91, 0xbe, 0xeb, 0x7c, 0x28, 0x7e, 0xb4, 0x2d, 0xe7, 0xc9, 0x52, 0xbf, 0xdb, + 0x59, 0x22, 0x7d, 0xd3, 0x8b, 0x4a, 0x0e, 0xde, 0x24, 0x56, 0x7f, 0x9f, 0xbc, 0xb9, 0xd4, 0xa1, + 0x36, 0x75, 0x09, 0xa3, 0xad, 0x5a, 0xdf, 0x75, 0x98, 0x83, 0xbe, 0x1c, 0x01, 0xd5, 0x02, 0xa0, + 0x5a, 0x50, 0xad, 0xd6, 0xef, 0x76, 0x6a, 0x1c, 0x28, 0x2a, 0x09, 0x80, 0xe6, 0xbf, 0x18, 0x6b, + 0x41, 0xc7, 0xe9, 0x38, 0x4b, 0x02, 0x6f, 0xcf, 0x6f, 0x8b, 0x2f, 0xf1, 0x21, 0x7e, 0x49, 0x39, + 0xf3, 0x7a, 0xf7, 0xb6, 0x57, 0x33, 0x1d, 0xde, 0xac, 0x25, 0xc3, 0x71, 0xe9, 0xd2, 0xc1, 0x40, + 0x5b, 0xe6, 0xdf, 0x8a, 0x78, 0x7a, 0xc4, 0xd8, 0x37, 0x6d, 0xea, 0x1e, 0x06, 0x7d, 0x59, 0x72, + 0xa9, 0xe7, 0xf8, 0xae, 0x41, 0xcf, 0x54, 0xcb, 0x5b, 0xea, 0x51, 0x46, 0xb2, 0x64, 0x2d, 0x0d, + 0xab, 0xe5, 0xfa, 0x36, 0x33, 0x7b, 0x83, 0x62, 0x7e, 0xe9, 0x79, 0x15, 0x3c, 0x63, 0x9f, 0xf6, + 0x48, 0xba, 0x9e, 0xfe, 0x9f, 0x53, 0x70, 0x79, 0x65, 0xcf, 0x63, 0x2e, 0x31, 0xd8, 0x43, 0xea, + 0x32, 0xfa, 0x14, 0xdd, 0x84, 0x82, 0x4d, 0x7a, 0xb4, 0xaa, 0xdd, 0xd4, 0x6e, 0x95, 0xeb, 0x53, + 0x9f, 0x1c, 0x2d, 0x5e, 0x3a, 0x3e, 0x5a, 0x2c, 0x3c, 0x20, 0x3d, 0x8a, 0x05, 0x05, 0x19, 0x30, + 0x21, 0x7b, 0x5b, 0xcd, 0xdf, 0xd4, 0x6e, 0x55, 0x96, 0xdf, 0xa9, 0x8d, 0x38, 0x4d, 0xb5, 0xa6, + 0x80, 0xa9, 0xc3, 0xf1, 0xd1, 0xe2, 0x84, 0xfc, 0x8d, 0x15, 0x34, 0xfa, 0x00, 0x0a, 0x9e, 0x69, + 0x77, 0xab, 0x05, 0x21, 0xe2, 0xed, 0xd1, 0x45, 0x98, 0x76, 0xb7, 0x5e, 0xe2, 0x3d, 0xe0, 0xbf, + 0xb0, 0x00, 0x45, 0xdf, 0xd1, 0x60, 0xce, 0x70, 0x6c, 0x46, 0xf8, 0x40, 0xed, 0xd0, 0x5e, 0xdf, + 0x22, 0x8c, 0x56, 0x8b, 0x42, 0xd4, 0xbd, 0x91, 0x45, 0xad, 0xa6, 0x11, 0xeb, 0xaf, 0x1c, 0x1f, + 0x2d, 0xce, 0x0d, 0x14, 0xe3, 0x41, 0xd9, 0xe8, 0x11, 0xe4, 0xfd, 0x56, 0xbb, 0x3a, 0x21, 0x9a, + 0xf0, 0xd5, 0x91, 0x9b, 0xb0, 0xbb, 0xb6, 0x5e, 0x9f, 0x3c, 0x3e, 0x5a, 0xcc, 0xef, 0xae, 0xad, + 0x63, 0x8e, 0x88, 0xba, 0x50, 0xe2, 0xab, 0xac, 0x45, 0x18, 0xa9, 0x4e, 0x0a, 0xf4, 0x95, 0x91, + 0xd1, 0xb7, 0x14, 0x50, 0x7d, 0xea, 0xf8, 0x68, 0xb1, 0x14, 0x7c, 0xe1, 0x50, 0x00, 0xfa, 0x43, + 0x0d, 0xa6, 0x6c, 0xa7, 0x45, 0x9b, 0xd4, 0xa2, 0x06, 0x73, 0xdc, 0x6a, 0xe9, 0x66, 0xfe, 0x56, + 0x65, 0xf9, 0xfd, 0x91, 0x25, 0x26, 0xd7, 0x66, 0xed, 0x41, 0x0c, 0xfb, 0x8e, 0xcd, 0xdc, 0xc3, + 0xfa, 0x55, 0xb5, 0x3e, 0xa7, 0xe2, 0x24, 0x9c, 0x68, 0x04, 0xda, 0x85, 0x0a, 0x73, 0x2c, 0xbe, + 0xee, 0x4d, 0xc7, 0xf6, 0xaa, 0x65, 0xd1, 0xa6, 0x85, 0x9a, 0xdc, 0x32, 0x5c, 0x72, 0x8d, 0xef, + 0xf9, 0xda, 0xc1, 0x9b, 0xb5, 0x9d, 0x90, 0xad, 0x7e, 0x45, 0x01, 0x57, 0xa2, 0x32, 0x0f, 0xc7, + 0x71, 0x10, 0x85, 0x19, 0x8f, 0x1a, 0xbe, 0x6b, 0xb2, 0x43, 0x3e, 0xc5, 0xf4, 0x29, 0xab, 0x82, + 0x18, 0xe0, 0x37, 0xb2, 0xa0, 0xb7, 0x9d, 0x56, 0x33, 0xc9, 0x5d, 0xbf, 0x72, 0x7c, 0xb4, 0x38, + 0x93, 0x2a, 0xc4, 0x69, 0x4c, 0x64, 0xc3, 0xac, 0xd9, 0x23, 0x1d, 0xba, 0xed, 0x5b, 0x56, 0x93, + 0x1a, 0x2e, 0x65, 0x5e, 0xb5, 0x22, 0xba, 0x70, 0x2b, 0x4b, 0x4e, 0xc3, 0x31, 0x88, 0xf5, 0xde, + 0xde, 0x87, 0xd4, 0x60, 0x98, 0xb6, 0xa9, 0x4b, 0x6d, 0x83, 0xd6, 0xab, 0xaa, 0x33, 0xb3, 0x9b, + 0x29, 0x24, 0x3c, 0x80, 0x8d, 0x36, 0x60, 0xae, 0xef, 0x9a, 0x8e, 0x68, 0x82, 0x45, 0x3c, 0x8f, + 0x6f, 0xfc, 0xea, 0x94, 0x50, 0x06, 0xd7, 0x15, 0xcc, 0xdc, 0x76, 0x9a, 0x01, 0x0f, 0xd6, 0x41, + 0xb7, 0xa0, 0x14, 0x14, 0x56, 0xa7, 0x6f, 0x6a, 0xb7, 0x8a, 0x72, 0xd9, 0x04, 0x75, 0x71, 0x48, + 0x45, 0xeb, 0x50, 0x22, 0xed, 0xb6, 0x69, 0x73, 0xce, 0xcb, 0x62, 0x08, 0x6f, 0x64, 0x75, 0x6d, + 0x45, 0xf1, 0x48, 0x9c, 0xe0, 0x0b, 0x87, 0x75, 0xd1, 0x3d, 0x40, 0x1e, 0x75, 0x0f, 0x4c, 0x83, + 0xae, 0x18, 0x86, 0xe3, 0xdb, 0x4c, 0xb4, 0x7d, 0x46, 0xb4, 0x7d, 0x5e, 0xb5, 0x1d, 0x35, 0x07, + 0x38, 0x70, 0x46, 0x2d, 0x74, 0x07, 0x26, 0x0f, 0x1c, 0xcb, 0xef, 0x51, 0xaf, 0x3a, 0x2b, 0x46, + 0x7b, 0x3e, 0xab, 0x49, 0x0f, 0x05, 0x4b, 0x7d, 0x46, 0x81, 0x4f, 0xca, 0x6f, 0x0f, 0x07, 0x75, + 0x91, 0x09, 0x13, 0x96, 0xd9, 0x33, 0x99, 0x57, 0x9d, 0x13, 0x1d, 0xbb, 0x33, 0xf2, 0x56, 0x90, + 0x5b, 0xa0, 0x21, 0xc0, 0xa4, 0xc6, 0x94, 0xbf, 0xb1, 0x12, 0x80, 0x0c, 0x28, 0x7a, 0x06, 0xb1, + 0x68, 0x15, 0x09, 0x49, 0x5f, 0x1b, 0x5d, 0x65, 0x72, 0x94, 0xfa, 0xb4, 0xea, 0x53, 0x51, 0x7c, + 0x62, 0x89, 0x3d, 0xff, 0x0e, 0xcc, 0x0d, 0x6c, 0x42, 0x34, 0x0b, 0xf9, 0x2e, 0x3d, 0x94, 0x27, + 0x06, 0xe6, 0x3f, 0xd1, 0x55, 0x28, 0x1e, 0x10, 0xcb, 0xa7, 0xd5, 0x9c, 0x28, 0x93, 0x1f, 0xbf, + 0x9c, 0xbb, 0xad, 0xe9, 0x8f, 0x60, 0x7a, 0xc5, 0x67, 0xfb, 0x8e, 0x6b, 0x7e, 0x24, 0xf6, 0x11, + 0x5a, 0x87, 0x22, 0x73, 0xba, 0xd4, 0x16, 0xd5, 0x2b, 0xcb, 0xaf, 0x67, 0x0d, 0xb3, 0x5c, 0x9b, + 0xf7, 0xe9, 0x61, 0x20, 0xb7, 0x5e, 0xe6, 0x2d, 0xdb, 0xe1, 0xf5, 0xb0, 0xac, 0xae, 0x7f, 0x1d, + 0x26, 0xea, 0x7e, 0xbb, 0x4d, 0xdd, 0x53, 0x9c, 0x60, 0x35, 0x28, 0xb0, 0xc3, 0xbe, 0x6a, 0x5d, + 0xb8, 0x34, 0x0a, 0x3b, 0x87, 0x7d, 0xfa, 0xec, 0x68, 0x11, 0x24, 0x0e, 0xff, 0xc2, 0x82, 0x4f, + 0xff, 0x89, 0x06, 0x57, 0x64, 0xa1, 0x5a, 0x3d, 0xab, 0x8e, 0xdd, 0x36, 0x3b, 0x88, 0x42, 0xd1, + 0xa5, 0x2d, 0xd3, 0x53, 0x6d, 0x5f, 0x1b, 0x79, 0xc8, 0x31, 0x47, 0x91, 0xa0, 0xb2, 0x6b, 0xa2, + 0x00, 0x4b, 0x74, 0xe4, 0x43, 0xf9, 0x43, 0xca, 0x3c, 0xe6, 0x52, 0xd2, 0x13, 0x6d, 0xae, 0x2c, + 0xdf, 0x1d, 0x59, 0xd4, 0x3d, 0xca, 0x9a, 0x02, 0x49, 0x89, 0x9b, 0x3e, 0x3e, 0x5a, 0x2c, 0x87, + 0x85, 0x38, 0x92, 0xa4, 0xff, 0x7b, 0x0e, 0xca, 0xe1, 0xe1, 0x85, 0x5e, 0x83, 0xa2, 0xd0, 0x15, + 0x6a, 0x58, 0xc3, 0xe5, 0x21, 0x54, 0x0a, 0x96, 0x34, 0xf4, 0x3a, 0x4c, 0x1a, 0x4e, 0xaf, 0x47, + 0xec, 0x56, 0x35, 0x77, 0x33, 0x7f, 0xab, 0x5c, 0xaf, 0xf0, 0x5d, 0xb1, 0x2a, 0x8b, 0x70, 0x40, + 0x43, 0x37, 0xa0, 0x40, 0xdc, 0x8e, 0x57, 0xcd, 0x0b, 0x1e, 0x71, 0x3a, 0xaf, 0xb8, 0x1d, 0x0f, + 0x8b, 0x52, 0xf4, 0x15, 0xc8, 0x53, 0xfb, 0xa0, 0x5a, 0x18, 0xbe, 0xed, 0xee, 0xd8, 0x07, 0x0f, + 0x89, 0x5b, 0xaf, 0xa8, 0x36, 0xe4, 0xef, 0xd8, 0x07, 0x98, 0xd7, 0x41, 0xef, 0xc3, 0x94, 0xdc, + 0x79, 0x5b, 0x7c, 0x23, 0x7b, 0xd5, 0xa2, 0xc0, 0x58, 0x1c, 0xbe, 0x75, 0x05, 0x5f, 0x74, 0x8a, + 0xc4, 0x0a, 0x3d, 0x9c, 0x80, 0x42, 0xef, 0x43, 0x39, 0xb0, 0xf2, 0x3c, 0x75, 0x4e, 0x67, 0x2a, + 0x60, 0xac, 0x98, 0x30, 0xfd, 0xa6, 0x6f, 0xba, 0xb4, 0x47, 0x6d, 0xe6, 0xd5, 0xe7, 0x94, 0x80, + 0x72, 0x40, 0xf5, 0x70, 0x84, 0xa6, 0xff, 0x57, 0x0e, 0x06, 0xad, 0x84, 0xa4, 0x40, 0xed, 0x3c, + 0x05, 0xa2, 0x3d, 0x98, 0x09, 0xf5, 0xfe, 0xb6, 0x63, 0x99, 0xc6, 0xa1, 0xda, 0x0a, 0xb7, 0x55, + 0xb5, 0x99, 0xcd, 0x24, 0xf9, 0xd9, 0xd1, 0xe2, 0xab, 0x83, 0x36, 0x72, 0x2d, 0x62, 0xc0, 0x69, + 0x40, 0x2e, 0x23, 0x7d, 0x3c, 0x4a, 0x73, 0xf1, 0xb5, 0x21, 0x3b, 0x7c, 0x84, 0xb3, 0x71, 0xf4, + 0x95, 0xa2, 0xff, 0x79, 0x0e, 0x0a, 0x77, 0x5a, 0x1d, 0xca, 0xb5, 0x45, 0xdb, 0x75, 0x7a, 0x69, + 0x6d, 0xb1, 0xee, 0x3a, 0x3d, 0x2c, 0x28, 0x68, 0x1e, 0x72, 0xcc, 0x51, 0x03, 0x04, 0x8a, 0x9e, + 0xdb, 0x71, 0x70, 0x8e, 0x39, 0xe8, 0x23, 0x00, 0xc3, 0xb1, 0x5b, 0xa6, 0x34, 0x2d, 0xf2, 0x63, + 0x5a, 0x90, 0xeb, 0x8e, 0xfb, 0x84, 0xb8, 0xad, 0xd5, 0x10, 0xb1, 0x7e, 0xf9, 0xf8, 0x68, 0x11, + 0xa2, 0x6f, 0x1c, 0x93, 0x86, 0x3a, 0xe1, 0xd9, 0x22, 0x8d, 0xe4, 0xd5, 0x91, 0xe5, 0xf2, 0x81, + 0x18, 0x7e, 0xb2, 0xe8, 0x7f, 0xa0, 0x01, 0x44, 0x2c, 0xe8, 0x6d, 0x98, 0xd9, 0x13, 0xca, 0x70, + 0x8b, 0x3c, 0x6d, 0x50, 0xbb, 0xc3, 0xf6, 0xc5, 0xe0, 0x15, 0xe4, 0xa4, 0xd5, 0x93, 0x24, 0x9c, + 0xe6, 0x45, 0xef, 0xc2, 0xac, 0x2c, 0xda, 0xf5, 0x88, 0xc2, 0x14, 0x83, 0x3b, 0x5d, 0xbf, 0xca, + 0x4d, 0x94, 0x7a, 0x8a, 0x86, 0x07, 0xb8, 0xf5, 0xb7, 0x60, 0x6e, 0x60, 0xa4, 0xd0, 0x22, 0x14, + 0xbb, 0xf4, 0x70, 0x93, 0x9f, 0x23, 0x5c, 0xa9, 0x08, 0x2d, 0x7a, 0x9f, 0x17, 0x60, 0x59, 0xae, + 0xff, 0xb7, 0x06, 0xa5, 0x75, 0xdf, 0x36, 0xc4, 0xa9, 0xf3, 0xfc, 0x33, 0x22, 0xd0, 0x51, 0xb9, + 0x4c, 0x1d, 0xe5, 0xc3, 0x44, 0xf7, 0x49, 0xa8, 0xc3, 0x2a, 0xcb, 0x5b, 0xa3, 0xcf, 0xb9, 0x6a, + 0x52, 0xed, 0xbe, 0xc0, 0x93, 0x66, 0xed, 0x65, 0xd5, 0xa0, 0x89, 0xfb, 0x8f, 0x84, 0x50, 0x25, + 0x6c, 0xfe, 0x2b, 0x50, 0x89, 0xb1, 0x9d, 0xe9, 0xe0, 0xfd, 0x0b, 0x0d, 0x66, 0x36, 0xa4, 0xfb, + 0xe7, 0xb8, 0xd2, 0xd9, 0x42, 0xd7, 0x21, 0xef, 0xf6, 0x7d, 0x51, 0x3f, 0x2f, 0xfd, 0x06, 0xbc, + 0xbd, 0x8b, 0x79, 0x19, 0xfa, 0x35, 0x28, 0xb5, 0x7c, 0x69, 0xea, 0xaa, 0x23, 0xa7, 0x16, 0xdb, + 0x5f, 0xa1, 0x93, 0x19, 0xf5, 0x8c, 0x3b, 0x01, 0x7c, 0xc7, 0xad, 0xa9, 0x5a, 0xd2, 0x4a, 0x0b, + 0xbe, 0x70, 0x88, 0xc6, 0xcf, 0x88, 0x9e, 0xd7, 0x69, 0x9a, 0x1f, 0x49, 0xff, 0xb1, 0x28, 0xcf, + 0x88, 0x2d, 0x59, 0x84, 0x03, 0x9a, 0xfe, 0x9d, 0x1c, 0x5c, 0xdb, 0xa0, 0x6c, 0x8d, 0xd0, 0x9e, + 0x63, 0xaf, 0xd1, 0xbe, 0xe5, 0x1c, 0x72, 0xd5, 0x86, 0xe9, 0x37, 0xd1, 0xbb, 0x00, 0xa6, 0xb7, + 0xd7, 0x3c, 0x30, 0xf8, 0x11, 0xad, 0xa6, 0xf0, 0xa6, 0x1a, 0x31, 0xd8, 0x6c, 0xd6, 0x15, 0xe5, + 0x59, 0xe2, 0x0b, 0xc7, 0xea, 0x44, 0x87, 0x59, 0xee, 0x84, 0xc3, 0xac, 0x09, 0xd0, 0x8f, 0x14, + 0x64, 0x5e, 0x70, 0xfe, 0x62, 0x20, 0xe6, 0x2c, 0xba, 0x31, 0x06, 0x33, 0x8e, 0xca, 0xfa, 0xdb, + 0x3c, 0xcc, 0x6f, 0x50, 0x16, 0x9e, 0xd5, 0xca, 0x16, 0x69, 0xf6, 0xa9, 0xc1, 0x47, 0xe5, 0x63, + 0x0d, 0x26, 0x2c, 0xb2, 0x47, 0x2d, 0x4f, 0x6c, 0x81, 0xca, 0xf2, 0xe3, 0x91, 0xd7, 0xe4, 0x70, + 0x29, 0xb5, 0x86, 0x90, 0x90, 0x5a, 0xa5, 0xb2, 0x10, 0x2b, 0xf1, 0xe8, 0x4b, 0x50, 0x31, 0x2c, + 0xdf, 0x63, 0xd4, 0xdd, 0x76, 0x5c, 0xb9, 0xb9, 0x8b, 0x91, 0x43, 0xb5, 0x1a, 0x91, 0x70, 0x9c, + 0x0f, 0x2d, 0x03, 0x18, 0x96, 0x49, 0x6d, 0x26, 0x6a, 0xc9, 0xb5, 0x81, 0x82, 0xf1, 0x5e, 0x0d, + 0x29, 0x38, 0xc6, 0xc5, 0x45, 0xf5, 0x1c, 0xdb, 0x64, 0x8e, 0x14, 0x55, 0x48, 0x8a, 0xda, 0x8a, + 0x48, 0x38, 0xce, 0x27, 0xaa, 0x51, 0xe6, 0x9a, 0x86, 0x27, 0xaa, 0x15, 0x53, 0xd5, 0x22, 0x12, + 0x8e, 0xf3, 0xf1, 0xed, 0x17, 0xeb, 0xff, 0x99, 0xb6, 0xdf, 0xdf, 0x95, 0x60, 0x21, 0x31, 0xac, + 0x8c, 0x30, 0xda, 0xf6, 0xad, 0x26, 0x65, 0xc1, 0x04, 0x7e, 0x09, 0x2a, 0xca, 0x11, 0x79, 0x10, + 0xa9, 0xa6, 0xb0, 0x51, 0xcd, 0x88, 0x84, 0xe3, 0x7c, 0xe8, 0xf7, 0xa2, 0x79, 0xcf, 0x89, 0x79, + 0x37, 0xce, 0x67, 0xde, 0x07, 0x1a, 0x78, 0xaa, 0xb9, 0x5f, 0x82, 0xb2, 0x4d, 0x98, 0x27, 0x36, + 0x92, 0xda, 0x33, 0xa1, 0x2d, 0xf2, 0x20, 0x20, 0xe0, 0x88, 0x07, 0x6d, 0xc3, 0x55, 0x35, 0xc4, + 0x77, 0x9e, 0xf6, 0x1d, 0x97, 0x51, 0x57, 0xd6, 0x2d, 0x88, 0xba, 0x37, 0x54, 0xdd, 0xab, 0x5b, + 0x19, 0x3c, 0x38, 0xb3, 0x26, 0xda, 0x82, 0x2b, 0x86, 0xb0, 0x6d, 0x31, 0xb5, 0x1c, 0xd2, 0x0a, + 0x00, 0x8b, 0x02, 0xf0, 0xa7, 0x15, 0xe0, 0x95, 0xd5, 0x41, 0x16, 0x9c, 0x55, 0x2f, 0xbd, 0x9a, + 0x27, 0x46, 0x5a, 0xcd, 0x93, 0xa3, 0xac, 0xe6, 0xd2, 0x68, 0xab, 0xb9, 0x7c, 0xba, 0xd5, 0xcc, + 0x47, 0x9e, 0xaf, 0x23, 0xea, 0x72, 0x87, 0x4c, 0xba, 0x58, 0x62, 0xe1, 0x41, 0x72, 0xe4, 0x9b, + 0x19, 0x3c, 0x38, 0xb3, 0x26, 0xda, 0x83, 0x79, 0x59, 0x7e, 0xc7, 0x36, 0xdc, 0xc3, 0x3e, 0x57, + 0xf7, 0x31, 0xdc, 0x8a, 0xc0, 0xd5, 0x15, 0xee, 0x7c, 0x73, 0x28, 0x27, 0x3e, 0x01, 0x05, 0xfd, + 0x0a, 0x4c, 0xcb, 0x59, 0xda, 0x22, 0xfd, 0x58, 0x6c, 0xe2, 0x15, 0x05, 0x3b, 0xbd, 0x1a, 0x27, + 0xe2, 0x24, 0x2f, 0x5a, 0x81, 0x99, 0xfe, 0x81, 0xc1, 0x7f, 0x6e, 0xb6, 0x1f, 0x50, 0xda, 0xa2, + 0x2d, 0x11, 0x9a, 0x28, 0xd7, 0x7f, 0x2a, 0x30, 0x7c, 0xb7, 0x93, 0x64, 0x9c, 0xe6, 0x47, 0xb7, + 0x61, 0xca, 0x63, 0xc4, 0x65, 0xca, 0xa9, 0x11, 0x01, 0x8b, 0x72, 0xe4, 0x41, 0x34, 0x63, 0x34, + 0x9c, 0xe0, 0x1c, 0x47, 0x7b, 0x3c, 0x93, 0x87, 0xa1, 0xf0, 0x0a, 0x53, 0x6a, 0xff, 0xb7, 0xd3, + 0x6a, 0xff, 0x83, 0x71, 0xb6, 0x7f, 0x86, 0x84, 0x53, 0x6d, 0xfb, 0x7b, 0x80, 0x5c, 0xe5, 0xc3, + 0x4a, 0x37, 0x26, 0xa6, 0xf9, 0xc3, 0xd0, 0x0b, 0x1e, 0xe0, 0xc0, 0x19, 0xb5, 0x50, 0x13, 0x5e, + 0xf1, 0xa8, 0xcd, 0x4c, 0x9b, 0x5a, 0x49, 0x38, 0x79, 0x24, 0xbc, 0xaa, 0xe0, 0x5e, 0x69, 0x66, + 0x31, 0xe1, 0xec, 0xba, 0xe3, 0x0c, 0xfe, 0x0f, 0xcb, 0xe2, 0xdc, 0x95, 0x43, 0x73, 0x6e, 0x6a, + 0xfb, 0xe3, 0xb4, 0xda, 0x7e, 0x3c, 0xfe, 0xbc, 0x8d, 0xa6, 0xb2, 0x97, 0x01, 0xc4, 0x2c, 0xc4, + 0x75, 0x76, 0xa8, 0xa9, 0x70, 0x48, 0xc1, 0x31, 0x2e, 0xbe, 0x0b, 0x83, 0x71, 0x8e, 0xab, 0xeb, + 0x70, 0x17, 0x36, 0xe3, 0x44, 0x9c, 0xe4, 0x1d, 0xaa, 0xf2, 0x8b, 0x23, 0xab, 0xfc, 0x7b, 0x80, + 0x4c, 0xdb, 0x64, 0xe1, 0x94, 0x4b, 0xbc, 0x89, 0x64, 0xe4, 0x6f, 0x73, 0x80, 0x03, 0x67, 0xd4, + 0x1a, 0xb2, 0x94, 0x27, 0xcf, 0x77, 0x29, 0x97, 0x46, 0x5f, 0xca, 0xe8, 0x31, 0x5c, 0x17, 0xa2, + 0xd4, 0xf8, 0x24, 0x81, 0xa5, 0xf2, 0xff, 0x19, 0x05, 0x7c, 0x1d, 0x0f, 0x63, 0xc4, 0xc3, 0x31, + 0xf8, 0xfc, 0x18, 0x2e, 0x6d, 0x71, 0xe1, 0xc4, 0x1a, 0x7e, 0x30, 0xac, 0x66, 0xf0, 0xe0, 0xcc, + 0x9a, 0x7c, 0x89, 0x31, 0xbe, 0x0c, 0xc9, 0x9e, 0x45, 0x5b, 0xe2, 0x20, 0x28, 0x45, 0x4b, 0x6c, + 0xa7, 0xd1, 0x54, 0x14, 0x1c, 0xe3, 0xca, 0xd2, 0xd5, 0x53, 0x67, 0xd4, 0xd5, 0x1b, 0x22, 0xcd, + 0xd3, 0x4e, 0x1c, 0x09, 0x4a, 0xe1, 0x87, 0xb1, 0xec, 0xd5, 0x34, 0x03, 0x1e, 0xac, 0x23, 0x8e, + 0x4a, 0xc3, 0x35, 0xfb, 0xcc, 0x4b, 0x62, 0x5d, 0x4e, 0x1d, 0x95, 0x19, 0x3c, 0x38, 0xb3, 0x26, + 0x37, 0x52, 0xf6, 0x29, 0xb1, 0xd8, 0x7e, 0x12, 0x70, 0x26, 0x69, 0xa4, 0xdc, 0x1d, 0x64, 0xc1, + 0x59, 0xf5, 0xc6, 0x51, 0x6f, 0xbf, 0x9f, 0x83, 0x2b, 0x1b, 0x54, 0xa5, 0x58, 0xb6, 0x9d, 0x56, + 0xa0, 0xd7, 0xfe, 0x9f, 0x7a, 0x59, 0x7f, 0xac, 0x01, 0xdc, 0xdd, 0xd9, 0xd9, 0x56, 0x2e, 0x72, + 0x0b, 0x0a, 0xc4, 0x57, 0x11, 0x8e, 0xca, 0xf2, 0xfa, 0xe8, 0x99, 0xac, 0x78, 0xd0, 0x5b, 0x85, + 0x13, 0x7c, 0xb6, 0x8f, 0x05, 0x3a, 0xfa, 0x59, 0x98, 0x54, 0x67, 0x83, 0x18, 0xab, 0x52, 0x94, + 0x51, 0x50, 0xe7, 0x07, 0x0e, 0xe8, 0xfa, 0x8f, 0x73, 0x70, 0x6d, 0xd3, 0x66, 0xd4, 0x6d, 0x32, + 0xda, 0x4f, 0x04, 0xa5, 0xd1, 0x6f, 0xc4, 0x72, 0x7d, 0xb2, 0xbd, 0xbf, 0x70, 0x3a, 0x9f, 0x5d, + 0xe6, 0x8b, 0xb6, 0x28, 0x23, 0xd1, 0xae, 0x8c, 0xca, 0x62, 0x09, 0x3e, 0x1f, 0x0a, 0x5e, 0x9f, + 0x1a, 0x2a, 0x22, 0xd0, 0x1c, 0x79, 0x34, 0xb2, 0x3b, 0xc0, 0x57, 0x5e, 0x14, 0x8b, 0x11, 0xeb, + 0x50, 0x88, 0x43, 0xdf, 0x86, 0x09, 0x8f, 0x11, 0xe6, 0x07, 0x11, 0xb6, 0xdd, 0xf3, 0x16, 0x2c, + 0xc0, 0xa3, 0x03, 0x52, 0x7e, 0x63, 0x25, 0x54, 0xff, 0xb1, 0x06, 0xf3, 0xd9, 0x15, 0x1b, 0xa6, + 0xc7, 0xd0, 0x37, 0x06, 0x86, 0xfd, 0x94, 0xa1, 0x12, 0x5e, 0x5b, 0x0c, 0xfa, 0xac, 0x12, 0x5c, + 0x0a, 0x4a, 0x62, 0x43, 0xce, 0xa0, 0x68, 0x32, 0xda, 0x0b, 0xac, 0x84, 0xf7, 0xce, 0xb9, 0xeb, + 0xb1, 0x5d, 0xc9, 0xa5, 0x60, 0x29, 0x4c, 0xff, 0x38, 0x37, 0xac, 0xcb, 0x7c, 0x5a, 0x50, 0x37, + 0x99, 0xf8, 0xb8, 0x37, 0x5e, 0xe2, 0xa3, 0xee, 0xc7, 0xda, 0x33, 0x98, 0xfe, 0xf8, 0xd6, 0x60, + 0xfa, 0xe3, 0xbd, 0xf1, 0xd3, 0x1f, 0xa9, 0x51, 0x18, 0x9a, 0x05, 0xf9, 0x61, 0x0e, 0x6e, 0x9c, + 0xb4, 0x6a, 0x50, 0x27, 0x5c, 0x9c, 0xda, 0xb8, 0xd7, 0x21, 0x4e, 0x5c, 0x86, 0x68, 0x19, 0x8a, + 0xfd, 0x7d, 0xe2, 0x05, 0xea, 0x34, 0x38, 0x75, 0x8a, 0xdb, 0xbc, 0xf0, 0xd9, 0xd1, 0x62, 0x45, + 0xaa, 0x61, 0xf1, 0x89, 0x25, 0x2b, 0x57, 0x2c, 0x3d, 0xea, 0x79, 0x91, 0x61, 0x17, 0x2a, 0x96, + 0x2d, 0x59, 0x8c, 0x03, 0x3a, 0x62, 0x30, 0x21, 0x9d, 0x25, 0x15, 0x4e, 0x6e, 0x8c, 0xdc, 0x8f, + 0x8c, 0x54, 0x59, 0xd4, 0x29, 0xe5, 0x77, 0x2b, 0x59, 0xfa, 0x5f, 0x5e, 0x86, 0x6b, 0xd9, 0x73, + 0xc2, 0xdb, 0x7e, 0x40, 0x5d, 0xcf, 0x74, 0x6c, 0x75, 0xfa, 0x44, 0x69, 0x56, 0x59, 0x8c, 0x03, + 0x3a, 0xba, 0x05, 0x25, 0x97, 0xf6, 0x2d, 0xd3, 0x20, 0x9e, 0x72, 0x3a, 0x44, 0xf4, 0x11, 0xab, + 0x32, 0x1c, 0x52, 0x87, 0x5c, 0xfd, 0xc8, 0xff, 0x1f, 0x5e, 0xfd, 0xf8, 0x33, 0x8d, 0xdb, 0x73, + 0x32, 0xe2, 0x30, 0x50, 0x41, 0xcd, 0xc5, 0x79, 0xb6, 0xec, 0x55, 0x69, 0x17, 0x0e, 0x11, 0x88, + 0x87, 0xb7, 0x05, 0x7d, 0x5f, 0x83, 0x6a, 0x2f, 0x65, 0x30, 0x5e, 0xe0, 0xed, 0x99, 0x1b, 0xc7, + 0x47, 0x8b, 0xd5, 0xad, 0x21, 0xf2, 0xf0, 0xd0, 0x96, 0xa0, 0xdf, 0x84, 0x4a, 0x9f, 0xaf, 0x0b, + 0x8f, 0x51, 0xdb, 0xa0, 0x2a, 0x57, 0x37, 0xfa, 0x6a, 0xde, 0x8e, 0xb0, 0x9a, 0xcc, 0x25, 0x8c, + 0x76, 0x0e, 0xeb, 0x33, 0xdc, 0xb5, 0x8b, 0x11, 0x70, 0x5c, 0x62, 0xe2, 0xce, 0xcd, 0xd6, 0x45, + 0xdf, 0xb9, 0xf9, 0x5e, 0xf6, 0x9d, 0x1b, 0x72, 0xce, 0x1a, 0xf2, 0xe5, 0xdd, 0x9b, 0x97, 0x77, + 0x6f, 0x5e, 0xd4, 0xdd, 0x9b, 0x5b, 0x50, 0xf2, 0x28, 0x63, 0xa6, 0xdd, 0xf1, 0xaa, 0xb3, 0x32, + 0x41, 0xc7, 0xa5, 0x36, 0x55, 0x19, 0x0e, 0xa9, 0xe8, 0xe7, 0xa1, 0x2c, 0x42, 0x6c, 0x2b, 0x6e, + 0xc7, 0xab, 0xce, 0x89, 0x4c, 0x9d, 0x38, 0xc9, 0x9b, 0x41, 0x21, 0x8e, 0xe8, 0xe8, 0x2d, 0x98, + 0x92, 0xa9, 0x44, 0x79, 0x04, 0x89, 0x7b, 0x32, 0xe5, 0xfa, 0x2c, 0x5f, 0xc1, 0xf5, 0x58, 0x39, + 0x4e, 0x70, 0x71, 0xd7, 0x95, 0x86, 0x71, 0xc8, 0xea, 0x95, 0xa4, 0xeb, 0x1a, 0x45, 0x28, 0x71, + 0x8c, 0x0b, 0xbd, 0x0a, 0x79, 0x66, 0x79, 0xd5, 0xab, 0x82, 0x39, 0x74, 0x31, 0x76, 0x1a, 0x4d, + 0xcc, 0xcb, 0xc7, 0xbf, 0x44, 0xf3, 0x3f, 0x1a, 0xcc, 0xa4, 0xee, 0x71, 0x70, 0x99, 0xbe, 0x6b, + 0xa9, 0x93, 0x32, 0x94, 0xb9, 0x8b, 0x1b, 0x98, 0x97, 0xa3, 0xc7, 0xca, 0x8f, 0xc9, 0x8d, 0xa9, + 0x8f, 0x1e, 0xac, 0xec, 0x34, 0xb9, 0xe3, 0x32, 0xe0, 0xc2, 0xdc, 0x4e, 0x8d, 0x6e, 0x3e, 0x19, + 0x17, 0x3d, 0x79, 0x84, 0x63, 0xc1, 0x81, 0xc2, 0x69, 0x82, 0x03, 0xfa, 0x67, 0x1a, 0x94, 0xef, + 0x93, 0x76, 0x97, 0x34, 0x4d, 0xbb, 0x8b, 0x5e, 0x87, 0xc9, 0x3d, 0xd7, 0xe9, 0x52, 0xd7, 0x53, + 0xd9, 0x5f, 0x91, 0x52, 0xac, 0xcb, 0x22, 0x1c, 0xd0, 0xb8, 0x3f, 0xca, 0x9c, 0xbe, 0x69, 0xa4, + 0xfd, 0xd1, 0x1d, 0x5e, 0x88, 0x25, 0x0d, 0x3d, 0x92, 0x73, 0x97, 0x1f, 0xf3, 0x26, 0xe6, 0x4e, + 0xa3, 0x29, 0x33, 0xaa, 0xc1, 0xac, 0xa3, 0x37, 0x12, 0xf6, 0x55, 0x79, 0xa8, 0x45, 0xf4, 0xbd, + 0x1c, 0x54, 0x64, 0xd7, 0xa4, 0x07, 0x7a, 0x9e, 0x9d, 0x7b, 0x47, 0x04, 0xcf, 0x3d, 0xbf, 0x47, + 0xdd, 0x0d, 0xd7, 0xf1, 0xfb, 0x6a, 0x96, 0xe2, 0xc1, 0x90, 0x88, 0x18, 0x06, 0xd0, 0xa3, 0xa2, + 0x60, 0x74, 0x0a, 0x17, 0x38, 0x3a, 0xc5, 0x13, 0x47, 0xe7, 0xaf, 0x34, 0x28, 0x37, 0xcc, 0x36, + 0x35, 0x0e, 0x0d, 0x8b, 0xa2, 0x6f, 0x40, 0xb5, 0x45, 0x2d, 0xca, 0xe8, 0x86, 0x4b, 0x0c, 0xba, + 0x4d, 0x5d, 0x53, 0xa8, 0x7a, 0xc7, 0x6e, 0x49, 0x6b, 0xbc, 0x18, 0x46, 0x2c, 0xaa, 0x6b, 0x43, + 0xf8, 0xf0, 0x50, 0x04, 0xb4, 0x09, 0x53, 0x2d, 0xea, 0x99, 0x2e, 0x6d, 0x6d, 0xc7, 0xec, 0xee, + 0xd7, 0x83, 0x25, 0xbd, 0x16, 0xa3, 0x3d, 0x3b, 0x5a, 0x9c, 0xde, 0x36, 0xfb, 0xd4, 0x32, 0x6d, + 0x2a, 0x0d, 0xf0, 0x44, 0x55, 0xbd, 0x08, 0xf9, 0x86, 0xd3, 0xd1, 0x7f, 0x27, 0x0f, 0xe1, 0x19, + 0x8e, 0x7e, 0x57, 0x83, 0x0a, 0xb1, 0x6d, 0x87, 0xa9, 0xc3, 0x51, 0x86, 0xef, 0xf1, 0xd8, 0xa6, + 0x42, 0x6d, 0x25, 0x02, 0x95, 0x27, 0x75, 0x78, 0xa0, 0xc6, 0x28, 0x38, 0x2e, 0x1b, 0xf9, 0xa9, + 0x60, 0xf4, 0xd6, 0xf8, 0xad, 0x38, 0x45, 0xe8, 0x79, 0xfe, 0x6b, 0x30, 0x9b, 0x6e, 0xec, 0x59, + 0x14, 0xe1, 0x38, 0x61, 0xaf, 0x3f, 0xd5, 0xa0, 0x14, 0x28, 0x33, 0xb4, 0x0a, 0x05, 0xdf, 0xa3, + 0xee, 0xd9, 0xee, 0x20, 0x0a, 0x0d, 0xb8, 0xeb, 0x51, 0x17, 0x8b, 0xca, 0xe8, 0x3d, 0x28, 0xf5, + 0x89, 0xe7, 0x3d, 0x71, 0xdc, 0x96, 0x52, 0xb3, 0xa7, 0x04, 0x92, 0x67, 0xb3, 0xaa, 0x8a, 0x43, + 0x10, 0xfd, 0x07, 0xd3, 0x50, 0x79, 0x40, 0x98, 0x79, 0x40, 0x85, 0x3f, 0x7c, 0x31, 0x0e, 0xd1, + 0x9f, 0x68, 0x70, 0x2d, 0x19, 0xb9, 0xbe, 0x40, 0xaf, 0x68, 0xfe, 0xf8, 0x68, 0xf1, 0x1a, 0xce, + 0x94, 0x86, 0x87, 0xb4, 0x42, 0xf8, 0x47, 0x03, 0x81, 0xf0, 0x8b, 0xf6, 0x8f, 0x9a, 0xc3, 0x04, + 0xe2, 0xe1, 0x6d, 0x79, 0xe9, 0x1f, 0x8d, 0xe0, 0x1f, 0x5d, 0xf8, 0x9b, 0x84, 0xef, 0x66, 0xfb, + 0x47, 0x0f, 0x47, 0xb7, 0x80, 0xa2, 0x1d, 0xf9, 0xd2, 0x29, 0x7a, 0xe9, 0x14, 0xbd, 0x28, 0xa7, + 0xa8, 0x9f, 0x72, 0x8a, 0xc6, 0x49, 0x46, 0xa8, 0x2c, 0xbf, 0x44, 0x1b, 0xe6, 0x5c, 0x8d, 0xef, + 0xa6, 0xfc, 0x51, 0x0e, 0xae, 0x64, 0x68, 0x07, 0xf4, 0x2e, 0xcc, 0x7a, 0xcc, 0x71, 0x49, 0x87, + 0x46, 0x13, 0x2a, 0x0f, 0x34, 0x71, 0x03, 0xb4, 0x99, 0xa2, 0xe1, 0x01, 0x6e, 0xf4, 0x18, 0x80, + 0x18, 0x06, 0xf5, 0xbc, 0x2d, 0xa7, 0x15, 0xd8, 0x65, 0xef, 0x70, 0x77, 0x61, 0x25, 0x2c, 0x7d, + 0x76, 0xb4, 0xf8, 0xc5, 0xac, 0x84, 0x51, 0xd0, 0x1e, 0x26, 0xef, 0x7a, 0x47, 0x15, 0x70, 0x0c, + 0x12, 0xfd, 0x3a, 0x80, 0xbc, 0xfd, 0x1d, 0xde, 0x53, 0x7c, 0x4e, 0x54, 0xbf, 0x16, 0xdc, 0xae, + 0xae, 0xfd, 0xaa, 0x4f, 0x6c, 0xc6, 0x57, 0x85, 0xb8, 0xbb, 0xfb, 0x30, 0x44, 0xc1, 0x31, 0x44, + 0xfd, 0x1f, 0x72, 0x50, 0x0a, 0xec, 0xc5, 0x17, 0x90, 0xb7, 0xe9, 0x24, 0xf2, 0x36, 0xa3, 0x3f, + 0x42, 0x09, 0x9a, 0x3c, 0x34, 0x53, 0xe3, 0xa4, 0x32, 0x35, 0x1b, 0xe3, 0x8b, 0x3a, 0x39, 0x37, + 0xf3, 0x4c, 0x83, 0xcb, 0x01, 0xab, 0xba, 0x9f, 0xfc, 0x65, 0x98, 0x76, 0x29, 0x69, 0xd5, 0x09, + 0x33, 0xf6, 0xc5, 0xf4, 0xc9, 0xdb, 0xc9, 0x73, 0xc7, 0x47, 0x8b, 0xd3, 0x38, 0x4e, 0xc0, 0x49, + 0x3e, 0x54, 0x03, 0xf0, 0x5b, 0xed, 0x47, 0x8e, 0x2b, 0x9c, 0x2d, 0x79, 0x27, 0x59, 0x4c, 0xe2, + 0xee, 0xda, 0xba, 0x2a, 0xc5, 0x31, 0x8e, 0xac, 0x8b, 0xd0, 0xf9, 0x31, 0x2f, 0x42, 0x17, 0xce, + 0x74, 0x11, 0xfa, 0x9f, 0x35, 0x98, 0x8a, 0x3a, 0x7f, 0xe1, 0xa9, 0xa8, 0x76, 0x32, 0x15, 0xb5, + 0x32, 0xf6, 0xdc, 0x0e, 0x49, 0x3e, 0x7d, 0xbf, 0x18, 0x75, 0x4b, 0xa4, 0x9b, 0xf6, 0x60, 0xde, + 0xcc, 0x4c, 0xc1, 0xc4, 0x54, 0x47, 0x78, 0xaf, 0x6c, 0x73, 0x28, 0x27, 0x3e, 0x01, 0x05, 0xf9, + 0x50, 0x3a, 0xa0, 0x2e, 0x33, 0x0d, 0x1a, 0xf4, 0x6f, 0xe3, 0x9c, 0x9e, 0x2d, 0x46, 0x63, 0xfa, + 0x50, 0x09, 0xc0, 0xa1, 0x28, 0xb4, 0x07, 0x45, 0xda, 0xea, 0xd0, 0xe0, 0x1e, 0xf9, 0xdb, 0x63, + 0xdd, 0xe1, 0x8f, 0xc6, 0x93, 0x7f, 0x79, 0x58, 0x42, 0x23, 0x0f, 0xca, 0x56, 0xe0, 0x32, 0x2b, + 0x83, 0xb9, 0x3e, 0xb2, 0x9c, 0xd0, 0xf9, 0x8e, 0xee, 0x75, 0x86, 0x45, 0x38, 0x92, 0x83, 0xba, + 0xe1, 0xeb, 0x84, 0xe2, 0x39, 0x69, 0x82, 0x13, 0xde, 0xbe, 0x79, 0x50, 0x7e, 0x42, 0x18, 0x75, + 0x7b, 0xc4, 0xed, 0x2a, 0x83, 0x76, 0xf4, 0x1e, 0x3e, 0x0a, 0x90, 0xa2, 0x1e, 0x86, 0x45, 0x38, + 0x92, 0xa3, 0xff, 0x20, 0x17, 0xa9, 0x9e, 0x17, 0x9d, 0x0b, 0x7c, 0x2b, 0x99, 0x0b, 0x5c, 0x48, + 0xe7, 0x02, 0x53, 0xc1, 0x88, 0xb3, 0x67, 0x03, 0x09, 0x54, 0x2c, 0xe2, 0xb1, 0xdd, 0x7e, 0x8b, + 0x30, 0x15, 0x95, 0xab, 0x2c, 0xff, 0xdc, 0xe9, 0x94, 0xc9, 0x8e, 0xd9, 0xa3, 0x91, 0xbd, 0xda, + 0x88, 0x60, 0x70, 0x1c, 0x53, 0xff, 0x0f, 0x0d, 0xe6, 0x06, 0xf2, 0xbf, 0x68, 0x1f, 0x26, 0x6c, + 0x61, 0x61, 0x8f, 0xfd, 0xa8, 0x2e, 0x66, 0xa8, 0xcb, 0x45, 0xa3, 0x0a, 0x14, 0x3e, 0xb2, 0xa1, + 0x44, 0x9f, 0x32, 0xea, 0xda, 0xc4, 0x52, 0x07, 0xe3, 0xf9, 0x3c, 0xe0, 0x13, 0xf6, 0xd4, 0x1d, + 0x85, 0x8c, 0x43, 0x19, 0xfa, 0x4f, 0x72, 0x50, 0x89, 0xf1, 0x3d, 0x2f, 0x62, 0x2b, 0xee, 0x15, + 0x4a, 0x57, 0x73, 0xd7, 0xb5, 0xd4, 0x44, 0xc7, 0xee, 0x15, 0x2a, 0x12, 0x6e, 0xe0, 0x38, 0x1f, + 0x5a, 0x06, 0xe8, 0x11, 0x8f, 0x51, 0x57, 0xe8, 0xc6, 0xd4, 0x6d, 0xbe, 0xad, 0x90, 0x82, 0x63, + 0x5c, 0xe8, 0xa6, 0x0a, 0x7f, 0x14, 0x92, 0xaf, 0x61, 0x86, 0xc4, 0x36, 0x8a, 0xe7, 0x10, 0xdb, + 0x40, 0x1d, 0x98, 0x0d, 0x5a, 0x1d, 0x50, 0xd5, 0xc6, 0x3d, 0x25, 0xb0, 0x34, 0x15, 0x53, 0x10, + 0x78, 0x00, 0x54, 0xff, 0x6b, 0x0d, 0xa6, 0x13, 0xf6, 0x2e, 0x7a, 0x2d, 0x7e, 0x79, 0x21, 0x16, + 0x29, 0x4d, 0x5c, 0x3a, 0x78, 0x03, 0x26, 0xe4, 0x00, 0xa9, 0x81, 0x0f, 0x37, 0xa2, 0x1c, 0x42, + 0xac, 0xa8, 0x7c, 0x4b, 0xa9, 0x50, 0x4a, 0x7a, 0x4b, 0xa9, 0x58, 0x0b, 0x0e, 0xe8, 0xe8, 0x0b, + 0xdc, 0x82, 0x97, 0xad, 0x53, 0x23, 0x1d, 0x1e, 0x0c, 0x41, 0x3f, 0x70, 0xc8, 0xa1, 0x7f, 0x0b, + 0xe4, 0xcb, 0x5b, 0x74, 0x1d, 0xf2, 0x3d, 0xd3, 0x56, 0xe1, 0x4c, 0x11, 0x34, 0xdd, 0x32, 0x6d, + 0xcc, 0xcb, 0x04, 0x89, 0x3c, 0x55, 0x01, 0x1e, 0x49, 0x22, 0x4f, 0x31, 0x2f, 0xe3, 0xb6, 0x89, + 0xe5, 0x38, 0xdd, 0x3d, 0x62, 0x74, 0x83, 0x80, 0xa8, 0xbc, 0x3e, 0x2b, 0x6c, 0x93, 0x46, 0x92, + 0x84, 0xd3, 0xbc, 0xe2, 0x79, 0x9c, 0x08, 0xad, 0x3f, 0x82, 0xbc, 0xe5, 0x74, 0xd4, 0x5e, 0x1c, + 0x3d, 0xe0, 0xdb, 0x70, 0x3a, 0xb2, 0x81, 0x0d, 0xa7, 0x83, 0x39, 0x22, 0x32, 0xa0, 0xd8, 0x25, + 0xed, 0x2e, 0x51, 0x5b, 0x6f, 0x74, 0x75, 0x1d, 0xa6, 0x01, 0xd4, 0x9b, 0x2f, 0xfe, 0x89, 0x25, + 0x36, 0x32, 0x60, 0xc2, 0x6f, 0x89, 0xff, 0x91, 0x18, 0xf7, 0xaf, 0x2a, 0x76, 0xd7, 0x84, 0x08, + 0xa1, 0x47, 0xe4, 0x6f, 0xac, 0xa0, 0xf5, 0x7f, 0xcc, 0x81, 0xfa, 0xf7, 0x0a, 0xe4, 0x43, 0xb9, + 0x13, 0xbc, 0xb1, 0x52, 0x63, 0x76, 0x77, 0x8c, 0x6b, 0xbd, 0x89, 0xd7, 0x5a, 0x32, 0xb3, 0x15, + 0x16, 0xe2, 0x48, 0x12, 0xa2, 0xc9, 0xb1, 0x5c, 0x1b, 0x73, 0x2c, 0xa5, 0xb8, 0xc1, 0xd1, 0x24, + 0x50, 0xd8, 0x67, 0xac, 0xaf, 0xc6, 0x72, 0xf4, 0xe7, 0x86, 0xd1, 0xf5, 0x3a, 0x19, 0x43, 0xe5, + 0xdf, 0x58, 0x40, 0xeb, 0x3d, 0x50, 0x27, 0x1d, 0x32, 0x12, 0x2f, 0x2b, 0x65, 0x6c, 0x7c, 0xe9, + 0x74, 0xe7, 0x4f, 0xf8, 0x2a, 0x30, 0xf6, 0xe0, 0x22, 0xf3, 0x09, 0xa5, 0xfe, 0xaf, 0x39, 0xc8, + 0xef, 0x34, 0x9a, 0xf2, 0xfe, 0xb0, 0x08, 0x74, 0xd0, 0x66, 0xd7, 0xec, 0x3f, 0xa4, 0xae, 0xd9, + 0x96, 0x4e, 0x6e, 0x29, 0x7e, 0x7f, 0x38, 0xcd, 0x81, 0x33, 0x6a, 0xa1, 0x0f, 0x60, 0xca, 0x20, + 0xab, 0xd4, 0x65, 0x52, 0x67, 0x9d, 0x2d, 0x14, 0x2c, 0xb2, 0x91, 0xab, 0x2b, 0x51, 0x75, 0x9c, + 0x00, 0x43, 0xbb, 0x00, 0x46, 0x04, 0x9d, 0x3f, 0x0b, 0xb4, 0x7c, 0x4a, 0x1a, 0x01, 0xc7, 0x80, + 0x10, 0x86, 0x72, 0x97, 0xb3, 0x0a, 0xd4, 0xc2, 0x59, 0x50, 0xc5, 0xa2, 0xbc, 0x1f, 0xd4, 0xc5, + 0x11, 0x8c, 0xfe, 0x23, 0x0d, 0xf2, 0xbb, 0x6b, 0xeb, 0xc8, 0x81, 0x72, 0x78, 0xe9, 0x45, 0xed, + 0x89, 0xfa, 0xf8, 0x51, 0x50, 0x29, 0x38, 0xfc, 0xc4, 0x91, 0x0c, 0xb4, 0x0f, 0x93, 0x7b, 0xbe, + 0x69, 0x31, 0xd3, 0x16, 0x61, 0xa2, 0x71, 0x1c, 0x95, 0xe0, 0x71, 0xa6, 0xca, 0xb9, 0x49, 0x54, + 0x1c, 0xc0, 0xeb, 0xdf, 0x06, 0xa5, 0x0b, 0xb8, 0x01, 0x7a, 0x11, 0x9d, 0x0c, 0x0d, 0xd0, 0xac, + 0x8e, 0xea, 0x7f, 0x9f, 0x83, 0x09, 0xf5, 0xaf, 0x3d, 0x17, 0x1f, 0x42, 0xa0, 0x89, 0x10, 0xc2, + 0xea, 0x98, 0xff, 0x63, 0x31, 0x34, 0x80, 0xd0, 0x4b, 0x05, 0x10, 0xc6, 0xfd, 0xc3, 0x8c, 0xe7, + 0x84, 0x0f, 0xfe, 0x49, 0x83, 0xcb, 0x92, 0x71, 0xd3, 0xf6, 0x18, 0xb1, 0x0d, 0xf1, 0xf7, 0x46, + 0x07, 0xa2, 0x64, 0x6c, 0x1b, 0x5e, 0xb9, 0x7f, 0xe2, 0xcc, 0x90, 0xbf, 0xb1, 0x82, 0xe6, 0xb6, + 0xc0, 0xbe, 0xe3, 0x31, 0xf1, 0x06, 0x39, 0x97, 0xb4, 0x05, 0xee, 0xaa, 0x72, 0x1c, 0x72, 0xa4, + 0x8d, 0x8c, 0xe2, 0x70, 0x23, 0x43, 0x7f, 0x02, 0x53, 0xf1, 0x7f, 0x0a, 0x79, 0x61, 0xc1, 0x10, + 0xfd, 0x53, 0x0d, 0x20, 0x90, 0x7c, 0xe1, 0x91, 0x88, 0x56, 0x32, 0x12, 0x31, 0xf6, 0x14, 0x65, + 0xc7, 0x21, 0xfe, 0xa6, 0x10, 0x74, 0x49, 0x44, 0x21, 0x3e, 0xd6, 0xe0, 0x32, 0x49, 0x78, 0xf6, + 0xaa, 0x67, 0xe7, 0x16, 0x28, 0xb8, 0xa6, 0x9a, 0x91, 0xfa, 0x4f, 0x2e, 0x9c, 0x12, 0x8b, 0x6e, + 0xc3, 0x54, 0x5f, 0xf9, 0x77, 0x0f, 0xa2, 0x15, 0x14, 0xa6, 0x1e, 0xb6, 0x63, 0x34, 0x9c, 0xe0, + 0x7c, 0x4e, 0x24, 0x25, 0x7f, 0x2e, 0x91, 0x94, 0x78, 0xee, 0xb1, 0x70, 0x62, 0xee, 0xd1, 0x86, + 0x72, 0xdb, 0x75, 0x7a, 0x22, 0x58, 0xa1, 0xfe, 0xab, 0x63, 0xcc, 0x00, 0x48, 0xa8, 0x30, 0xd7, + 0x03, 0x5c, 0x1c, 0x89, 0xe0, 0x27, 0x03, 0x73, 0xa4, 0xb4, 0x89, 0xf3, 0x90, 0x16, 0x6e, 0xc3, + 0x1d, 0x89, 0x8a, 0x03, 0x78, 0xfd, 0x5f, 0x72, 0xc1, 0x3e, 0x6c, 0xa6, 0x2e, 0xef, 0x6a, 0x43, + 0x2e, 0xef, 0xaa, 0xf7, 0x17, 0x71, 0x77, 0xfd, 0x0d, 0x98, 0x70, 0x29, 0xf1, 0x1c, 0x5b, 0xbd, + 0x64, 0x0a, 0x95, 0x18, 0x16, 0xa5, 0x58, 0x51, 0xe3, 0x6e, 0x7d, 0xee, 0x39, 0x6e, 0xfd, 0x17, + 0x62, 0x73, 0x93, 0x17, 0x7b, 0x3a, 0xdc, 0x66, 0x19, 0xf3, 0x23, 0x3c, 0x16, 0x95, 0x36, 0x2b, + 0xa6, 0x3d, 0x16, 0x95, 0xda, 0x0a, 0x39, 0x50, 0x0b, 0xa6, 0xb8, 0x7b, 0x2f, 0xbc, 0x96, 0xd6, + 0x0a, 0x1b, 0x21, 0x66, 0x10, 0xae, 0xe0, 0x46, 0x0c, 0x07, 0x27, 0x50, 0xf5, 0xaf, 0x42, 0x14, + 0x8d, 0x41, 0x4b, 0x50, 0xee, 0xbb, 0x4e, 0x9f, 0x74, 0x08, 0xa3, 0xca, 0x5c, 0x0b, 0x57, 0xc0, + 0x76, 0x40, 0xc0, 0x11, 0x4f, 0xbd, 0xf6, 0xc9, 0xe7, 0x0b, 0x97, 0x3e, 0xfd, 0x7c, 0xe1, 0xd2, + 0x67, 0x9f, 0x2f, 0x5c, 0xfa, 0xad, 0xe3, 0x05, 0xed, 0x93, 0xe3, 0x05, 0xed, 0xd3, 0xe3, 0x05, + 0xed, 0xb3, 0xe3, 0x05, 0xed, 0x47, 0xc7, 0x0b, 0xda, 0x77, 0xff, 0x6d, 0xe1, 0xd2, 0xd7, 0x4b, + 0xc1, 0x34, 0xff, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x21, 0x4d, 0xa6, 0x88, 0xcb, 0x50, 0x00, + 0x00, } func (m *AbstractVertex) Marshal() (dAtA []byte, err error) { @@ -4555,6 +4589,11 @@ func (m *Scale) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LookbackSeconds != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.LookbackSeconds)) + i-- + dAtA[i] = 0x18 + } if m.Max != nil { i = encodeVarintGenerated(dAtA, i, uint64(*m.Max)) i-- @@ -4923,6 +4962,49 @@ func (m *Vertex) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VertexInstance) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VertexInstance) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VertexInstance) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i = encodeVarintGenerated(dAtA, i, uint64(m.Replica)) + i-- + dAtA[i] = 0x18 + i -= len(m.Hostname) + copy(dAtA[i:], m.Hostname) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Hostname))) + i-- + dAtA[i] = 0x12 + if m.Vertex != nil { + { + size, err := m.Vertex.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *VertexLimits) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6175,6 +6257,9 @@ func (m *Scale) Size() (n int) { if m.Max != nil { n += 1 + sovGenerated(uint64(*m.Max)) } + if m.LookbackSeconds != nil { + n += 1 + sovGenerated(uint64(*m.LookbackSeconds)) + } return n } @@ -6300,6 +6385,22 @@ func (m *Vertex) Size() (n int) { return n } +func (m *VertexInstance) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Vertex != nil { + l = m.Vertex.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + l = len(m.Hostname) + n += 1 + l + sovGenerated(uint64(l)) + n += 1 + sovGenerated(uint64(m.Replica)) + return n +} + func (m *VertexLimits) Size() (n int) { if m == nil { return 0 @@ -7137,6 +7238,7 @@ func (this *Scale) String() string { s := strings.Join([]string{`&Scale{`, `Min:` + valueToStringGenerated(this.Min) + `,`, `Max:` + valueToStringGenerated(this.Max) + `,`, + `LookbackSeconds:` + valueToStringGenerated(this.LookbackSeconds) + `,`, `}`, }, "") return s @@ -7226,6 +7328,18 @@ func (this *Vertex) String() string { }, "") return s } +func (this *VertexInstance) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VertexInstance{`, + `Vertex:` + strings.Replace(this.Vertex.String(), "Vertex", "Vertex", 1) + `,`, + `Hostname:` + fmt.Sprintf("%v", this.Hostname) + `,`, + `Replica:` + fmt.Sprintf("%v", this.Replica) + `,`, + `}`, + }, "") + return s +} func (this *VertexLimits) String() string { if this == nil { return "nil" @@ -16305,6 +16419,26 @@ func (m *Scale) Unmarshal(dAtA []byte) error { } } m.Max = &v + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LookbackSeconds", wireType) + } + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.LookbackSeconds = &v default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -17258,6 +17392,143 @@ func (m *Vertex) Unmarshal(dAtA []byte) error { } return nil } +func (m *VertexInstance) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VertexInstance: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VertexInstance: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Vertex", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Vertex == nil { + m.Vertex = &Vertex{} + } + if err := m.Vertex.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hostname", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hostname = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Replica", wireType) + } + m.Replica = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Replica |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *VertexLimits) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/apis/numaflow/v1alpha1/generated.proto b/pkg/apis/numaflow/v1alpha1/generated.proto index f6df41b29..e535a8ad1 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.proto +++ b/pkg/apis/numaflow/v1alpha1/generated.proto @@ -807,9 +807,14 @@ message Scale { optional int32 min = 1; // Maximum replicas - // +kubebuilder:default=1 + // +kubebuilder:default=100 // +optional optional int32 max = 2; + + // Lookback seconds to calculate the average pending messages and processing rate + // +kubebuilder:default=180 + // +optional + optional int32 lookbackSeconds = 3; } message Sink { @@ -890,6 +895,15 @@ message Vertex { optional VertexStatus status = 3; } +// VertexInstance is a wrapper of a vertex instance, which contains the vertex spec and the instance information such as hostname and replica index. +message VertexInstance { + optional Vertex vertex = 1; + + optional string hostname = 2; + + optional int32 replica = 3; +} + message VertexLimits { // Read batch size // +optional diff --git a/pkg/apis/numaflow/v1alpha1/vertex_instance.go b/pkg/apis/numaflow/v1alpha1/vertex_instance.go new file mode 100644 index 000000000..1ed1f4321 --- /dev/null +++ b/pkg/apis/numaflow/v1alpha1/vertex_instance.go @@ -0,0 +1,8 @@ +package v1alpha1 + +// VertexInstance is a wrapper of a vertex instance, which contains the vertex spec and the instance information such as hostname and replica index. +type VertexInstance struct { + Vertex *Vertex `json:"vertex,omitempty" protobuf:"bytes,1,opt,name=vertex"` + Hostname string `json:"hostname,omitempty" protobuf:"bytes,2,opt,name=hostname"` + Replica int32 `json:"replica,omitempty" protobuf:"varint,3,opt,name=replica"` +} diff --git a/pkg/apis/numaflow/v1alpha1/vertex_types.go b/pkg/apis/numaflow/v1alpha1/vertex_types.go index 7ef27cb4b..5ceb1bac0 100644 --- a/pkg/apis/numaflow/v1alpha1/vertex_types.go +++ b/pkg/apis/numaflow/v1alpha1/vertex_types.go @@ -371,9 +371,13 @@ type Scale struct { // +optional Min *int32 `json:"min,omitempty" protobuf:"varint,1,opt,name=min"` // Maximum replicas - // +kubebuilder:default=1 + // +kubebuilder:default=100 // +optional Max *int32 `json:"max,omitempty" protobuf:"varint,2,opt,name=max"` + // Lookback seconds to calculate the average pending messages and processing rate + // +kubebuilder:default=180 + // +optional + LookbackSeconds *int32 `json:"lookbackSeconds,omitempty" protobuf:"varint,3,opt,name=lookbackSeconds"` } type VertexLimits struct { diff --git a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go index 6c2677e62..08db1584a 100644 --- a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go @@ -1249,6 +1249,11 @@ func (in *Scale) DeepCopyInto(out *Scale) { *out = new(int32) **out = **in } + if in.LookbackSeconds != nil { + in, out := &in.LookbackSeconds, &out.LookbackSeconds + *out = new(int32) + **out = **in + } return } @@ -1449,6 +1454,27 @@ func (in *Vertex) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VertexInstance) DeepCopyInto(out *VertexInstance) { + *out = *in + if in.Vertex != nil { + in, out := &in.Vertex, &out.Vertex + *out = new(Vertex) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VertexInstance. +func (in *VertexInstance) DeepCopy() *VertexInstance { + if in == nil { + return nil + } + out := new(VertexInstance) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VertexLimits) DeepCopyInto(out *VertexLimits) { *out = *in diff --git a/pkg/isb/interfaces.go b/pkg/isb/interfaces.go index 1b2f7dcaf..9d34434ec 100644 --- a/pkg/isb/interfaces.go +++ b/pkg/isb/interfaces.go @@ -9,12 +9,13 @@ package isb import ( "context" "io" + "math" "strconv" ) const ( - PendingNotAvailable = int64(-1) - RateNotAvailable = float64(-1) + PendingNotAvailable = int64(math.MinInt64) + RateNotAvailable = float64(math.MinInt) ) // Ratable is the interface that wraps the Rate method. @@ -23,6 +24,12 @@ type Ratable interface { Rate(context.Context) (float64, error) } +// LagReader is the interface that wraps the Pending method. +type LagReader interface { + // Pending returns the pending messages number. + Pending(context.Context) (int64, error) +} + // BufferWriter is the buffer to which we are writing. type BufferWriter interface { BufferWriterInformation @@ -42,8 +49,6 @@ type BufferReader interface { Read(context.Context, int64) ([]*ReadMessage, error) // Ack acknowledges an array of offset. Ack(context.Context, []Offset) []error - // Pending returns the pending messages number. - Pending(context.Context) (int64, error) } // BufferReaderInformation has information regarding the buffer we are reading from. diff --git a/pkg/isb/jetstream/options.go b/pkg/isb/jetstream/options.go index 04c4b6e06..9e975243a 100644 --- a/pkg/isb/jetstream/options.go +++ b/pkg/isb/jetstream/options.go @@ -57,9 +57,16 @@ func WithRefreshInterval(refreshInterval time.Duration) WriteOption { } // WithUsingWriteInfoAsRate sets whether to check sequence for rate calculation -func WithUsingWriteInfoAsRate(check bool) WriteOption { +func WithUsingWriteInfoAsRate(yes bool) WriteOption { return func(o *writeOptions) error { - o.useWriteInfoAsRate = check + o.useWriteInfoAsRate = yes + return nil + } +} + +func WithWriteRateLookbackSeconds(seconds int64) WriteOption { + return func(o *writeOptions) error { + o.rateLookbackSeconds = seconds return nil } } @@ -102,7 +109,7 @@ func WithAckInfoCheckInterval(t time.Duration) ReadOption { } } -func WithRateLookbackSeconds(seconds int64) ReadOption { +func WithAckRateLookbackSeconds(seconds int64) ReadOption { return func(o *readOptions) error { o.rateLookbackSeconds = seconds return nil diff --git a/pkg/isb/jetstream/reader.go b/pkg/isb/jetstream/reader.go index 8933d7449..2d9507cf5 100644 --- a/pkg/isb/jetstream/reader.go +++ b/pkg/isb/jetstream/reader.go @@ -159,8 +159,8 @@ func (jr *jetStreamReader) Rate(_ context.Context) (float64, error) { endSeqInfo := timestampedSeqs[len(timestampedSeqs)-1] startSeqInfo := timestampedSeqs[len(timestampedSeqs)-2] for i := len(timestampedSeqs) - 3; i >= 0; i-- { + startSeqInfo = timestampedSeqs[i] if endSeqInfo.timestamp-timestampedSeqs[i].timestamp > jr.opts.rateLookbackSeconds { - startSeqInfo = timestampedSeqs[i] break } } diff --git a/pkg/isb/jetstream/writer.go b/pkg/isb/jetstream/writer.go index 6f3fed3c2..7ce3c39a1 100644 --- a/pkg/isb/jetstream/writer.go +++ b/pkg/isb/jetstream/writer.go @@ -148,7 +148,7 @@ func (jw *jetStreamWriter) Close() error { // Rate returns the writting rate (tps) func (jw *jetStreamWriter) Rate(_ context.Context) (float64, error) { - if jw.opts.useWriteInfoAsRate { + if !jw.opts.useWriteInfoAsRate { return isb.RateNotAvailable, nil } timestampedSeqs := jw.writtenInfo.Items() @@ -156,10 +156,10 @@ func (jw *jetStreamWriter) Rate(_ context.Context) (float64, error) { return isb.RateNotAvailable, nil } endSeqInfo := timestampedSeqs[len(timestampedSeqs)-1] - startSeqInfo := timestampedSeqs[len(timestampedSeqs)-1] + startSeqInfo := timestampedSeqs[len(timestampedSeqs)-2] for i := len(timestampedSeqs) - 3; i >= 0; i-- { + startSeqInfo = timestampedSeqs[i] if endSeqInfo.timestamp-timestampedSeqs[i].timestamp > jw.opts.rateLookbackSeconds { - startSeqInfo = timestampedSeqs[i] break } } diff --git a/pkg/isb/redis/read.go b/pkg/isb/redis/read.go index cf8ac38a8..b6502785e 100644 --- a/pkg/isb/redis/read.go +++ b/pkg/isb/redis/read.go @@ -303,7 +303,6 @@ func (br *BufferRead) convertXStreamToMessages(xstreams []redis.XStream, message } messages = append(messages, &readMessage) } - } } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 89e633e31..beee16cfd 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -7,17 +7,146 @@ import ( "net/http" "net/http/pprof" "os" + "time" dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" + "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/shared/logging" + sharedqueue "github.com/numaproj/numaflow/pkg/shared/queue" sharedtls "github.com/numaproj/numaflow/pkg/shared/tls" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" ) -func StartMetricsServer(ctx context.Context) (func(ctx context.Context) error, error) { +var ( + processingRate = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "vertex_processing_rate", + Help: "Message processing rate, tps. It represents the rate of a vertex instead of a pod.", + }, []string{"vertex", "pipeline"}) + + pendingMessages = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "vertex_pending_messages", + Help: "Pending messages. It meants the pending messages of a vertex, not a pod.", + }, []string{"vertex", "pipeline"}) +) + +// timestampedPending is a helper struct to wrap a pending number and timestamp pair +type timestampedPending struct { + pending int64 + // timestamp in seconds + timestamp int64 +} + +type metricsServer struct { + vertex *dfv1.Vertex + rater isb.Ratable + lagReader isb.LagReader + // pendingLookbackSeconds is the look back seconds for pending calculation + pendingLookbackSeconds int64 + refreshInterval time.Duration + // pendingInfo stores a list of pending/timestamp(seconds) information + pendingInfo *sharedqueue.OverflowQueue[timestampedPending] +} + +type Option func(*metricsServer) + +func WithRater(r isb.Ratable) Option { + return func(m *metricsServer) { + m.rater = r + } +} + +func WithLagReader(r isb.LagReader) Option { + return func(m *metricsServer) { + m.lagReader = r + } +} + +func WithRefreshInterval(d time.Duration) Option { + return func(m *metricsServer) { + m.refreshInterval = d + } +} + +func WithPendingLookbackSeconds(seconds int64) Option { + return func(m *metricsServer) { + m.pendingLookbackSeconds = seconds + } +} + +// NewMetricsServer returns a Prometheus metrics server instance, which can be used to start an HTTPS service to expose Prometheus metrics. +func NewMetricsServer(vertex *dfv1.Vertex, opts ...Option) *metricsServer { + m := new(metricsServer) + m.vertex = vertex + m.refreshInterval = 5 * time.Second // Default refersh interval + m.pendingLookbackSeconds = 180 // Default + for _, opt := range opts { + if opt != nil { + opt(m) + } + } + if m.lagReader != nil { + m.pendingInfo = sharedqueue.New[timestampedPending](1800) + } + return m +} + +func (ms *metricsServer) rateAndPending(ctx context.Context) { + if ms.lagReader == nil && ms.rater == nil { + return + } + labels := map[string]string{"vertex": ms.vertex.Spec.Name, "pipeline": ms.vertex.Spec.PipelineName} + log := logging.FromContext(ctx) + ticker := time.NewTicker(ms.refreshInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if ms.rater != nil { + if r, err := ms.rater.Rate(ctx); err != nil { + log.Errorw("failed to get processing rate", zap.Error(err)) + } else { + if r != isb.RateNotAvailable { + processingRate.With(labels).Set(r) + } + } + } + if ms.lagReader != nil { + if pending, err := ms.lagReader.Pending(ctx); err != nil { + log.Errorw("failed to get pending messages", zap.Error(err)) + } else { + if pending != isb.PendingNotAvailable { + now := time.Now().Unix() + ts := timestampedPending{pending: pending, timestamp: now} + ms.pendingInfo.Append(ts) + // Calculate avg pending + items := ms.pendingInfo.Items() + total := pending + num := int64(1) + for i := len(items) - 2; i >= 0; i-- { + if now-items[i].timestamp < ms.pendingLookbackSeconds { + total += items[i].pending + num++ + } else { + break + } + } + pendingMessages.With(labels).Set(float64(total / num)) + } + } + } + case <-ctx.Done(): + return + } + } +} + +// Start function starts the HTTPS service to expose metrics, it returns a shutdown function and an error if any +func (ms *metricsServer) Start(ctx context.Context) (func(ctx context.Context) error, error) { log := logging.FromContext(ctx) - log.Info("generating self-signed certificate") + log.Info("Generating self-signed certificate") cer, err := sharedtls.GenerateX509KeyPair() if err != nil { return nil, fmt.Errorf("failed to generate cert: %w", err) @@ -46,6 +175,7 @@ func StartMetricsServer(ctx context.Context) (func(ctx context.Context) error, e Handler: mux, TLSConfig: &tls.Config{Certificates: []tls.Certificate{*cer}, MinVersion: tls.VersionTLS12}, } + go ms.rateAndPending(ctx) go func() { log.Info("Starting metrics HTTPS server") if err := httpServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 12f75cea0..e302d5741 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -15,7 +15,8 @@ import ( func Test_StartMetricsServer(t *testing.T) { t.SkipNow() // flaky - s, err := StartMetricsServer(context.TODO()) + ms := NewMetricsServer(&dfv1.Vertex{}) + s, err := ms.Start(context.TODO()) assert.NoError(t, err) assert.NotNil(t, s) e := httpexpect.WithConfig(httpexpect.Config{ diff --git a/pkg/sinks/sink.go b/pkg/sinks/sink.go index b0106a199..ae6dfd4e9 100644 --- a/pkg/sinks/sink.go +++ b/pkg/sinks/sink.go @@ -21,10 +21,8 @@ import ( ) type SinkProcessor struct { - ISBSvcType dfv1.ISBSvcType - Vertex *dfv1.Vertex - Hostname string - Replica int + ISBSvcType dfv1.ISBSvcType + VertexInstance *dfv1.VertexInstance } func (u *SinkProcessor) Start(ctx context.Context) error { @@ -33,17 +31,23 @@ func (u *SinkProcessor) Start(ctx context.Context) error { defer cancel() var reader isb.BufferReader var err error - fromBufferName := u.Vertex.GetFromBuffers()[0].Name + fromBufferName := u.VertexInstance.Vertex.GetFromBuffers()[0].Name switch u.ISBSvcType { case dfv1.ISBSvcTypeRedis: redisClient := clients.NewInClusterRedisClient() fromGroup := fromBufferName + "-group" - consumer := fmt.Sprintf("%s-%v", u.Vertex.Name, u.Replica) + consumer := fmt.Sprintf("%s-%v", u.VertexInstance.Vertex.Name, u.VertexInstance.Replica) reader = redisisb.NewBufferRead(ctx, redisClient, fromBufferName, fromGroup, consumer) case dfv1.ISBSvcTypeJetStream: - streamName := fmt.Sprintf("%s-%s", u.Vertex.Spec.PipelineName, fromBufferName) + streamName := fmt.Sprintf("%s-%s", u.VertexInstance.Vertex.Spec.PipelineName, fromBufferName) + readOptions := []jetstreamisb.ReadOption{ + jetstreamisb.WithUsingAckInfoAsRate(true), + } + if x := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; x != nil { + readOptions = append(readOptions, jetstreamisb.WithAckRateLookbackSeconds(int64(*x))) + } jetStreamClient := clients.NewInClusterJetStreamClient() - reader, err = jetstreamisb.NewJetStreamBufferReader(ctx, jetStreamClient, fromBufferName, streamName, streamName) + reader, err = jetstreamisb.NewJetStreamBufferReader(ctx, jetStreamClient, fromBufferName, streamName, streamName, readOptions...) if err != nil { return err } @@ -69,7 +73,18 @@ func (u *SinkProcessor) Start(ctx context.Context) error { } }() - if shutdown, err := metrics.StartMetricsServer(ctx); err != nil { + metricsOpts := []metrics.Option{} + if x, ok := reader.(isb.LagReader); ok { + metricsOpts = append(metricsOpts, metrics.WithLagReader(x)) + if s := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; s != nil { + metricsOpts = append(metricsOpts, metrics.WithPendingLookbackSeconds(int64(*s))) + } + } + if x, ok := reader.(isb.Ratable); ok { + metricsOpts = append(metricsOpts, metrics.WithRater(x)) + } + ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) + if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) } else { defer func() { _ = shutdown(context.Background()) }() @@ -85,14 +100,14 @@ func (u *SinkProcessor) Start(ctx context.Context) error { // getSinker takes in the logger from the parent context func (u *SinkProcessor) getSinker(reader isb.BufferReader, logger *zap.SugaredLogger) (Sinker, error) { - sink := u.Vertex.Spec.Sink + sink := u.VertexInstance.Vertex.Spec.Sink // TODO: add watermark if x := sink.Log; x != nil { - return logsink.NewToLog(u.Vertex, reader, logsink.WithLogger(logger)) + return logsink.NewToLog(u.VertexInstance.Vertex, reader, logsink.WithLogger(logger)) } else if x := sink.Kafka; x != nil { - return kafkasink.NewToKafka(u.Vertex, reader, kafkasink.WithLogger(logger)) + return kafkasink.NewToKafka(u.VertexInstance.Vertex, reader, kafkasink.WithLogger(logger)) } else if x := sink.UDSink; x != nil { - return udsink.NewUserDefinedSink(u.Vertex, reader, udsink.WithLogger(logger)) + return udsink.NewUserDefinedSink(u.VertexInstance.Vertex, reader, udsink.WithLogger(logger)) } return nil, fmt.Errorf("invalid sink spec") } diff --git a/pkg/sources/generator/tickgen.go b/pkg/sources/generator/tickgen.go index a5e05b847..9ef0b7f3b 100644 --- a/pkg/sources/generator/tickgen.go +++ b/pkg/sources/generator/tickgen.go @@ -15,7 +15,6 @@ import ( "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/isb/forward" "github.com/numaproj/numaflow/pkg/shared/logging" - "github.com/numaproj/numaflow/pkg/sources/types" "github.com/numaproj/numaflow/pkg/udf/applier" "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/progress" @@ -115,13 +114,13 @@ func WithReadTimeOut(timeout time.Duration) Option { // msgSize - size of each generated message // timeunit - unit of time per tick. could be any golang time.Duration. // writers - destinations to write to -func NewMemGen(metadata *types.SourceMetadata, rpu int, msgSize int32, timeunit time.Duration, writers []isb.BufferWriter, opts ...Option) (*memgen, error) { +func NewMemGen(vertexInstance *dfv1.VertexInstance, rpu int, msgSize int32, timeunit time.Duration, writers []isb.BufferWriter, opts ...Option) (*memgen, error) { gensrc := &memgen{ rpu: rpu, msgSize: msgSize, timeunit: timeunit, - name: metadata.Vertex.Spec.Name, - pipelineName: metadata.Vertex.Spec.PipelineName, + name: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, genfn: recordGenerator, progressor: &watermark{ sourcePublish: nil, @@ -152,7 +151,7 @@ func NewMemGen(metadata *types.SourceMetadata, rpu int, msgSize int32, timeunit } forwardOpts := []forward.Option{forward.WithLogger(gensrc.logger)} - if x := metadata.Vertex.Spec.Limits; x != nil { + if x := vertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { forwardOpts = append(forwardOpts, forward.WithReadBatchSize(int64(*x.ReadBatchSize))) } @@ -162,7 +161,7 @@ func NewMemGen(metadata *types.SourceMetadata, rpu int, msgSize int32, timeunit var err error // TODO: pass it as option if val, ok := os.LookupEnv(dfv1.EnvWatermarkOn); ok && val == "true" { - err = gensrc.buildWMProgressor(metadata) + err = gensrc.buildWMProgressor(vertexInstance) if err != nil { return nil, err } @@ -170,7 +169,7 @@ func NewMemGen(metadata *types.SourceMetadata, rpu int, msgSize int32, timeunit } // we pass in the context to forwarder as well so that it can shut down when we cancel the context - forwarder, err := forward.NewInterStepDataForward(metadata.Vertex, gensrc, destinations, forward.All, applier.Terminal, wmProgressor, forwardOpts...) + forwarder, err := forward.NewInterStepDataForward(vertexInstance.Vertex, gensrc, destinations, forward.All, applier.Terminal, wmProgressor, forwardOpts...) if err != nil { return nil, err } @@ -252,10 +251,6 @@ func (mg *memgen) Start() <-chan struct{} { return mg.forwarder.Start() } -func (mg *memgen) Pending(_ context.Context) (int64, error) { - return isb.PendingNotAvailable, nil -} - // generator fires once per time unit and generates records and writes them to the channel func (mg *memgen) generator(ctx context.Context, rate int, timeunit time.Duration) { go func() { diff --git a/pkg/sources/generator/tickgen_test.go b/pkg/sources/generator/tickgen_test.go index 908dd84be..9d89a292e 100644 --- a/pkg/sources/generator/tickgen_test.go +++ b/pkg/sources/generator/tickgen_test.go @@ -6,7 +6,6 @@ import ( "time" dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" - "github.com/numaproj/numaflow/pkg/sources/types" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/numaproj/numaflow/pkg/isb" @@ -20,7 +19,7 @@ func TestRead(t *testing.T) { vertex := &dfv1.Vertex{ObjectMeta: v1.ObjectMeta{ Name: "memgen", }} - m := &types.SourceMetadata{ + m := &dfv1.VertexInstance{ Vertex: vertex, Hostname: "TestRead", Replica: 0, @@ -48,7 +47,7 @@ func TestStop(t *testing.T) { vertex := &dfv1.Vertex{ObjectMeta: v1.ObjectMeta{ Name: "memgen", }} - m := &types.SourceMetadata{ + m := &dfv1.VertexInstance{ Vertex: vertex, Hostname: "TestRead", Replica: 0, diff --git a/pkg/sources/generator/watermark.go b/pkg/sources/generator/watermark.go index 1c2c12272..79194d1c1 100644 --- a/pkg/sources/generator/watermark.go +++ b/pkg/sources/generator/watermark.go @@ -3,8 +3,8 @@ package generator import ( "fmt" + dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" "github.com/numaproj/numaflow/pkg/isb" - "github.com/numaproj/numaflow/pkg/sources/types" "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/progress" "github.com/numaproj/numaflow/pkg/watermark/publish" @@ -12,7 +12,7 @@ import ( // buildWMProgressor builds the watermark progressor. It will create a sourcePublisher so it can publish the watermark // it reads from the source. The sourcePublisher is passed in as the input for generic progressor. -func (mg *memgen) buildWMProgressor(metadata *types.SourceMetadata) error { +func (mg *memgen) buildWMProgressor(vertexInstance *dfv1.VertexInstance) error { ctx := mg.lifecycleCtx js, err := progress.GetJetStreamConnection(mg.lifecycleCtx) @@ -21,30 +21,30 @@ func (mg *memgen) buildWMProgressor(metadata *types.SourceMetadata) error { } // publish source watermark and this is very much dependent on the source - sourcePublishKeySpace := fmt.Sprintf("source-%s", progress.GetPublishKeySpace(metadata.Vertex)) + sourcePublishKeySpace := fmt.Sprintf("source-%s", progress.GetPublishKeySpace(vertexInstance.Vertex)) // TODO: remove this once bucket creation has been moved to controller err = progress.CreateProcessorBucketIfMissing(fmt.Sprintf("%s_PROCESSORS", sourcePublishKeySpace), js) if err != nil { return err } - publishEntity := processor.NewProcessorEntity(fmt.Sprintf("source-%s-%d", metadata.Vertex.Name, metadata.Replica), sourcePublishKeySpace, processor.WithSeparateOTBuckets(false)) + publishEntity := processor.NewProcessorEntity(fmt.Sprintf("source-%s-%d", vertexInstance.Vertex.Name, vertexInstance.Replica), sourcePublishKeySpace, processor.WithSeparateOTBuckets(false)) // for tickgen you need the default heartbeat system because there are no concept of source partitions etc heartbeatBucket, err := progress.GetHeartbeatBucket(js, sourcePublishKeySpace) if err != nil { return err } // use this while reading the data from the source. - mg.progressor.sourcePublish = publish.NewPublish(mg.lifecycleCtx, publishEntity, progress.GetPublishKeySpace(metadata.Vertex), js, heartbeatBucket) + mg.progressor.sourcePublish = publish.NewPublish(mg.lifecycleCtx, publishEntity, progress.GetPublishKeySpace(vertexInstance.Vertex), js, heartbeatBucket) // fall back on the generic progressor and use the source publisher as the input to the generic progressor. // use the source Publisher as the source // TODO: remove this once bucket creation has been moved to controller - err = progress.CreateProcessorBucketIfMissing(fmt.Sprintf("%s_PROCESSORS", progress.GetPublishKeySpace(metadata.Vertex)), js) + err = progress.CreateProcessorBucketIfMissing(fmt.Sprintf("%s_PROCESSORS", progress.GetPublishKeySpace(vertexInstance.Vertex)), js) if err != nil { return err } - var wmProgressor = progress.NewGenericProgress(ctx, fmt.Sprintf("%s-%d", metadata.Vertex.Name, metadata.Replica), sourcePublishKeySpace, progress.GetPublishKeySpace(metadata.Vertex), js) + var wmProgressor = progress.NewGenericProgress(ctx, fmt.Sprintf("%s-%d", vertexInstance.Vertex.Name, vertexInstance.Replica), sourcePublishKeySpace, progress.GetPublishKeySpace(vertexInstance.Vertex), js) mg.progressor.wmProgressor = wmProgressor mg.logger.Info("Initialized watermark progressor") diff --git a/pkg/sources/generator/watermark_test.go b/pkg/sources/generator/watermark_test.go index 4ab819d71..dd3a5c626 100644 --- a/pkg/sources/generator/watermark_test.go +++ b/pkg/sources/generator/watermark_test.go @@ -9,7 +9,6 @@ import ( dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/isb/simplebuffer" - "github.com/numaproj/numaflow/pkg/sources/types" "github.com/stretchr/testify/assert" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,7 +26,7 @@ func TestWatermark(t *testing.T) { vertex := &dfv1.Vertex{ObjectMeta: v1.ObjectMeta{ Name: "memgen", }} - m := &types.SourceMetadata{ + m := &dfv1.VertexInstance{ Vertex: vertex, Hostname: "TestRead", Replica: 0, diff --git a/pkg/sources/http/http.go b/pkg/sources/http/http.go index 59639f12a..e279aeadd 100644 --- a/pkg/sources/http/http.go +++ b/pkg/sources/http/http.go @@ -174,10 +174,6 @@ func (h *httpSource) GetName() string { return h.name } -func (h *httpSource) Pending(_ context.Context) (int64, error) { - return isb.PendingNotAvailable, nil -} - func (h *httpSource) Read(ctx context.Context, count int64) ([]*isb.ReadMessage, error) { msgs := []*isb.ReadMessage{} timeout := time.After(h.readTimeout) diff --git a/pkg/sources/source.go b/pkg/sources/source.go index 3676e8965..dd207e6f4 100644 --- a/pkg/sources/source.go +++ b/pkg/sources/source.go @@ -5,7 +5,6 @@ import ( "fmt" "sync" - "github.com/numaproj/numaflow/pkg/sources/types" "go.uber.org/zap" dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" @@ -21,10 +20,8 @@ import ( ) type SourceProcessor struct { - ISBSvcType dfv1.ISBSvcType - Vertex *dfv1.Vertex - Hostname string - Replica int + ISBSvcType dfv1.ISBSvcType + VertexInstance *dfv1.VertexInstance } func (u *SourceProcessor) Start(ctx context.Context) error { @@ -34,7 +31,7 @@ func (u *SourceProcessor) Start(ctx context.Context) error { var writers []isb.BufferWriter switch u.ISBSvcType { case dfv1.ISBSvcTypeRedis: - for _, e := range u.Vertex.Spec.ToEdges { + for _, e := range u.VertexInstance.Vertex.Spec.ToEdges { writeOpts := []redisisb.Option{} if x := e.Limits; x != nil && x.BufferMaxLength != nil { writeOpts = append(writeOpts, redisisb.WithMaxLength(int64(*x.BufferMaxLength))) @@ -42,23 +39,28 @@ func (u *SourceProcessor) Start(ctx context.Context) error { if x := e.Limits; x != nil && x.BufferUsageLimit != nil { writeOpts = append(writeOpts, redisisb.WithBufferUsageLimit(float64(*x.BufferUsageLimit)/100)) } - buffer := dfv1.GenerateEdgeBufferName(u.Vertex.Namespace, u.Vertex.Spec.PipelineName, e.From, e.To) + buffer := dfv1.GenerateEdgeBufferName(u.VertexInstance.Vertex.Namespace, u.VertexInstance.Vertex.Spec.PipelineName, e.From, e.To) group := buffer + "-group" redisClient := clients.NewInClusterRedisClient() writer := redisisb.NewBufferWrite(ctx, redisClient, buffer, group, writeOpts...) writers = append(writers, writer) } case dfv1.ISBSvcTypeJetStream: - for _, e := range u.Vertex.Spec.ToEdges { - writeOpts := []jetstreamisb.WriteOption{} + for _, e := range u.VertexInstance.Vertex.Spec.ToEdges { + writeOpts := []jetstreamisb.WriteOption{ + jetstreamisb.WithUsingWriteInfoAsRate(true), + } + if x := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; x != nil { + writeOpts = append(writeOpts, jetstreamisb.WithWriteRateLookbackSeconds(int64(*x))) + } if x := e.Limits; x != nil && x.BufferMaxLength != nil { writeOpts = append(writeOpts, jetstreamisb.WithMaxLength(int64(*x.BufferMaxLength))) } if x := e.Limits; x != nil && x.BufferUsageLimit != nil { writeOpts = append(writeOpts, jetstreamisb.WithBufferUsageLimit(float64(*x.BufferUsageLimit)/100)) } - buffer := dfv1.GenerateEdgeBufferName(u.Vertex.Namespace, u.Vertex.Spec.PipelineName, e.From, e.To) - streamName := fmt.Sprintf("%s-%s", u.Vertex.Spec.PipelineName, buffer) + buffer := dfv1.GenerateEdgeBufferName(u.VertexInstance.Vertex.Namespace, u.VertexInstance.Vertex.Spec.PipelineName, e.From, e.To) + streamName := fmt.Sprintf("%s-%s", u.VertexInstance.Vertex.Spec.PipelineName, buffer) jetStreamClient := clients.NewInClusterJetStreamClient() writer, err := jetstreamisb.NewJetStreamBufferWriter(ctx, jetStreamClient, buffer, streamName, streamName, writeOpts...) if err != nil { @@ -67,14 +69,14 @@ func (u *SourceProcessor) Start(ctx context.Context) error { writers = append(writers, writer) } default: - return fmt.Errorf("unrecognized isbs type %q", u.ISBSvcType) + return fmt.Errorf("unrecognized isb svc type %q", u.ISBSvcType) } sourcer, err := u.getSourcer(writers, log) if err != nil { return fmt.Errorf("failed to find a sourcer, error: %w", err) } - log.Infow("Start processing source messages", zap.String("isbs", string(u.ISBSvcType)), zap.Any("to", u.Vertex.GetToBuffers())) + log.Infow("Start processing source messages", zap.String("isbs", string(u.ISBSvcType)), zap.Any("to", u.VertexInstance.Vertex.GetToBuffers())) stopped := sourcer.Start() wg := &sync.WaitGroup{} wg.Add(1) @@ -87,7 +89,18 @@ func (u *SourceProcessor) Start(ctx context.Context) error { } }() - if shutdown, err := metrics.StartMetricsServer(ctx); err != nil { + metricsOpts := []metrics.Option{} + if x, ok := sourcer.(isb.LagReader); ok { + metricsOpts = append(metricsOpts, metrics.WithLagReader(x)) + if s := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; s != nil { + metricsOpts = append(metricsOpts, metrics.WithPendingLookbackSeconds(int64(*s))) + } + } + if x, ok := writers[0].(isb.Ratable); ok { // Only need to use the rate of one of the writer + metricsOpts = append(metricsOpts, metrics.WithRater(x)) + } + ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) + if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) } else { defer func() { _ = shutdown(context.Background()) }() @@ -103,18 +116,13 @@ func (u *SourceProcessor) Start(ctx context.Context) error { // getSourcer is used to send the sourcer information func (u *SourceProcessor) getSourcer(writers []isb.BufferWriter, logger *zap.SugaredLogger) (Sourcer, error) { - src := u.Vertex.Spec.Source - m := &types.SourceMetadata{ - Vertex: u.Vertex, - Hostname: u.Hostname, - Replica: u.Replica, - } + src := u.VertexInstance.Vertex.Spec.Source if x := src.Generator; x != nil { - return generator.NewMemGen(m, int(*x.RPU), *x.MsgSize, x.Duration.Duration, writers, generator.WithLogger(logger)) + return generator.NewMemGen(u.VertexInstance, int(*x.RPU), *x.MsgSize, x.Duration.Duration, writers, generator.WithLogger(logger)) } else if x := src.Kafka; x != nil { - return kafka.NewKafkaSource(u.Vertex, writers, kafka.WithGroupName(x.ConsumerGroupName), kafka.WithLogger(logger)) + return kafka.NewKafkaSource(u.VertexInstance.Vertex, writers, kafka.WithGroupName(x.ConsumerGroupName), kafka.WithLogger(logger)) } else if x := src.HTTP; x != nil { - return http.New(u.Vertex, writers, http.WithLogger(logger)) + return http.New(u.VertexInstance.Vertex, writers, http.WithLogger(logger)) } return nil, fmt.Errorf("invalid source spec") } diff --git a/pkg/sources/types/metadata.go b/pkg/sources/types/metadata.go deleted file mode 100644 index 26322cf0b..000000000 --- a/pkg/sources/types/metadata.go +++ /dev/null @@ -1,9 +0,0 @@ -package types - -import dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" - -type SourceMetadata struct { - Vertex *dfv1.Vertex - Hostname string - Replica int -} diff --git a/pkg/udf/udf.go b/pkg/udf/udf.go index d50922089..a250648fe 100644 --- a/pkg/udf/udf.go +++ b/pkg/udf/udf.go @@ -23,10 +23,8 @@ import ( ) type UDFProcessor struct { - ISBSvcType dfv1.ISBSvcType - Vertex *dfv1.Vertex - Hostname string - Replica int + ISBSvcType dfv1.ISBSvcType + VertexInstance *dfv1.VertexInstance } func (u *UDFProcessor) Start(ctx context.Context) error { @@ -35,16 +33,16 @@ func (u *UDFProcessor) Start(ctx context.Context) error { defer cancel() var reader isb.BufferReader var err error - fromBufferName := u.Vertex.GetFromBuffers()[0].Name - toBuffers := u.Vertex.GetToBuffers() + fromBufferName := u.VertexInstance.Vertex.GetFromBuffers()[0].Name + toBuffers := u.VertexInstance.Vertex.GetToBuffers() writers := make(map[string]isb.BufferWriter) switch u.ISBSvcType { case dfv1.ISBSvcTypeRedis: redisClient := clients.NewInClusterRedisClient() fromGroup := fromBufferName + "-group" - consumer := fmt.Sprintf("%s-%v", u.Vertex.Name, u.Replica) + consumer := fmt.Sprintf("%s-%v", u.VertexInstance.Vertex.Name, u.VertexInstance.Replica) reader = redisisb.NewBufferRead(ctx, redisClient, fromBufferName, fromGroup, consumer) - for _, e := range u.Vertex.Spec.ToEdges { + for _, e := range u.VertexInstance.Vertex.Spec.ToEdges { writeOpts := []redisisb.Option{} if x := e.Limits; x != nil && x.BufferMaxLength != nil { writeOpts = append(writeOpts, redisisb.WithMaxLength(int64(*x.BufferMaxLength))) @@ -52,17 +50,23 @@ func (u *UDFProcessor) Start(ctx context.Context) error { if x := e.Limits; x != nil && x.BufferUsageLimit != nil { writeOpts = append(writeOpts, redisisb.WithBufferUsageLimit(float64(*x.BufferUsageLimit)/100)) } - buffer := dfv1.GenerateEdgeBufferName(u.Vertex.Namespace, u.Vertex.Spec.PipelineName, e.From, e.To) + buffer := dfv1.GenerateEdgeBufferName(u.VertexInstance.Vertex.Namespace, u.VertexInstance.Vertex.Spec.PipelineName, e.From, e.To) writer := redisisb.NewBufferWrite(ctx, redisClient, buffer, buffer+"-group", writeOpts...) writers[buffer] = writer } case dfv1.ISBSvcTypeJetStream: - fromStreamName := fmt.Sprintf("%s-%s", u.Vertex.Spec.PipelineName, fromBufferName) - reader, err = jetstreamisb.NewJetStreamBufferReader(ctx, clients.NewInClusterJetStreamClient(), fromBufferName, fromStreamName, fromStreamName) + fromStreamName := fmt.Sprintf("%s-%s", u.VertexInstance.Vertex.Spec.PipelineName, fromBufferName) + readOptions := []jetstreamisb.ReadOption{ + jetstreamisb.WithUsingAckInfoAsRate(true), + } + if x := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; x != nil { + readOptions = append(readOptions, jetstreamisb.WithAckRateLookbackSeconds(int64(*x))) + } + reader, err = jetstreamisb.NewJetStreamBufferReader(ctx, clients.NewInClusterJetStreamClient(), fromBufferName, fromStreamName, fromStreamName, readOptions...) if err != nil { return err } - for _, e := range u.Vertex.Spec.ToEdges { + for _, e := range u.VertexInstance.Vertex.Spec.ToEdges { writeOpts := []jetstreamisb.WriteOption{} if x := e.Limits; x != nil && x.BufferMaxLength != nil { writeOpts = append(writeOpts, jetstreamisb.WithMaxLength(int64(*x.BufferMaxLength))) @@ -70,8 +74,8 @@ func (u *UDFProcessor) Start(ctx context.Context) error { if x := e.Limits; x != nil && x.BufferUsageLimit != nil { writeOpts = append(writeOpts, jetstreamisb.WithBufferUsageLimit(float64(*x.BufferUsageLimit)/100)) } - buffer := dfv1.GenerateEdgeBufferName(u.Vertex.Namespace, u.Vertex.Spec.PipelineName, e.From, e.To) - streamName := fmt.Sprintf("%s-%s", u.Vertex.Spec.PipelineName, buffer) + buffer := dfv1.GenerateEdgeBufferName(u.VertexInstance.Vertex.Namespace, u.VertexInstance.Vertex.Spec.PipelineName, e.From, e.To) + streamName := fmt.Sprintf("%s-%s", u.VertexInstance.Vertex.Spec.PipelineName, buffer) writer, err := jetstreamisb.NewJetStreamBufferWriter(ctx, clients.NewInClusterJetStreamClient(), buffer, streamName, streamName, writeOpts...) if err != nil { return err @@ -89,11 +93,11 @@ func (u *UDFProcessor) Start(ctx context.Context) error { result = append(result, _key) return result, nil } - for _, to := range u.Vertex.Spec.ToEdges { + for _, to := range u.VertexInstance.Vertex.Spec.ToEdges { // If returned key is not "ALL" or "DROP", and there's no conditions defined in the edge, // treat it as "ALL"? if to.Conditions == nil || len(to.Conditions.KeyIn) == 0 || sharedutil.StringSliceContains(to.Conditions.KeyIn, _key) { - result = append(result, dfv1.GenerateEdgeBufferName(u.Vertex.Namespace, u.Vertex.Spec.PipelineName, to.From, to.To)) + result = append(result, dfv1.GenerateEdgeBufferName(u.VertexInstance.Vertex.Namespace, u.VertexInstance.Vertex.Spec.PipelineName, to.From, to.To)) } } return result, nil @@ -106,7 +110,7 @@ func (u *UDFProcessor) Start(ctx context.Context) error { } log.Infow("Start processing udf messages", zap.String("isbs", string(u.ISBSvcType)), zap.String("from", fromBufferName), zap.Any("to", toBuffers)) opts := []forward.Option{forward.WithLogger(log)} - if x := u.Vertex.Spec.Limits; x != nil { + if x := u.VertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { opts = append(opts, forward.WithReadBatchSize(int64(*x.ReadBatchSize))) } @@ -122,14 +126,14 @@ func (u *UDFProcessor) Start(ctx context.Context) error { return err } // TODO: remove this once bucket creation has been moved to controller - err = progress.CreateProcessorBucketIfMissing(fmt.Sprintf("%s_PROCESSORS", progress.GetPublishKeySpace(u.Vertex)), js) + err = progress.CreateProcessorBucketIfMissing(fmt.Sprintf("%s_PROCESSORS", progress.GetPublishKeySpace(u.VertexInstance.Vertex)), js) if err != nil { return err } - wmProgressor = progress.NewGenericProgress(ctx, fmt.Sprintf("%s-%d", u.Vertex.Name, u.Replica), progress.GetFetchKeyspace(u.Vertex), progress.GetPublishKeySpace(u.Vertex), js) + wmProgressor = progress.NewGenericProgress(ctx, fmt.Sprintf("%s-%d", u.VertexInstance.Vertex.Name, u.VertexInstance.Replica), progress.GetFetchKeyspace(u.VertexInstance.Vertex), progress.GetPublishKeySpace(u.VertexInstance.Vertex), js) } - forwarder, err := forward.NewInterStepDataForward(u.Vertex, reader, writers, conditionalForwarder, udfHandler, wmProgressor, opts...) + forwarder, err := forward.NewInterStepDataForward(u.VertexInstance.Vertex, reader, writers, conditionalForwarder, udfHandler, wmProgressor, opts...) if err != nil { return err } @@ -146,7 +150,18 @@ func (u *UDFProcessor) Start(ctx context.Context) error { } }() - if shutdown, err := metrics.StartMetricsServer(ctx); err != nil { + metricsOpts := []metrics.Option{} + if x, ok := reader.(isb.LagReader); ok { + metricsOpts = append(metricsOpts, metrics.WithLagReader(x)) + if s := u.VertexInstance.Vertex.Spec.Scale.LookbackSeconds; s != nil { + metricsOpts = append(metricsOpts, metrics.WithPendingLookbackSeconds(int64(*s))) + } + } + if x, ok := reader.(isb.Ratable); ok { + metricsOpts = append(metricsOpts, metrics.WithRater(x)) + } + ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) + if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) } else { defer func() { _ = shutdown(context.Background()) }() diff --git a/test/fixtures/expect.go b/test/fixtures/expect.go index 6e36647ac..113a97ec5 100644 --- a/test/fixtures/expect.go +++ b/test/fixtures/expect.go @@ -68,7 +68,7 @@ func (t *Expect) VertexPodsRunning() *Expect { // check pods running timeout := 2 * time.Minute for _, v := range t.pipeline.Spec.Vertices { - if err := WaitForVertexPodRunning(t.kubeClient, Namespace, t.pipeline.Name, v.Name, timeout); err != nil { + if err := WaitForVertexPodRunning(t.kubeClient, t.vertexClient, Namespace, t.pipeline.Name, v.Name, timeout); err != nil { t.t.Fatalf("expected vertex %q pod running: %v", v.Name, err) } } diff --git a/test/fixtures/util.go b/test/fixtures/util.go index c50abac1f..820ea881d 100644 --- a/test/fixtures/util.go +++ b/test/fixtures/util.go @@ -200,7 +200,7 @@ func WaitForPipelineRunning(ctx context.Context, pipelineClient flowpkg.Pipeline } } -func WaitForVertexPodRunning(kubeClient kubernetes.Interface, namespace, pipelineName, vertexName string, timeout time.Duration) error { +func WaitForVertexPodRunning(kubeClient kubernetes.Interface, vertexClient flowpkg.VertexInterface, namespace, pipelineName, vertexName string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() labelSelector := fmt.Sprintf("%s=%s,%s=%s", dfv1.KeyPipelineName, pipelineName, dfv1.KeyVertexName, vertexName) @@ -210,11 +210,16 @@ func WaitForVertexPodRunning(kubeClient kubernetes.Interface, namespace, pipelin return fmt.Errorf("timeout after %v waiting for vertex pod running", timeout) default: } + vertexList, err := vertexClient.List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return fmt.Errorf("error getting vertex list: %w", err) + } + ok := len(vertexList.Items) == 1 podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector, FieldSelector: "status.phase=Running"}) if err != nil { return fmt.Errorf("error getting vertex pod name: %w", err) } - ok := len(podList.Items) > 0 + ok = ok && len(podList.Items) > 0 && len(podList.Items) == int(*vertexList.Items[0].Spec.Replicas) // pod number should equal to desired replicas for _, p := range podList.Items { ok = ok && p.Status.Phase == corev1.PodRunning }