diff --git a/api/v1/rainbow.proto b/api/v1/rainbow.proto index 1b6c822..88f047d 100644 --- a/api/v1/rainbow.proto +++ b/api/v1/rainbow.proto @@ -39,12 +39,15 @@ message RegisterRequest { // we currently accept nodes, tasks, and the command message SubmitJobRequest { string name = 1; - string cluster = 2; - string token = 3; - int32 nodes = 4; - int32 tasks = 5; - string command = 6; - google.protobuf.Timestamp sent = 7; + repeated Cluster clusters = 2; + string jobspec = 3; + google.protobuf.Timestamp sent = 4; + + message Cluster { + string name = 1; + string token = 2; + } + } // RequestJobsRequest is used by a cluster (or other entity that can run jobs) diff --git a/backends/memory/cluster.go b/backends/memory/cluster.go index 6c4ee44..1937741 100644 --- a/backends/memory/cluster.go +++ b/backends/memory/cluster.go @@ -267,7 +267,7 @@ func (g *ClusterGraph) LoadClusterNodes( if !ok { return fmt.Errorf("destination %s is defined as an edge, but missing as node in graph", edge.Label) } - fmt.Printf("Adding edge from %s -%s-> %s\n", ss.Vertices[src].Type, edge.Relation, ss.Vertices[dest].Type) + // fmt.Printf("Adding edge from %s -%s-> %s\n", ss.Vertices[src].Type, edge.Relation, ss.Vertices[dest].Type) err := ss.AddEdge(src, dest, 0, edge.Relation) if err != nil { return err diff --git a/backends/memory/subsystem.go b/backends/memory/subsystem.go index 645fa6a..a9d4e80 100644 --- a/backends/memory/subsystem.go +++ b/backends/memory/subsystem.go @@ -26,6 +26,8 @@ func NewSubsystem() *Subsystem { // DFSForMatch WILL is a depth first search for matches // It starts by looking at total cluster resources on the top level, // and then traverses into those that match the first check +// THIS IS EXPERIMENTAL and likely wrong, or missing details, +// which is OK as we will only be using it for prototyping. func (s *Subsystem) DFSForMatch(jobspec *js.Jobspec) ([]string, error) { // Return a list of matching clusters @@ -252,7 +254,7 @@ func (s *Subsystem) depthFirstSearch(matches []string, jobspec *js.Jobspec) ([]s fmt.Printf(" ❌️ %s not a match, %s\n", cluster, reason) return false } else { - reason := fmt.Sprintf("%d of needed %s satisfied", foundMatches, resource.Type) + reason := fmt.Sprintf("%d/%d of needed %s satisfied", foundMatches, resource.Count, resource.Type) fmt.Printf(" ⏳️ %s still contender, %s\n", cluster, reason) } } diff --git a/cmd/rainbow/submit/submit.go b/cmd/rainbow/submit/submit.go index bb5da84..4925093 100644 --- a/cmd/rainbow/submit/submit.go +++ b/cmd/rainbow/submit/submit.go @@ -6,9 +6,9 @@ import ( "log" "strings" + js "github.com/compspec/jobspec-go/pkg/jobspec/v1" "github.com/converged-computing/rainbow/pkg/client" "github.com/converged-computing/rainbow/pkg/config" - "github.com/converged-computing/rainbow/pkg/jobspec" ) // Run will check a manifest list of artifacts against a host machine @@ -38,7 +38,7 @@ func Run( } // Convert the simple command / nodes / etc into a JobSpec - js, err := jobspec.NewSimpleJobspec(jobName, command, int32(nodes), int32(tasks)) + js, err := js.NewSimpleJobspec(jobName, command, int32(nodes), int32(tasks)) if err != nil { return nil } diff --git a/docs/commands.md b/docs/commands.md index 8463121..bf09324 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -154,6 +154,11 @@ and then working on the next interaction, the client submit command, which is go ## Submit Job +Submission has two steps that are discussed below. + +### 1. Satisfy Request + +The satisfy request interacts with the graph database and determines if any clusters can satisfy the jobspec. To submit a job, we need the client `token` associated with a cluster. We are going to use the following strategy, and allow the following submission types: - **simple**: for basic users, a command and the most basic of parameters will be provided and converted to a Jobspec. @@ -253,8 +258,22 @@ cluster keebler does not have sufficient resource type node - actual 3 vs needed match: 😥️ no clusters could satisfy this request. We are sad ``` -Note that the above is not technically a graph search yet - we are just checking the global resources of each cluster. I need to perform the DFS when I better -understand / think about a good strategy for that, likely reading Fluxion code. +Note that the above has a two step process: + +- A quick check against clusters in the graph database if total resources can be satisfied. +- For that set, a (Vanessa written and janky) "DFS" that likely has bugs that traverses the graph + +This will be improved upon with Fluxion and actual graph databases, but this is OK for the prototype. + +### 2. Pre-Assignment + +When the initial satisfy resquest is done (the step above) and we have a list of clusters, we can then tell rainbow about them. +This means that a list of clusters is returned that is passed from the same client request to rainbow +to do assignment, and logically, if there are no clusters that can sastify, that response is returned to the client. + + +## Request Jobs + ## Request Jobs diff --git a/go.mod b/go.mod index 201b3a1..a27d2ce 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,8 @@ go 1.20 require ( github.com/akamensky/argparse v1.4.0 - github.com/compspec/jobspec-go v0.0.0-20240226213125-007327866207 + github.com/compspec/jobspec-go v0.0.0-20240302201731-e7fb2bf2627f github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.6.0 github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/errors v0.9.1 @@ -17,8 +16,10 @@ require ( require ( github.com/golang/protobuf v1.5.3 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index b7e13e8..4f901dd 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,23 @@ github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= -github.com/compspec/jobspec-go v0.0.0-20240226213125-007327866207 h1:p872BOJceUTU2+FOXKjVz68/VwAkN0zGdSigiWTMao0= -github.com/compspec/jobspec-go v0.0.0-20240226213125-007327866207/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM= +github.com/compspec/jobspec-go v0.0.0-20240302201731-e7fb2bf2627f h1:JHOVu3snvprXuO3UDT2FngfmfDGj+2g5inZof2my9IA= +github.com/compspec/jobspec-go v0.0.0-20240302201731-e7fb2bf2627f/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM= github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h1:Tk//RW3uKn4A7N8gpHRXs+ZGlR7Fxkwh+4/Iml0GBV4= github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= @@ -36,3 +37,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/api/v1/rainbow.pb.go b/pkg/api/v1/rainbow.pb.go index 45037c0..871e3f1 100644 --- a/pkg/api/v1/rainbow.pb.go +++ b/pkg/api/v1/rainbow.pb.go @@ -325,13 +325,10 @@ type SubmitJobRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Cluster string `protobuf:"bytes,2,opt,name=cluster,proto3" json:"cluster,omitempty"` - Token string `protobuf:"bytes,3,opt,name=token,proto3" json:"token,omitempty"` - Nodes int32 `protobuf:"varint,4,opt,name=nodes,proto3" json:"nodes,omitempty"` - Tasks int32 `protobuf:"varint,5,opt,name=tasks,proto3" json:"tasks,omitempty"` - Command string `protobuf:"bytes,6,opt,name=command,proto3" json:"command,omitempty"` - Sent *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=sent,proto3" json:"sent,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Clusters []*SubmitJobRequest_Cluster `protobuf:"bytes,2,rep,name=clusters,proto3" json:"clusters,omitempty"` + Jobspec string `protobuf:"bytes,3,opt,name=jobspec,proto3" json:"jobspec,omitempty"` + Sent *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=sent,proto3" json:"sent,omitempty"` } func (x *SubmitJobRequest) Reset() { @@ -373,37 +370,16 @@ func (x *SubmitJobRequest) GetName() string { return "" } -func (x *SubmitJobRequest) GetCluster() string { +func (x *SubmitJobRequest) GetClusters() []*SubmitJobRequest_Cluster { if x != nil { - return x.Cluster + return x.Clusters } - return "" -} - -func (x *SubmitJobRequest) GetToken() string { - if x != nil { - return x.Token - } - return "" -} - -func (x *SubmitJobRequest) GetNodes() int32 { - if x != nil { - return x.Nodes - } - return 0 -} - -func (x *SubmitJobRequest) GetTasks() int32 { - if x != nil { - return x.Tasks - } - return 0 + return nil } -func (x *SubmitJobRequest) GetCommand() string { +func (x *SubmitJobRequest) GetJobspec() string { if x != nil { - return x.Command + return x.Jobspec } return "" } @@ -817,6 +793,61 @@ func (x *AcceptJobsResponse) GetStatus() AcceptJobsResponse_ResultType { return AcceptJobsResponse_RESULT_TYPE_UNSPECIFIED } +type SubmitJobRequest_Cluster struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"` +} + +func (x *SubmitJobRequest_Cluster) Reset() { + *x = SubmitJobRequest_Cluster{} + if protoimpl.UnsafeEnabled { + mi := &file_rainbow_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SubmitJobRequest_Cluster) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitJobRequest_Cluster) ProtoMessage() {} + +func (x *SubmitJobRequest_Cluster) ProtoReflect() protoreflect.Message { + mi := &file_rainbow_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitJobRequest_Cluster.ProtoReflect.Descriptor instead. +func (*SubmitJobRequest_Cluster) Descriptor() ([]byte, []int) { + return file_rainbow_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *SubmitJobRequest_Cluster) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SubmitJobRequest_Cluster) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + var File_rainbow_proto protoreflect.FileDescriptor var file_rainbow_proto_rawDesc = []byte{ @@ -835,146 +866,149 @@ var file_rainbow_proto_rawDesc = []byte{ 0x73, 0x74, 0x65, 0x6d, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, - 0x73, 0x65, 0x6e, 0x74, 0x22, 0xcc, 0x01, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, + 0x73, 0x65, 0x6e, 0x74, 0x22, 0xfb, 0x01, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x14, 0x0a, - 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6e, 0x6f, - 0x64, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, - 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x73, - 0x65, 0x6e, 0x74, 0x22, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, - 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, - 0x6d, 0x61, 0x78, 0x4a, 0x6f, 0x62, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, - 0x61, 0x78, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x22, 0x8d, 0x01, 0x0a, 0x11, 0x41, 0x63, 0x63, 0x65, 0x70, - 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x05, 0x52, 0x06, - 0x6a, 0x6f, 0x62, 0x69, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x22, 0xb0, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x53, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, - 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x7a, 0x0a, - 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x52, - 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x52, - 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, - 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x4e, 0x49, - 0x45, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, - 0x5f, 0x45, 0x58, 0x49, 0x53, 0x54, 0x53, 0x10, 0x04, 0x22, 0xfd, 0x01, 0x0a, 0x11, 0x53, 0x75, - 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x14, - 0x0a, 0x05, 0x6a, 0x6f, 0x62, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6a, - 0x6f, 0x62, 0x69, 0x64, 0x12, 0x54, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, - 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x5d, 0x0a, 0x0a, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x55, 0x42, 0x4d, - 0x49, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, - 0x53, 0x53, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, 0x5f, 0x45, - 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, - 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x03, 0x22, 0x8d, 0x03, 0x0a, 0x13, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, - 0x12, 0x51, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, - 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, - 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x6a, - 0x6f, 0x62, 0x73, 0x12, 0x56, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x3e, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x54, 0x0a, + 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6a, 0x6f, 0x62, 0x73, 0x70, 0x65, 0x63, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x73, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, + 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x1a, 0x33, 0x0a, + 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x22, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x61, 0x78, 0x4a, 0x6f, 0x62, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x61, + 0x78, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x04, 0x73, 0x65, 0x6e, 0x74, 0x22, 0x8d, 0x01, 0x0a, 0x11, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, + 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x6a, 0x6f, 0x62, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x05, 0x52, 0x06, 0x6a, + 0x6f, 0x62, 0x69, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x04, 0x73, 0x65, 0x6e, 0x74, 0x22, 0xb0, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x53, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, + 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x7a, 0x0a, 0x0a, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, + 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, + 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, + 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x13, + 0x0a, 0x0f, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, + 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, + 0x45, 0x58, 0x49, 0x53, 0x54, 0x53, 0x10, 0x04, 0x22, 0xfd, 0x01, 0x0a, 0x11, 0x53, 0x75, 0x62, + 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x6a, 0x6f, 0x62, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6a, 0x6f, + 0x62, 0x69, 0x64, 0x12, 0x54, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4a, - 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x73, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, - 0x42, 0x53, 0x5f, 0x4e, 0x4f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x53, 0x10, 0x00, 0x12, 0x18, - 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, 0x5f, 0x53, - 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, - 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, - 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x03, 0x22, 0xdf, 0x01, 0x0a, 0x12, 0x41, 0x63, - 0x63, 0x65, 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x55, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x3d, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, - 0x31, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x72, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x50, 0x41, 0x52, 0x54, 0x49, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, - 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, - 0x53, 0x53, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x32, 0xe0, 0x03, 0x0a, 0x10, - 0x52, 0x61, 0x69, 0x6e, 0x62, 0x6f, 0x77, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x12, 0x6d, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2f, 0x2e, 0x63, - 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, - 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, + 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x5d, 0x0a, 0x0a, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x55, 0x42, 0x4d, 0x49, + 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x12, 0x0a, 0x0e, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, + 0x53, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, 0x5f, 0x45, 0x52, + 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x55, 0x42, 0x4d, 0x49, 0x54, 0x5f, + 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x03, 0x22, 0x8d, 0x03, 0x0a, 0x13, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, + 0x51, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x70, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x30, 0x2e, 0x63, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x6a, 0x6f, + 0x62, 0x73, 0x12, 0x56, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x3e, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, + 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x4a, 0x6f, + 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x73, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, + 0x53, 0x5f, 0x4e, 0x4f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x53, 0x10, 0x00, 0x12, 0x18, 0x0a, + 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, 0x5f, 0x53, 0x55, + 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x51, 0x55, 0x45, + 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, + 0x17, 0x0a, 0x13, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4a, 0x4f, 0x42, 0x53, 0x5f, + 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x03, 0x22, 0xdf, 0x01, 0x0a, 0x12, 0x41, 0x63, 0x63, + 0x65, 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x55, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x3d, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x72, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x50, 0x41, 0x52, 0x54, 0x49, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, + 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, + 0x53, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x32, 0xe0, 0x03, 0x0a, 0x10, 0x52, + 0x61, 0x69, 0x6e, 0x62, 0x6f, 0x77, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x12, + 0x6d, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2f, 0x2e, 0x63, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, - 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, - 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, - 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, - 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x76, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, - 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, - 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, - 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x73, 0x0a, 0x0a, 0x41, 0x63, 0x63, - 0x65, 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, - 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4a, - 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, - 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x63, 0x65, - 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x33, - 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, - 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x2d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, - 0x2f, 0x72, 0x61, 0x69, 0x6e, 0x62, 0x6f, 0x77, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x70, + 0x0a, 0x09, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x30, 0x2e, 0x63, 0x6f, + 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, + 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, + 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x75, 0x62, 0x6d, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x76, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, + 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, + 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x73, 0x0a, 0x0a, 0x41, 0x63, 0x63, 0x65, + 0x70, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x67, + 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x76, + 0x65, 0x72, 0x67, 0x65, 0x64, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x70, + 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x33, 0x5a, + 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x76, + 0x65, 0x72, 0x67, 0x65, 0x64, 0x2d, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x2f, + 0x72, 0x61, 0x69, 0x6e, 0x62, 0x6f, 0x77, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -990,7 +1024,7 @@ func file_rainbow_proto_rawDescGZIP() []byte { } var file_rainbow_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_rainbow_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_rainbow_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_rainbow_proto_goTypes = []interface{}{ (RegisterResponse_ResultType)(0), // 0: convergedcomputing.org.grpc.v1.RegisterResponse.ResultType (SubmitJobResponse_ResultType)(0), // 1: convergedcomputing.org.grpc.v1.SubmitJobResponse.ResultType @@ -1004,32 +1038,34 @@ var file_rainbow_proto_goTypes = []interface{}{ (*SubmitJobResponse)(nil), // 9: convergedcomputing.org.grpc.v1.SubmitJobResponse (*RequestJobsResponse)(nil), // 10: convergedcomputing.org.grpc.v1.RequestJobsResponse (*AcceptJobsResponse)(nil), // 11: convergedcomputing.org.grpc.v1.AcceptJobsResponse - nil, // 12: convergedcomputing.org.grpc.v1.RequestJobsResponse.JobsEntry - (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*SubmitJobRequest_Cluster)(nil), // 12: convergedcomputing.org.grpc.v1.SubmitJobRequest.Cluster + nil, // 13: convergedcomputing.org.grpc.v1.RequestJobsResponse.JobsEntry + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp } var file_rainbow_proto_depIdxs = []int32{ - 13, // 0: convergedcomputing.org.grpc.v1.RegisterRequest.sent:type_name -> google.protobuf.Timestamp - 13, // 1: convergedcomputing.org.grpc.v1.SubmitJobRequest.sent:type_name -> google.protobuf.Timestamp - 13, // 2: convergedcomputing.org.grpc.v1.RequestJobsRequest.sent:type_name -> google.protobuf.Timestamp - 13, // 3: convergedcomputing.org.grpc.v1.AcceptJobsRequest.sent:type_name -> google.protobuf.Timestamp - 0, // 4: convergedcomputing.org.grpc.v1.RegisterResponse.status:type_name -> convergedcomputing.org.grpc.v1.RegisterResponse.ResultType - 1, // 5: convergedcomputing.org.grpc.v1.SubmitJobResponse.status:type_name -> convergedcomputing.org.grpc.v1.SubmitJobResponse.ResultType - 12, // 6: convergedcomputing.org.grpc.v1.RequestJobsResponse.jobs:type_name -> convergedcomputing.org.grpc.v1.RequestJobsResponse.JobsEntry - 2, // 7: convergedcomputing.org.grpc.v1.RequestJobsResponse.status:type_name -> convergedcomputing.org.grpc.v1.RequestJobsResponse.ResultType - 3, // 8: convergedcomputing.org.grpc.v1.AcceptJobsResponse.status:type_name -> convergedcomputing.org.grpc.v1.AcceptJobsResponse.ResultType - 4, // 9: convergedcomputing.org.grpc.v1.RainbowScheduler.Register:input_type -> convergedcomputing.org.grpc.v1.RegisterRequest - 5, // 10: convergedcomputing.org.grpc.v1.RainbowScheduler.SubmitJob:input_type -> convergedcomputing.org.grpc.v1.SubmitJobRequest - 6, // 11: convergedcomputing.org.grpc.v1.RainbowScheduler.RequestJobs:input_type -> convergedcomputing.org.grpc.v1.RequestJobsRequest - 7, // 12: convergedcomputing.org.grpc.v1.RainbowScheduler.AcceptJobs:input_type -> convergedcomputing.org.grpc.v1.AcceptJobsRequest - 8, // 13: convergedcomputing.org.grpc.v1.RainbowScheduler.Register:output_type -> convergedcomputing.org.grpc.v1.RegisterResponse - 9, // 14: convergedcomputing.org.grpc.v1.RainbowScheduler.SubmitJob:output_type -> convergedcomputing.org.grpc.v1.SubmitJobResponse - 10, // 15: convergedcomputing.org.grpc.v1.RainbowScheduler.RequestJobs:output_type -> convergedcomputing.org.grpc.v1.RequestJobsResponse - 11, // 16: convergedcomputing.org.grpc.v1.RainbowScheduler.AcceptJobs:output_type -> convergedcomputing.org.grpc.v1.AcceptJobsResponse - 13, // [13:17] is the sub-list for method output_type - 9, // [9:13] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 14, // 0: convergedcomputing.org.grpc.v1.RegisterRequest.sent:type_name -> google.protobuf.Timestamp + 12, // 1: convergedcomputing.org.grpc.v1.SubmitJobRequest.clusters:type_name -> convergedcomputing.org.grpc.v1.SubmitJobRequest.Cluster + 14, // 2: convergedcomputing.org.grpc.v1.SubmitJobRequest.sent:type_name -> google.protobuf.Timestamp + 14, // 3: convergedcomputing.org.grpc.v1.RequestJobsRequest.sent:type_name -> google.protobuf.Timestamp + 14, // 4: convergedcomputing.org.grpc.v1.AcceptJobsRequest.sent:type_name -> google.protobuf.Timestamp + 0, // 5: convergedcomputing.org.grpc.v1.RegisterResponse.status:type_name -> convergedcomputing.org.grpc.v1.RegisterResponse.ResultType + 1, // 6: convergedcomputing.org.grpc.v1.SubmitJobResponse.status:type_name -> convergedcomputing.org.grpc.v1.SubmitJobResponse.ResultType + 13, // 7: convergedcomputing.org.grpc.v1.RequestJobsResponse.jobs:type_name -> convergedcomputing.org.grpc.v1.RequestJobsResponse.JobsEntry + 2, // 8: convergedcomputing.org.grpc.v1.RequestJobsResponse.status:type_name -> convergedcomputing.org.grpc.v1.RequestJobsResponse.ResultType + 3, // 9: convergedcomputing.org.grpc.v1.AcceptJobsResponse.status:type_name -> convergedcomputing.org.grpc.v1.AcceptJobsResponse.ResultType + 4, // 10: convergedcomputing.org.grpc.v1.RainbowScheduler.Register:input_type -> convergedcomputing.org.grpc.v1.RegisterRequest + 5, // 11: convergedcomputing.org.grpc.v1.RainbowScheduler.SubmitJob:input_type -> convergedcomputing.org.grpc.v1.SubmitJobRequest + 6, // 12: convergedcomputing.org.grpc.v1.RainbowScheduler.RequestJobs:input_type -> convergedcomputing.org.grpc.v1.RequestJobsRequest + 7, // 13: convergedcomputing.org.grpc.v1.RainbowScheduler.AcceptJobs:input_type -> convergedcomputing.org.grpc.v1.AcceptJobsRequest + 8, // 14: convergedcomputing.org.grpc.v1.RainbowScheduler.Register:output_type -> convergedcomputing.org.grpc.v1.RegisterResponse + 9, // 15: convergedcomputing.org.grpc.v1.RainbowScheduler.SubmitJob:output_type -> convergedcomputing.org.grpc.v1.SubmitJobResponse + 10, // 16: convergedcomputing.org.grpc.v1.RainbowScheduler.RequestJobs:output_type -> convergedcomputing.org.grpc.v1.RequestJobsResponse + 11, // 17: convergedcomputing.org.grpc.v1.RainbowScheduler.AcceptJobs:output_type -> convergedcomputing.org.grpc.v1.AcceptJobsResponse + 14, // [14:18] is the sub-list for method output_type + 10, // [10:14] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_rainbow_proto_init() } @@ -1134,6 +1170,18 @@ func file_rainbow_proto_init() { return nil } } + file_rainbow_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubmitJobRequest_Cluster); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1141,7 +1189,7 @@ func file_rainbow_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_rainbow_proto_rawDesc, NumEnums: 4, - NumMessages: 9, + NumMessages: 10, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/client/endpoint.go b/pkg/client/endpoint.go index d860b2f..9ff4a1c 100644 --- a/pkg/client/endpoint.go +++ b/pkg/client/endpoint.go @@ -55,28 +55,38 @@ func (c *RainbowClient) SubmitJob( if err != nil { return response, err } + + // Cut out early (without contacting rainbow) if there are no matches if len(matches) > 0 { log.Printf("🎯️ We found %d matches! %s\b", len(matches), matches) } else { - log.Println("😥️ There were no matches for this job") + return response, fmt.Errorf("😥️ There were no matches for this job") } // Now contact the rainbow server with clusters... - // ctx, cancel := context.WithTimeout(ctx, time.Second) - // defer cancel() + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + // Prepare clusters for submit jobs request + clusters := make([]*pb.SubmitJobRequest_Cluster, len(cfg.Clusters)) + for i, cluster := range cfg.Clusters { + clusters[i] = &pb.SubmitJobRequest_Cluster{Token: cluster.Token, Name: cluster.Name} + } + + // Jobspec gets converted back to string for easier serialization + out, err := job.JobspecToYaml() + if err != nil { + return response, err + } // Validate that the cluster exists, and we have the right token. // The response is the same either way - not found does not reveal // additional information to the client trying to find it - /* response, err := c.service.SubmitJob(ctx, &pb.SubmitJobRequest{ - Name: job.Name, - Token: token, - Nodes: job.Nodes, - Tasks: job.Tasks, - Cluster: cluster, - Command: job.Command, - Sent: ts.Now(), - })*/ - return response, nil + response, err = c.service.SubmitJob(ctx, &pb.SubmitJobRequest{ + Name: job.GetJobName(), + Clusters: clusters, + Jobspec: string(out), + Sent: ts.Now(), + }) + return response, err } // RequestJobs requests jobs for a specific cluster diff --git a/pkg/config/config.go b/pkg/config/config.go index b3a2f6d..8de7ed4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -105,6 +105,13 @@ func NewRainbowClientConfig( return &config, err } +// NewRainbowServerConfig creates a default empty config for a server +func NewRainbowServerConfig(name string) *RainbowConfig { + config := RainbowConfig{Scheduler: RainbowScheduler{Name: name}} + config.Clusters = make([]ClusterCredential, 0) + return &config +} + // Load a filename into the rainbow config func (cfg *RainbowConfig) Load(filename string) error { diff --git a/pkg/database/database.go b/pkg/database/database.go index 61fd6f5..ad3f231 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -218,6 +218,7 @@ func (db *Database) createTables() error { } defer conn.Close() + // TODO none of these have logic for what to do on delete // Create the clusters table, where we store the name and secret // obviously the secret should not be stored in plain text - it's fine for now createClusterTableSQL := ` @@ -228,30 +229,41 @@ func (db *Database) createTables() error { ); ` + // Contender assignments of jobs to clusters + // Assignment is an integer identifier (this can be changed / extended) + // 0: unassigned + // -1: rejected + // 1: assigned + createAssignTableSQL := ` + CREATE TABLE assign ( + assignId integer NOT NULL PRIMARY KEY AUTOINCREMENT, + cluster TEXT, + job integer, + status integer, + FOREIGN KEY(job) REFERENCES jobs(idJob), + FOREIGN KEY(cluster) REFERENCES clusters(name) + );` + + // A job has a unique id and (when assigned) a cluster createJobsTableSQL := ` CREATE TABLE jobs ( idJob integer NOT NULL PRIMARY KEY AUTOINCREMENT, cluster TEXT, name TEXT, - nodes integer, - tasks integer, - command TEXT, + jobspec string, FOREIGN KEY(cluster) REFERENCES clusters(name) );` - for table, statement := range map[string]string{"cluster": createClusterTableSQL, "jobs": createJobsTableSQL} { - log.Printf(" create %s table...\n", table) - query, err := conn.Prepare(statement) // Prepare SQL Statement - if err != nil { - return err - } - // Execute SQL query - _, err = query.Exec() - if err != nil { - return err - } - log.Printf(" %s table created\n", table) + // Create single query for tables + createSQL := createClusterTableSQL + "\n" + createAssignTableSQL + "\n" + createJobsTableSQL + log.Println(" 🏓️ creating tables...") + + // Execute SQL query + _, err = conn.Exec(createSQL) + if err != nil { + return err } + log.Println(" 🏓️ tables created") return nil } @@ -276,6 +288,9 @@ func InitDatabase(filepath string, cleanup bool) (*Database, error) { return nil, err } err = db.createTables() + if err != nil { + return nil, err + } } return &db, err } diff --git a/pkg/database/jobs.go b/pkg/database/jobs.go index 0b4e56e..7bb0626 100644 --- a/pkg/database/jobs.go +++ b/pkg/database/jobs.go @@ -28,53 +28,104 @@ func (j *Job) ToJson() (string, error) { return string(b), nil } -// SubmitJob adds the job to the database -func (db *Database) SubmitJob( - job *pb.SubmitJobRequest, - cluster *Cluster, -) (*pb.SubmitJobResponse, error) { +// addJob adds a job to the jobs table +func (db *Database) addJob(job *pb.SubmitJobRequest) (*Job, error) { - response := &pb.SubmitJobResponse{} + j := Job{} conn, err := db.connect() if err != nil { - return response, err + return &j, err } defer conn.Close() - // Prepare the sql to insert the job - fields := "(cluster, name, nodes, tasks, command)" - values := fmt.Sprintf( - "(\"%s\", \"%s\",\"%d\",\"%d\",\"%s\")", - cluster.Name, job.Name, job.Nodes, job.Tasks, job.Command, - ) + // The jobspec is added once to the database, first without assignment + fields := "(name, jobspec)" + values := fmt.Sprintf("(\"%s\", \"%s\")", job.Name, job.Jobspec) // Submit the query to get the global id (jobid, not submit yet) query := fmt.Sprintf("INSERT into jobs %s VALUES %s", fields, values) - // From this point on (until the end) any early return is an error - response.Status = pb.SubmitJobResponse_SUBMIT_ERROR - // Since we want to get a result back, we use query statement, err := conn.Prepare(query) if err != nil { - return response, err + return &j, err } defer statement.Close() // We expect only one job rows, err := statement.Query() if err != nil { - return response, err + return &j, err } // Unwrap into job - j := Job{} for rows.Next() { err := rows.Scan(&j.Id, &j.Cluster, &j.Name, &j.Nodes, &j.Tasks, &j.Command) if err != nil { - return response, err + return &j, err } } + return &j, nil +} + +// addAssignments add jobs and clsuters (not assigned) to the assignment database +func (db *Database) addAssignments(job *Job, clusters []*Cluster) error { + conn, err := db.connect() + if err != nil { + return err + } + defer conn.Close() + + // The jobspec is added once to the database, first without assignment + fields := "(cluster, job, status)" + + // Assemble values, one set for each cluster + values := "" + for _, cluster := range clusters { + if values == "" { + // Status of 0 means unassigned (we don't know) + values = fmt.Sprintf("(\"%s\", \"%d\", \"%d\")", cluster.Name, job.Id, 0) + } else { + values += fmt.Sprintf(",(\"%s\", \"%d\",\"%d\")", cluster.Name, job.Id, 0) + } + } + + // Submit the query to get the global id (jobid, not submit yet) + query := fmt.Sprintf("INSERT into assign %s VALUES %s", fields, values) + + // Since we want to get a result back, we use query + statement, err := conn.Prepare(query) + if err != nil { + return err + } + defer statement.Close() + _, err = statement.Query() + return err +} + +// SubmitJob adds the job to the database +func (db *Database) SubmitJob( + job *pb.SubmitJobRequest, + clusters []*Cluster, +) (*pb.SubmitJobResponse, error) { + + response := &pb.SubmitJobResponse{} + + // Add the job to the database + // TODO: should we do a check to see if we have the job already? + // could create a hash / use the jobspec. Do we allow that? + j, err := db.addJob(job) + if err != nil { + response.Status = pb.SubmitJobResponse_SUBMIT_ERROR + return response, err + } + + // Now that the job is added to the database, add to the assignment table + err = db.addAssignments(j, clusters) + if err != nil { + response.Status = pb.SubmitJobResponse_SUBMIT_ERROR + return response, err + } // Success! response.Status = pb.SubmitJobResponse_SUBMIT_SUCCESS diff --git a/pkg/jobspec/jobspec.go b/pkg/jobspec/jobspec.go deleted file mode 100644 index 1a0cefb..0000000 --- a/pkg/jobspec/jobspec.go +++ /dev/null @@ -1,77 +0,0 @@ -package jobspec - -import ( - "fmt" - "strings" - - js "github.com/compspec/jobspec-go/pkg/jobspec/v1" -) - -// NewSimpleJobSpec generates a simple jobspec for nodes, command, tasks, and (optionally) a name -// TODO make these functions in jobspec directly -// we also want converters, from a batch job or Kubernetes abstraction, etc. -func NewSimpleJobspec(name, command string, nodes, tasks int32) (*js.Jobspec, error) { - - // If no name provided for the slot, use the first - // work of the command - if name == "" { - parts := strings.Split(command, " ") - name = strings.ToLower(parts[0]) - } - if nodes < 1 { - return nil, fmt.Errorf("nodes for the job must be >= 1") - } - if command == "" { - return nil, fmt.Errorf("a command must be provided") - } - - // The node resource is what we are asking for - nodeResource := js.Resource{ - Type: "node", - Count: nodes, - } - - // The slot is where we are doing an assessment for scheduling - slot := js.Resource{ - Type: "slot", - Count: int32(1), - Label: name, - } - - // If tasks are defined, this is total tasks across the nodes - // We add to the slot - if tasks != 0 { - taskResource := js.Resource{ - Type: "core", - Count: tasks, - } - slot.With = []js.Resource{taskResource} - } - - // And then the entire resource spec is added to the top level node resource - nodeResource.With = []js.Resource{slot} - - // Tasks reference the slot and command - // Note: if we need better split can use "github.com/google/shlex" - cmd := strings.Split(command, " ") - taskResource := []js.Tasks{ - { - Command: cmd, - Slot: name, - Count: js.Count{PerSlot: int32(1)}, - }} - - // Attributes are for the system, we aren't going to add them yet - // attributes: - // system: - // duration: 3600. - // cwd: "/home/flux" - // environment: - // HOME: "/home/flux" - // This is verison 1 as defined by v1 above - return &js.Jobspec{ - Version: 1, - Resources: []js.Resource{nodeResource}, - Tasks: taskResource, - }, nil -} diff --git a/pkg/server/endpoint.go b/pkg/server/endpoint.go index 2e83fca..38ebfb3 100644 --- a/pkg/server/endpoint.go +++ b/pkg/server/endpoint.go @@ -5,6 +5,7 @@ import ( "log" pb "github.com/converged-computing/rainbow/pkg/api/v1" + "github.com/converged-computing/rainbow/pkg/database" "github.com/converged-computing/rainbow/pkg/graph" "github.com/pkg/errors" @@ -51,18 +52,40 @@ func (s *Server) SubmitJob(_ context.Context, in *pb.SubmitJobRequest) (*pb.Subm return nil, errors.New("request is required") } - // Nogo without a token - if in.Token == "" { - return nil, errors.New("a cluster token is required") + // Keep a list of clusters to send to the database + clusters := []*database.Cluster{} + + // We submit work to one or more clusters, which must be validated via token + // This is a very simple auth setup that needs to be improved upon, but + // should work for a prototype + for _, cluster := range in.Clusters { + + // No good if no name + if cluster.Name == "" { + log.Println("warning: cluster in request is missing a name and cannot be considered") + continue + } + // No good if no token + if cluster.Token == "" { + log.Printf("warning: cluster %s does not have a token and cannot be considered\n", cluster.Name) + continue + } + + // Validate the token for the named cluster (if it exists) + cluster, err := s.db.ValidateClusterToken(cluster.Name, cluster.Token) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) } - // Validate the token for the cluster (if it exists) - cluster, err := s.db.ValidateClusterToken(in.Cluster, in.Token) - if err != nil { - return nil, err + // Only proceed if we can consider at least one cluster + if len(clusters) == 0 { + return nil, errors.New("one or more authenticated clusters are required") } - log.Printf("📝️ received job %s for cluster %s", in.Name, cluster.Name) - return s.db.SubmitJob(in, cluster) + + log.Printf("📝️ received job %s for %d contender clusters", in.Name, len(clusters)) + return s.db.SubmitJob(in, clusters) } // RequestJobs receives a cluster / instance / other receiving entity request for jobs