-
Notifications
You must be signed in to change notification settings - Fork 2
/
ensign.proto
236 lines (206 loc) · 9.23 KB
/
ensign.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
syntax = "proto3";
package ensign.v1beta1;
import "api/v1beta1/event.proto";
import "api/v1beta1/topic.proto";
import "api/v1beta1/groups.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";
// The Ensign service is meant to allow publishers (producers) and subscribers
// (consumers) of events to interact with the Ensign eventing system; e.g. this is a
// user-oriented API that is the basis of the user SDKs that we will build. There are
// two primary interactions that the user client may have: publishing or subscribing to
// topics to send and receive events or managing topics that are available.
service Ensign {
// Both the Publish and Subscribe RPCs are bidirectional streaming to allow for acks
// and nacks of events to be sent between Ensign and the client. The Publish stream
// is opened and the client sends events and receives acks/nacks -- when the client
// closes the publish stream, the server sends back information about the current
// state of the topic. When the Subscribe stream is opened, the client must send an
// open stream message with the subscription info before receiving events. Once it
// receives events it must send back acks/nacks up the stream so that Ensign
// advances the topic offset for the rest of the clients in the group.
rpc Publish(stream PublisherRequest) returns (stream PublisherReply) {}
rpc Subscribe(stream SubscribeRequest) returns (stream SubscribeReply) {}
// This is a simple topic management interface. Right now we assume that topics are
// immutable, therefore there is no update topic RPC call. There are two ways to
// delete a topic - archiving it makes the topic readonly so that no events can be
// published to it, but it can still be read. Destroying the topic deletes it and
// removes all of its data, freeing up the topic name to be used again.
rpc ListTopics(PageInfo) returns (TopicsPage) {}
rpc CreateTopic(Topic) returns (Topic) {}
rpc RetrieveTopic(Topic) returns (Topic) {}
rpc DeleteTopic(TopicMod) returns (TopicTombstone) {}
rpc TopicNames(PageInfo) returns (TopicNamesPage) {}
rpc TopicExists(TopicName) returns (TopicExistsInfo) {}
// Info provides statistics and metrics describing the state of a project
rpc Info(InfoRequest) returns (ProjectInfo) {}
// Implements a client-side heartbeat that can also be used by monitoring tools.
rpc Status(HealthCheck) returns (ServiceState) {}
}
// PublisherRequest messages are sent from the publisher to the server. Generally they
// are events that need to be published but the first message must be a stream
// initialization message to ensure that the publisher is authenticated and can identify
// itself. This message can be extended to allow for other interactions between the
// publisher and the server including topic queries, redirects, etc.
message PublisherRequest {
oneof embed {
EventWrapper event = 1;
OpenStream open_stream = 2;
}
}
// PublisherReply messages are sent back to publishers from the server. Generally they
// are responses to receiving events (e.g. ack and nack) but the last message contains
// information about the performance of the publisher and the topic itself.
// This message can be extended to allow more general interactions between the publisher
// and the server including redirects, topic queries, etc.
message PublisherReply {
oneof embed {
Ack ack = 1;
Nack nack = 2;
StreamReady ready = 3;
CloseStream close_stream = 4;
}
}
// SubscribeRequest messages are sent to the server from subscribers. Generally they are
// responses to receiving events (e.g. ack and nack) but the first message must contain
// subscription information about the topic and the group so that Ensign can start
// sending the client events from the specified topic down the stream.
message SubscribeRequest {
oneof embed {
Ack ack = 1;
Nack nack = 2;
Subscription subscription = 3;
}
}
// SubscribeReply messages are sent to the subscriber from the server. In most cases
// this message is an event that matches the subscription information. However, this
// message can also contain control messages such as redirects, topic information, etc.
message SubscribeReply {
oneof embed {
EventWrapper event = 1;
StreamReady ready = 2;
CloseStream close_stream = 3;
}
}
// Ack represents the receipt and final handling of an event. This datatype should be
// small so that throughput is not affected and generally only contains the ID of the
// event being acknowledged. When Ensign commits an event to the log from the producer,
// the commit timestamp is returned to help determine event latency. When clients ack an
// event back to the Ensign server, they only need contain the id.
message Ack {
bytes id = 1;
google.protobuf.Timestamp committed = 2;
}
// Nack means that an event could not be handled or committed. This datatype should be
// small so that throughput is not affected and generally only conains the id of the
// event and the error code describing what went wrong. Longer error messages are
// optional and should only be used when something abnormal has occurred. The Ensign
// server will return a Nack if the event could not be appended to the log. Clients
// should return a Nack if the event couldn't be handled or processed so that Ensign
// ensures another client retrieves the event.
message Nack {
enum Code {
UNKNOWN = 0;
// Server-side NACK codes.
MAX_EVENT_SIZE_EXCEEDED = 1;
TOPIC_UKNOWN = 2;
TOPIC_ARCHVIVED = 3;
TOPIC_DELETED = 4;
PERMISSION_DENIED = 5;
CONSENSUS_FAILURE = 6;
SHARDING_FAILURE = 7;
REDIRECT = 8;
INTERNAL = 9;
// Client-side NACK codes
UNPROCESSED = 100;
TIMEOUT = 101;
UNHANDLED_MIMETYPE = 102;
UNKNOWN_TYPE = 103;
DELIVER_AGAIN_ANY = 104;
DELIVER_AGAIN_NOT_ME = 105;
}
bytes id = 1;
Code code = 2;
string error = 3;
}
// OpenStream is the first message that should be sent in a Publish stream in order to
// identify and authenticate the publisher. Optionally, the publisher can specify the
// topics that they want to publish to in order to check that they are being sent to the
// correct node.
message OpenStream {
string client_id = 1;
repeated string topics = 2;
}
// CloseStream returns some basic stats and topic information to the publisher or
// subscriber when the stream is closed and provides feedback that the stream was closed
// successfully.
message CloseStream {
uint64 events = 1;
uint64 topics = 2;
uint64 consumers = 3;
}
// Sent in response to an OpenStream or Subscription message so that the client knows
// it can start sending or receiving events from the stream.
message StreamReady {
string client_id = 1; // repeated back to the client for sanity
string server_id = 2; // the node that the stream is conneced to
// Maps the topic name to the topic ID (ULID bytes) that are available on this node.
// TODO: handle topic redirects to other nodes.
map<string,bytes> topics = 3;
}
// Subscription is used to initialize a subscribe stream so that the Ensign node returns
// the correct events to the subscriber based on the query or the topics they request.
message Subscription {
string client_id = 1;
repeated string topics = 2;
string query = 3;
ConsumerGroup group = 4;
}
// InfoRequest allows the project info to be filtered by a list of specific topics.
message InfoRequest {
repeated bytes topics = 1;
}
// ProjectInfo describes overall project statistics for the project described in the
// authentication claims that the user connects with.
message ProjectInfo {
string project_id = 1;
uint64 topics = 2;
uint64 readonly_topics = 3;
uint64 events = 4;
}
// HealthCheck is used to query the service state of an Ensign node.
message HealthCheck {
// The number of failed health checks that proceeded the current check.
uint32 attempts = 1;
// The timestamp of the last health check, successful or otherwise.
google.protobuf.Timestamp last_checked_at = 2;
}
// ServiceState describes the health status of the Ensign node and can be used for
// heartbeats and monitoring.
message ServiceState {
enum Status {
UNKNOWN = 0;
HEALTHY = 1;
UNHEALTHY = 2;
DANGER = 3;
OFFLINE = 4;
MAINTENANCE = 5;
}
// Current service status as defined by the recieving system. The system is obliged
// to respond with the closest matching status in a best-effort fashion. Alerts will
// be triggered on service status changes if the system does not respond and the
// previous system state was not unknown.
Status status = 1;
// The current version of the node running the Ensign service
string version = 2;
// How long the node has been up and running since it was last rebooted
google.protobuf.Duration uptime = 3;
// Hint to the client when to check the health status again.
google.protobuf.Timestamp not_before = 4;
google.protobuf.Timestamp not_after = 5;
}
// A basic request for paginated list queries.
message PageInfo {
uint32 page_size = 1;
string next_page_token = 2;
}