forked from pingcap/kvproto
-
Notifications
You must be signed in to change notification settings - Fork 0
/
import_sstpb.proto
380 lines (308 loc) · 11.2 KB
/
import_sstpb.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
syntax = "proto3";
package import_sstpb;
import "metapb.proto";
import "errorpb.proto";
import "kvrpcpb.proto";
import "gogoproto/gogo.proto";
import "rustproto.proto";
import "brpb.proto";
option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
option (rustproto.lite_runtime_all) = true;
option java_package = "org.tikv.kvproto";
// ImportSST provides a service to import a generated SST file to a region in TiKV.
//
// In order to import an SST file to a region, the user should:
// 1. Retrieve the meta of the region according to the SST file's range.
// 2. Upload the SST file to the servers where the region's peers locate in.
// 3. Issue an ingest request to the region's leader with the SST file's metadata.
//
// It's the user's responsibility to make sure that the SST file is uploaded to
// the servers where the region's peers locate in, before issue the ingest
// request to the region's leader. However, the region can be scheduled (so the
// location of the region's peers will be changed) or split/merged (so the range
// of the region will be changed), after the SST file is uploaded, but before
// the SST file is ingested. So, the region's epoch is provided in the SST
// file's metadata, to guarantee that the region's epoch must be the same
// between the SST file is uploaded and ingested later.
service ImportSST {
// Switch to normal/import mode.
rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {}
// Upload an SST file to a server.
rpc Upload(stream UploadRequest) returns (UploadResponse) {}
// Ingest an uploaded SST file to a region.
rpc Ingest(IngestRequest) returns (IngestResponse) {}
// Compact the specific range for better performance.
rpc Compact(CompactRequest) returns (CompactResponse) {}
rpc SetDownloadSpeedLimit(SetDownloadSpeedLimitRequest) returns (SetDownloadSpeedLimitResponse) {}
// Download an SST file from an external storage, and performs key-rewrite
// after downloading.
rpc Download(DownloadRequest) returns (DownloadResponse) {}
// Open a write stream to generate sst files
rpc Write(stream WriteRequest) returns (WriteResponse) {}
rpc RawWrite(stream RawWriteRequest) returns (RawWriteResponse) {}
// Ingest Multiple files in one request
rpc MultiIngest(MultiIngestRequest) returns (IngestResponse) {}
// Collect duplicate data from TiKV.
rpc DuplicateDetect(DuplicateDetectRequest) returns (stream DuplicateDetectResponse) {}
// Apply download & apply increment kv files to TiKV.
rpc Apply(ApplyRequest) returns (ApplyResponse) {}
// ClearFiles clear applied file after restore succeed.
rpc ClearFiles(ClearRequest) returns (ClearResponse) {}
}
enum SwitchMode {
Normal = 0;
Import = 1;
}
message SwitchModeRequest {
SwitchMode mode = 1;
}
message SwitchModeResponse {
}
message Range {
bytes start = 1;
bytes end = 2;
}
message SSTMeta {
bytes uuid = 1;
Range range = 2;
uint32 crc32 = 3;
uint64 length = 4;
string cf_name = 5;
uint64 region_id = 6;
metapb.RegionEpoch region_epoch = 7;
bool end_key_exclusive = 8;
// total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size
// set these values can save time from tikv upload keys and size to PD through Heartbeat.
uint64 total_kvs = 9;
uint64 total_bytes = 10;
// API version implies the encode of the key and value.
kvrpcpb.APIVersion api_version = 11;
// cipher_iv is used to encrypt/decrypt sst
bytes cipher_iv = 12;
}
// A rewrite rule is applied on the *encoded* keys (the internal storage
// representation).
message RewriteRule {
bytes old_key_prefix = 1;
bytes new_key_prefix = 2;
uint64 new_timestamp = 3;
}
message UploadRequest {
oneof chunk {
SSTMeta meta = 1;
bytes data = 2;
}
}
message UploadResponse {
}
message IngestRequest {
kvrpcpb.Context context = 1;
SSTMeta sst = 2;
}
message MultiIngestRequest {
kvrpcpb.Context context = 1;
repeated SSTMeta ssts = 2;
}
message IngestResponse {
errorpb.Error error = 1;
}
message CompactRequest {
// Compact files in the range and above the output level.
// Compact all files if the range is not specified.
// Compact all files to the bottommost level if the output level is -1.
Range range = 1;
int32 output_level = 2;
}
message CompactResponse {
}
message DownloadRequest {
// The SST meta used to identify the downloaded file.
// Must be the same among all nodes in the same Raft group.
// Note: the "crc32" and "cf_name" fields are ignored in this request,
// and the "range" field represents the closed key range after rewrite
// (as origin keys in encoded representation).
SSTMeta sst = 2 [(gogoproto.nullable) = false];
// The url field is deprecated, use storage_backend instead
reserved 8; reserved "url";
// The file name of the SST file.
string name = 9;
// Performs a key prefix rewrite after downloading the SST file.
// All keys in the SST will be rewritten as:
//
// new_key = new_key_prefix + old_key[len(old_key_prefix)..]
//
// When used for TiDB, rewriting the prefix changes the table ID. Please
// note that key-rewrite is applied on the origin keys in encoded
// representation (the SST itself should still use data keys in encoded
// representation).
//
// You need to ensure that the keys before and after rewriting are in the
// same order, otherwise the RPC request will fail.
RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false];
backup.StorageBackend storage_backend = 14;
bool is_raw_kv = 15;
// cipher_info is used to decrypt sst when download sst
backup.CipherInfo cipher_info = 16;
}
// For now it is just used for distinguishing the error of the request with the error
// of gRPC, add more concrete types if it is necessary later.
message Error {
string message = 1;
// We meet some internal errors of the store.
errorpb.Error store_error = 2;
}
message DownloadResponse {
// The actual key range (after rewrite) of the downloaded SST. The range is
// inclusive in both ends.
Range range = 1 [(gogoproto.nullable) = false];
// Whether the SST is empty. An empty SST is prohibited in TiKV, do not
// ingest if this field is true.
// (Deprecated, should be replaced by checking `length == 0` in the future)
bool is_empty = 2;
Error error = 3;
// The CRC32 checksum of the rewritten SST file (implementation can return
// zero, indicating the CRC32 was not calculated).
uint32 crc32 = 4;
// The actual length of the rewritten SST file.
uint64 length = 5;
}
message SetDownloadSpeedLimitRequest {
// The download speed limit (bytes/second). Set to 0 for unlimited speed.
uint64 speed_limit = 1;
}
message SetDownloadSpeedLimitResponse {
}
message Pair {
bytes key = 1;
bytes value = 2;
enum OP {
Put = 0;
Delete = 1;
}
OP op = 3;
}
message WriteBatch {
uint64 commit_ts = 1;
repeated Pair pairs = 2;
}
message WriteRequest {
oneof chunk {
SSTMeta meta = 1;
WriteBatch batch = 2;
}
}
message WriteResponse {
Error error = 1;
repeated SSTMeta metas = 2;
}
message RawWriteBatch {
uint64 ttl = 1;
repeated Pair pairs = 2;
// To be compatible with the key encoding of API V2.
// This field should be generated from the client instead of the server,
// since the message will be send to all the replicas of a region.
// Otherwise, the underlying data generated by the server would be inconsistent which is hard to scale
// for other features like MVCC over RawKV.
uint64 ts = 3;
}
message RawWriteRequest {
oneof chunk {
SSTMeta meta = 1;
RawWriteBatch batch = 2;
}
}
message RawWriteResponse {
Error error = 1;
repeated SSTMeta metas = 2;
}
message DuplicateDetectRequest {
kvrpcpb.Context context = 1;
bytes start_key = 2;
bytes end_key = 3;
// Return only the keys found by scanning, not their values.
bool key_only = 4;
// We only check the data whose timestamp is larger than `min_commit_ts`. `min_commit_ts` is exclueded.
uint64 min_commit_ts = 5;
}
message KvPair {
bytes key = 1;
bytes value = 2;
uint64 commit_ts = 3;
}
message DuplicateDetectResponse {
errorpb.Error region_error = 1;
Error key_error = 2;
// The these keys will be in asc order (but commit time is in desc order),
// and the content is just like following:
// [
// {key: "key1", value: "value11", commit_ts: 1005},
// {key: "key1", value: "value12", commit_ts: 1004},
// {key: "key1", value: "value13", commit_ts: 1001},
// {key: "key2", value: "value21", commit_ts: 1004},
// {key: "key2", value: "value22", commit_ts: 1002},
// ...
// ]
repeated KvPair pairs = 3;
}
message KVMeta {
// The file name of the KV file.
string name = 1;
// file offset, sometimes only need to get a part of data from the merged file
uint64 offset = 11;
// file length for check.
uint64 length = 2;
// file length after compressed
uint64 compress_length = 12;
// tell us which cf should apply. WRITE_CF or DEFAULT_CF e.g.
string cf = 3;
// is_delete represents whether we should delete the kv in tikv.
// it may not be too much delete file. only rollBack operation will generate delete kv file.
bool is_delete = 4;
// the key ts space being smaller than start_ts can be filter.
uint64 start_ts = 10;
// the key ts space large than restore_ts can be filter.
uint64 restore_ts = 5;
bytes start_key = 6;
bytes end_key = 7;
// used for checksum when download kv file.
bytes sha256 = 8;
// the key ts space less than start_snapshot_ts can be filter.
// Deprecated: this field 'start_snapshot_ts' is replaced by the field 'start_ts'.
uint64 start_snapshot_ts = 9;
}
message ApplyRequest {
// The meta of the KV file.
KVMeta meta = 1;
// Performs a key prefix rewrite after downloading the file.
// All keys in the files will be rewritten as:
//
// new_key = new_key_prefix + old_key[len(old_key_prefix)..]
//
// When used for TiDB, rewriting the prefix changes the table ID. Please
// note that key-rewrite is applied on the origin keys in encoded
// representation.
//
// You need to ensure that the keys before and after rewriting are in the
// same order, otherwise the RPC request will fail.
RewriteRule rewrite_rule = 2 [(gogoproto.nullable) = false];
backup.StorageBackend storage_backend = 3;
// context represents region info and it used to build raft commands.
kvrpcpb.Context context = 4;
// cipher_info is used to decrypt kv file when download file.
backup.CipherInfo cipher_info = 11;
}
message ApplyResponse {
// The actual key range (after rewrite) of the downloaded file. The range is
// inclusive in both ends.
Range range = 1 [(gogoproto.nullable) = false];
Error error = 2;
}
message ClearRequest {
// clear files in import directory with given prefix.
string prefix = 1;
}
message ClearResponse {
Error error = 1;
}