feat(s3): define stream and object protocol between controller and broker

clients/src/main/resources/common/message/CommitCompactObjectRequest.json

@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 74,
+  "type": "request",
+  "listeners": [
+    "controller",
+    "broker"
+  ],
+  "name": "CommitCompactObjectRequest",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "BrokerId",
+      "type": "int32",
+      "versions": "0+",
+      "entityType": "brokerId",
+      "about": "The ID of the requesting broker"
+    },
+    {
+      "name": "ObjectId",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The ID of the WAL S3 object to commit"
+    },
+    {
+      "name": "ObjectSize",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The size of the WAL S3 object to commit"
+    },
+    {
+      "name": "ObjectStreamRanges",
+      "type": "[]ObjectStreamRange",
+      "versions": "0+",
+      "about": "The stream ranges of the WAL S3 object to commit",
+      "fields": [
+        {
+          "name": "StreamId",
+          "type": "int32",
+          "versions": "0+",
+          "about": "The ID of the stream"
+        },
+        {
+          "name": "StreamEpoch",
+          "type": "int64",
+          "versions": "0+",
+          "entityType": "streamEpoch",
+          "about": "The epoch of the requesting stream in the requesting broker"
+        },
+        {
+          "name": "StartOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The start offset of the stream range"
+        },
+        {
+          "name": "EndOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The end offset of the stream range"
+        }
+      ]
+    },
+    {
+      "name": "StreamObjects",
+      "type": "[]StreamObject",
+      "versions": "0+",
+      "about": "The stream objects to commit",
+      "fields": [
+        {
+          "name": "ObjectId",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The ID of the WAL S3 object to commit"
+        },
+        {
+          "name": "ObjectSize",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The size of the WAL S3 object to commit"
+        },
+        {
+          "name": "StreamId",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The ID of the stream",
+          "entityType": "streamId"
+        },
+        {
+          "name": "StartOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The start offset of the stream range"
+        },
+        {
+          "name": "EndOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The end offset of the stream range"
+        },
+        {
+          "name": "sourceObjectIds",
+          "type": "[]int64",
+          "versions": "0+",
+          "about": "The IDs of the source S3 objects"
+        }
+      ]
+    }
+  ]
+}

clients/src/main/resources/common/message/CommitCompactObjectResponse.json

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 74,
+  "type": "response",
+  "name": "CommitCompactObjectResponse",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "ErrorCode",
+      "type": "int16",
+      "versions": "0+",
+      "about": "The top level response error code"
+    }
+  ]
+}

clients/src/main/resources/common/message/CommitStreamObjectRequest.json

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 75,
+  "type": "request",
+  "listeners": [
+    "controller",
+    "broker"
+  ],
+  "name": "CommitStreamObjectRequest",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "ObjectId",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The ID of the Stream S3 object to commit"
+    },
+    {
+      "name": "ObjectSize",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The size of the Stream S3 object to commit"
+    },
+    {
+      "name": "StreamId",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The ID of the stream",
+      "entityType": "streamId"
+    },
+    {
+      "name": "StartOffset",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The start offset of the stream range"
+    },
+    {
+      "name": "EndOffset",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The end offset of the stream range"
+    },
+    {
+      "name": "sourceObjectIds",
+      "type": "[]int64",
+      "versions": "0+",
+      "about": "The IDs of the source S3 objects"
+    }
+  ]
+}

clients/src/main/resources/common/message/CommitStreamObjectResponse.json

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 75,
+  "type": "response",
+  "name": "CommitStreamObjectResponse",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "ErrorCode",
+      "type": "int16",
+      "versions": "0+",
+      "about": "The top level response error code"
+    }
+  ]
+}

clients/src/main/resources/common/message/CommitWALObjectRequest.json

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 73,
+  "type": "request",
+  "listeners": [
+    "controller",
+    "broker"
+  ],
+  "name": "CommitWALObjectRequest",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "BrokerId",
+      "type": "int32",
+      "versions": "0+",
+      "entityType": "brokerId",
+      "about": "The ID of the requesting broker"
+    },
+    {
+      "name": "ObjectId",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The ID of the WAL S3 object to commit"
+    },
+    {
+      "name": "ObjectSize",
+      "type": "int64",
+      "versions": "0+",
+      "about": "The size of the WAL S3 object to commit"
+    },
+    {
+      "name": "ObjectStreamRanges",
+      "type": "[]ObjectStreamRange",
+      "versions": "0+",
+      "about": "The stream ranges of the WAL S3 object to commit",
+      "fields": [
+        {
+          "name": "StreamId",
+          "type": "int32",
+          "versions": "0+",
+          "about": "The ID of the stream"
+        },
+        {
+          "name": "StreamEpoch",
+          "type": "int64",
+          "versions": "0+",
+          "entityType": "streamEpoch",
+          "about": "The epoch of the requesting stream in the requesting broker"
+        },
+        {
+          "name": "StartOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The start offset of the stream range"
+        },
+        {
+          "name": "EndOffset",
+          "type": "int64",
+          "versions": "0+",
+          "about": "The end offset of the stream range"
+        }
+      ]
+    }
+  ]
+}

clients/src/main/resources/common/message/CommitWALObjectResponse.json

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+  "apiKey": 73,
+  "type": "response",
+  "name": "CommitWALObjectResponse",
+  "validVersions": "0",
+  "flexibleVersions": "0+",
+  "fields": [
+    {
+      "name": "ErrorCode",
+      "type": "int16",
+      "versions": "0+",
+      "about": "The top level response error code"
+    },
+    {
+      "name": "FailedStreamIds",
+      "type": "[]int64",
+      "versions": "0+",
+      "about": "Failed to commit WAL objects' id"
+    }
+  ]
+}