diff --git a/samples/pipeline-examples.ipynb b/samples/pipeline-examples.ipynb index 3e6de4ab10..24842db75f 100644 --- a/samples/pipeline-examples.ipynb +++ b/samples/pipeline-examples.ipynb @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "id": "f9e073d7", "metadata": {}, "outputs": [ @@ -83,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "id": "997b4028", "metadata": {}, "outputs": [ @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "id": "3d017ede", "metadata": {}, "outputs": [ @@ -146,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "id": "following-winning", "metadata": {}, "outputs": [ @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "id": "artistic-kentucky", "metadata": {}, "outputs": [ @@ -178,7 +178,7 @@ " {\r\n", " \"pipeline\": {\r\n", " \"name\": \"tfsimples\",\r\n", - " \"uid\": \"cg4617kqieos73a7nuj0\",\r\n", + " \"uid\": \"cgm2pdosogbs73emfvm0\",\r\n", " \"version\": 1,\r\n", " \"steps\": [\r\n", " {\r\n", @@ -206,7 +206,7 @@ " \"pipelineVersion\": 1,\r\n", " \"status\": \"PipelineReady\",\r\n", " \"reason\": \"created pipeline\",\r\n", - " \"lastChangeTimestamp\": \"2023-03-08T10:17:03.193598898Z\",\r\n", + " \"lastChangeTimestamp\": \"2023-04-04T13:57:11.631385497Z\",\r\n", " \"modelsReady\": true\r\n", " }\r\n", " }\r\n", @@ -221,7 +221,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "id": "87f10a5c", "metadata": {}, "outputs": [ @@ -296,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "id": "5c33133e", "metadata": {}, "outputs": [ @@ -490,7 +490,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 9, "id": "dimensional-hours", "metadata": {}, "outputs": [ @@ -508,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 10, "id": "outside-inspiration", "metadata": {}, "outputs": [ @@ -526,6 +526,391 @@ "!seldon model unload tfsimple2" ] }, + { + "cell_type": "markdown", + "id": "5c6a8e27", + "metadata": {}, + "source": [ + "### Model Chaining from inputs\n", + "\n", + "Chain the output of one model into the next. Shows using the input and outputs and combining." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "7cbf98a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Model\n", + "metadata:\n", + " name: tfsimple1\n", + "spec:\n", + " storageUri: \"gs://seldon-models/triton/simple\"\n", + " requirements:\n", + " - tensorflow\n", + " memory: 100Ki\n", + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Model\n", + "metadata:\n", + " name: tfsimple2\n", + "spec:\n", + " storageUri: \"gs://seldon-models/triton/simple\"\n", + " requirements:\n", + " - tensorflow\n", + " memory: 100Ki\n" + ] + } + ], + "source": [ + "!cat ./models/tfsimple1.yaml\n", + "!cat ./models/tfsimple2.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "f0674da8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\n", + "{}\n" + ] + } + ], + "source": [ + "!seldon model load -f ./models/tfsimple1.yaml \n", + "!seldon model load -f ./models/tfsimple2.yaml " + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "53ab8c9d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\n", + "{}\n" + ] + } + ], + "source": [ + "!seldon model status tfsimple1 -w ModelAvailable | jq -M .\n", + "!seldon model status tfsimple2 -w ModelAvailable | jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "aa3c0927", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\r\n", + "kind: Pipeline\r\n", + "metadata:\r\n", + " name: tfsimples-input\r\n", + "spec:\r\n", + " steps:\r\n", + " - name: tfsimple1\r\n", + " - name: tfsimple2\r\n", + " inputs:\r\n", + " - tfsimple1.inputs.INPUT0\r\n", + " - tfsimple1.outputs.OUTPUT1\r\n", + " tensorMap:\r\n", + " tfsimple1.outputs.OUTPUT1: INPUT1\r\n", + " output:\r\n", + " steps:\r\n", + " - tfsimple2\r\n" + ] + } + ], + "source": [ + "!cat ./pipelines/tfsimples-input.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "5c0aa3a7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline load -f ./pipelines/tfsimples-input.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "0fc44852", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"pipelineName\": \"tfsimples-input\",\r\n", + " \"versions\": [\r\n", + " {\r\n", + " \"pipeline\": {\r\n", + " \"name\": \"tfsimples-input\",\r\n", + " \"uid\": \"cgm33165u83c73dgvr00\",\r\n", + " \"version\": 1,\r\n", + " \"steps\": [\r\n", + " {\r\n", + " \"name\": \"tfsimple1\"\r\n", + " },\r\n", + " {\r\n", + " \"name\": \"tfsimple2\",\r\n", + " \"inputs\": [\r\n", + " \"tfsimple1.inputs.INPUT0\",\r\n", + " \"tfsimple1.outputs.OUTPUT1\"\r\n", + " ],\r\n", + " \"tensorMap\": {\r\n", + " \"tfsimple1.outputs.OUTPUT1\": \"INPUT1\"\r\n", + " }\r\n", + " }\r\n", + " ],\r\n", + " \"output\": {\r\n", + " \"steps\": [\r\n", + " \"tfsimple2.outputs\"\r\n", + " ]\r\n", + " },\r\n", + " \"kubernetesMeta\": {}\r\n", + " },\r\n", + " \"state\": {\r\n", + " \"pipelineVersion\": 1,\r\n", + " \"status\": \"PipelineReady\",\r\n", + " \"reason\": \"created pipeline\",\r\n", + " \"lastChangeTimestamp\": \"2023-04-04T14:17:41.667004853Z\",\r\n", + " \"modelsReady\": true\r\n", + " }\r\n", + " }\r\n", + " ]\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline status tfsimples-input -w PipelineReady| jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "904cf65a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"model_name\": \"\",\r\n", + " \"outputs\": [\r\n", + " {\r\n", + " \"data\": [\r\n", + " 1,\r\n", + " 2,\r\n", + " 3,\r\n", + " 4,\r\n", + " 5,\r\n", + " 6,\r\n", + " 7,\r\n", + " 8,\r\n", + " 9,\r\n", + " 10,\r\n", + " 11,\r\n", + " 12,\r\n", + " 13,\r\n", + " 14,\r\n", + " 15,\r\n", + " 16\r\n", + " ],\r\n", + " \"name\": \"OUTPUT0\",\r\n", + " \"shape\": [\r\n", + " 1,\r\n", + " 16\r\n", + " ],\r\n", + " \"datatype\": \"INT32\"\r\n", + " },\r\n", + " {\r\n", + " \"data\": [\r\n", + " 1,\r\n", + " 2,\r\n", + " 3,\r\n", + " 4,\r\n", + " 5,\r\n", + " 6,\r\n", + " 7,\r\n", + " 8,\r\n", + " 9,\r\n", + " 10,\r\n", + " 11,\r\n", + " 12,\r\n", + " 13,\r\n", + " 14,\r\n", + " 15,\r\n", + " 16\r\n", + " ],\r\n", + " \"name\": \"OUTPUT1\",\r\n", + " \"shape\": [\r\n", + " 1,\r\n", + " 16\r\n", + " ],\r\n", + " \"datatype\": \"INT32\"\r\n", + " }\r\n", + " ]\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline infer tfsimples-input \\\n", + " '{\"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "58a2f2ef", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"outputs\": [\r\n", + " {\r\n", + " \"name\": \"OUTPUT0\",\r\n", + " \"datatype\": \"INT32\",\r\n", + " \"shape\": [\r\n", + " \"1\",\r\n", + " \"16\"\r\n", + " ],\r\n", + " \"contents\": {\r\n", + " \"intContents\": [\r\n", + " 1,\r\n", + " 2,\r\n", + " 3,\r\n", + " 4,\r\n", + " 5,\r\n", + " 6,\r\n", + " 7,\r\n", + " 8,\r\n", + " 9,\r\n", + " 10,\r\n", + " 11,\r\n", + " 12,\r\n", + " 13,\r\n", + " 14,\r\n", + " 15,\r\n", + " 16\r\n", + " ]\r\n", + " }\r\n", + " },\r\n", + " {\r\n", + " \"name\": \"OUTPUT1\",\r\n", + " \"datatype\": \"INT32\",\r\n", + " \"shape\": [\r\n", + " \"1\",\r\n", + " \"16\"\r\n", + " ],\r\n", + " \"contents\": {\r\n", + " \"intContents\": [\r\n", + " 1,\r\n", + " 2,\r\n", + " 3,\r\n", + " 4,\r\n", + " 5,\r\n", + " 6,\r\n", + " 7,\r\n", + " 8,\r\n", + " 9,\r\n", + " 10,\r\n", + " 11,\r\n", + " 12,\r\n", + " 13,\r\n", + " 14,\r\n", + " 15,\r\n", + " 16\r\n", + " ]\r\n", + " }\r\n", + " }\r\n", + " ]\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline infer tfsimples-input --inference-mode grpc \\\n", + " '{\"model_name\":\"simple\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "1cd53dd6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline unload tfsimples-input" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "91cce2e0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\n", + "{}\n" + ] + } + ], + "source": [ + "!seldon model unload tfsimple1\n", + "!seldon model unload tfsimple2" + ] + }, { "cell_type": "markdown", "id": "fca716ed", diff --git a/samples/pipeline-examples.md b/samples/pipeline-examples.md index 696f92fe39..6fff7081af 100644 --- a/samples/pipeline-examples.md +++ b/samples/pipeline-examples.md @@ -108,7 +108,7 @@ seldon pipeline status tfsimples -w PipelineReady| jq -M . { "pipeline": { "name": "tfsimples", - "uid": "cg4617kqieos73a7nuj0", + "uid": "cgm2pdosogbs73emfvm0", "version": 1, "steps": [ { @@ -136,7 +136,7 @@ seldon pipeline status tfsimples -w PipelineReady| jq -M . "pipelineVersion": 1, "status": "PipelineReady", "reason": "created pipeline", - "lastChangeTimestamp": "2023-03-08T10:17:03.193598898Z", + "lastChangeTimestamp": "2023-04-04T13:57:11.631385497Z", "modelsReady": true } } @@ -386,6 +386,295 @@ seldon model unload tfsimple2 ``` +### Model Chaining from inputs + +Chain the output of one model into the next. Shows using the input and outputs and combining. + +```bash +cat ./models/tfsimple1.yaml +cat ./models/tfsimple2.yaml +``` + +```yaml +apiVersion: mlops.seldon.io/v1alpha1 +kind: Model +metadata: + name: tfsimple1 +spec: + storageUri: "gs://seldon-models/triton/simple" + requirements: + - tensorflow + memory: 100Ki +apiVersion: mlops.seldon.io/v1alpha1 +kind: Model +metadata: + name: tfsimple2 +spec: + storageUri: "gs://seldon-models/triton/simple" + requirements: + - tensorflow + memory: 100Ki + +``` + +```bash +seldon model load -f ./models/tfsimple1.yaml +seldon model load -f ./models/tfsimple2.yaml +``` + +```json +{} +{} + +``` + +```bash +seldon model status tfsimple1 -w ModelAvailable | jq -M . +seldon model status tfsimple2 -w ModelAvailable | jq -M . +``` + +```json +{} +{} + +``` + +```bash +cat ./pipelines/tfsimples-input.yaml +``` + +```yaml +apiVersion: mlops.seldon.io/v1alpha1 +kind: Pipeline +metadata: + name: tfsimples-input +spec: + steps: + - name: tfsimple1 + - name: tfsimple2 + inputs: + - tfsimple1.inputs.INPUT0 + - tfsimple1.outputs.OUTPUT1 + tensorMap: + tfsimple1.outputs.OUTPUT1: INPUT1 + output: + steps: + - tfsimple2 + +``` + +```bash +seldon pipeline load -f ./pipelines/tfsimples-input.yaml +``` + +```json +{} + +``` + +```bash +seldon pipeline status tfsimples-input -w PipelineReady| jq -M . +``` + +```json +{ + "pipelineName": "tfsimples-input", + "versions": [ + { + "pipeline": { + "name": "tfsimples-input", + "uid": "cgm33165u83c73dgvr00", + "version": 1, + "steps": [ + { + "name": "tfsimple1" + }, + { + "name": "tfsimple2", + "inputs": [ + "tfsimple1.inputs.INPUT0", + "tfsimple1.outputs.OUTPUT1" + ], + "tensorMap": { + "tfsimple1.outputs.OUTPUT1": "INPUT1" + } + } + ], + "output": { + "steps": [ + "tfsimple2.outputs" + ] + }, + "kubernetesMeta": {} + }, + "state": { + "pipelineVersion": 1, + "status": "PipelineReady", + "reason": "created pipeline", + "lastChangeTimestamp": "2023-04-04T14:17:41.667004853Z", + "modelsReady": true + } + } + ] +} + +``` + +```bash +seldon pipeline infer tfsimples-input \ + '{"inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}' | jq -M . +``` + +```json +{ + "model_name": "", + "outputs": [ + { + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "name": "OUTPUT0", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + }, + { + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "name": "OUTPUT1", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + } + ] +} + +``` + +```bash +seldon pipeline infer tfsimples-input --inference-mode grpc \ + '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' | jq -M . +``` + +```json +{ + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + "1", + "16" + ], + "contents": { + "intContents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + } + }, + { + "name": "OUTPUT1", + "datatype": "INT32", + "shape": [ + "1", + "16" + ], + "contents": { + "intContents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + } + } + ] +} + +``` + +```bash +seldon pipeline unload tfsimples-input +``` + +```json +{} + +``` + +```bash +seldon model unload tfsimple1 +seldon model unload tfsimple2 +``` + +```json +{} +{} + +``` + ### Model Join Join two flows of data from two models as input to a third model. This shows how individual flows of data can be combined. diff --git a/samples/pipelines/tfsimples-input.yaml b/samples/pipelines/tfsimples-input.yaml new file mode 100644 index 0000000000..d96c5926d8 --- /dev/null +++ b/samples/pipelines/tfsimples-input.yaml @@ -0,0 +1,16 @@ +apiVersion: mlops.seldon.io/v1alpha1 +kind: Pipeline +metadata: + name: tfsimples-input +spec: + steps: + - name: tfsimple1 + - name: tfsimple2 + inputs: + - tfsimple1.inputs.INPUT0 + - tfsimple1.outputs.OUTPUT1 + tensorMap: + tfsimple1.outputs.OUTPUT1: INPUT1 + output: + steps: + - tfsimple2 diff --git a/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/Joiner.kt b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/Joiner.kt index 52df7029ac..a76a38b5a1 100644 --- a/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/Joiner.kt +++ b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/Joiner.kt @@ -192,8 +192,13 @@ class Joiner( if (right == null) { return left } - val leftRequest = V2Dataplane.ModelInferRequest.parseFrom(left) - val rightRequest = V2Dataplane.ModelInferRequest.parseFrom(right) + var leftRequest = V2Dataplane.ModelInferRequest.parseFrom(left) + var rightRequest = V2Dataplane.ModelInferRequest.parseFrom(right) + if (leftRequest.rawInputContentsCount > 0 && rightRequest.rawInputContentsCount == 0) { + rightRequest = convertRequestToRawInputContents(rightRequest) + } else if (rightRequest.rawInputContentsCount > 0 && leftRequest.rawInputContentsCount == 0) { + leftRequest = convertRequestToRawInputContents(leftRequest) + } val request = V2Dataplane.ModelInferRequest .newBuilder() .setId(leftRequest.id) @@ -213,8 +218,13 @@ class Joiner( if (right == null) { return left } - val leftResponse = V2Dataplane.ModelInferResponse.parseFrom(left) - val rightResponse = V2Dataplane.ModelInferResponse.parseFrom(right) + var leftResponse = V2Dataplane.ModelInferResponse.parseFrom(left) + var rightResponse = V2Dataplane.ModelInferResponse.parseFrom(right) + if (leftResponse.rawOutputContentsCount > 0 && rightResponse.rawOutputContentsCount == 0) { + rightResponse = convertResponseToRawOutputContents(rightResponse) + } else if (rightResponse.rawOutputContentsCount > 0 && leftResponse.rawOutputContentsCount == 0) { + leftResponse = convertResponseToRawOutputContents(leftResponse) + } val response = V2Dataplane.ModelInferResponse .newBuilder() .setId(leftResponse.id) diff --git a/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/StreamTransforms.kt b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/StreamTransforms.kt index 77402aceba..f24eb9dfdb 100644 --- a/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/StreamTransforms.kt +++ b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/StreamTransforms.kt @@ -23,7 +23,6 @@ import io.seldon.mlops.chainer.ChainerOuterClass.PipelineTensorMapping import io.seldon.mlops.chainer.ChainerOuterClass.Batch import io.seldon.mlops.inference.v2.V2Dataplane.ModelInferRequest import io.seldon.mlops.inference.v2.V2Dataplane.ModelInferResponse -import jdk.incubator.vector.VectorOperators.Test import org.apache.kafka.streams.kstream.KStream import org.apache.kafka.streams.kstream.ValueTransformerSupplier @@ -94,7 +93,7 @@ fun KStream.batchMessages(batchProperties: Batch): KS } /** - * Convert the output from one model (a response) to the input for another model (a request). + * Convert the output from one model (a response) to the input of another model (a request). */ private fun convertToRequest( response: ModelInferResponse, @@ -308,7 +307,7 @@ fun KStream.convertToResponse( } /** - * Convert the output from one model (a response) to the input for another model (a request). + * Convert the input from one model (a request) to the output for another model (a response). */ private fun convertToResponse( request: ModelInferRequest, @@ -359,4 +358,4 @@ private fun convertInputToOutputTensor( req.setContents(input.contents) } return req.build() -} \ No newline at end of file +} diff --git a/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/protocol.kt b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/protocol.kt new file mode 100644 index 0000000000..a4958d047c --- /dev/null +++ b/scheduler/data-flow/src/main/kotlin/io/seldon/dataflow/kafka/protocol.kt @@ -0,0 +1,274 @@ +package io.seldon.dataflow.kafka + +import com.google.protobuf.kotlin.toByteString +import io.seldon.mlops.inference.v2.V2Dataplane +import java.lang.UnsupportedOperationException +import java.nio.ByteBuffer +import java.nio.ByteOrder + +enum class DataType { + BOOL, UINT8, UINT16, UINT32, UINT64, INT8, INT16, INT32, INT64, FP16, FP32, FP64, BYTES +} + +fun convertRequestToRawInputContents(request: V2Dataplane.ModelInferRequest): V2Dataplane.ModelInferRequest { + val builder = request.toBuilder() + request.inputsList.forEachIndexed { idx, input -> + val v = when (DataType.valueOf(input.datatype)) { + DataType.UINT8 -> { + input.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(1) + .put(it.toByte()) + .array() + .toList() + }.toByteArray() + } + DataType.UINT16 -> { + input.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(UShort.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putShort(it.toShort()) + .array() + .toList() + }.toByteArray() + } + DataType.UINT32 -> { + input.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(UInt.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it) + .array() + .toList() + }.toByteArray() + } + DataType.UINT64 -> { + input.contents.uint64ContentsList.flatMap { + ByteBuffer + .allocate(ULong.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(it) + .array() + .toList() + }.toByteArray() + } + DataType.INT8 -> { + input.contents.intContentsList.flatMap { + ByteBuffer + .allocate(1) + .put(it.toByte()) + .array() + .toList() + }.toByteArray() + } + DataType.INT16 -> { + input.contents.intContentsList.flatMap { + ByteBuffer + .allocate(Short.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putShort(it.toShort()) + .array() + .toList() + }.toByteArray() + } + DataType.INT32 -> { + input.contents.intContentsList.flatMap { + ByteBuffer + .allocate(Int.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it) + .array() + .toList() + }.toByteArray() + } + DataType.INT64 -> { + input.contents.int64ContentsList.flatMap { + ByteBuffer + .allocate(Long.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(it) + .array() + .toList() + }.toByteArray() + } + DataType.BOOL -> { + input.contents.boolContentsList.flatMap { + ByteBuffer + .allocate(1) + .put(if (it) {1} else {0}) + .array() + .toList() + }.toByteArray() + } + DataType.FP16, // may need to handle this separately in future + DataType.FP32 -> { + input.contents.fp32ContentsList.flatMap { + ByteBuffer + .allocate(Float.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putFloat(it) + .array() + .toList() + }.toByteArray() + } + DataType.FP64 -> { + input.contents.fp64ContentsList.flatMap { + ByteBuffer + .allocate(Double.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putDouble(it) + .array() + .toList() + }.toByteArray() + } + DataType.BYTES -> { + input.contents.bytesContentsList.flatMap { + ByteBuffer + .allocate(it.size() + Int.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it.size()) + .put(it.toByteArray()) + .array() + .toList() + }.toByteArray() + } + } + // Add raw contents + builder.addRawInputContents(v.toByteString()) + // Clear the contents now we have added the raw inputs + builder.getInputsBuilder(idx).clearContents() + } + return builder.build() +} + +fun convertResponseToRawOutputContents(request: V2Dataplane.ModelInferResponse): V2Dataplane.ModelInferResponse { + val builder = request.toBuilder() + request.outputsList.forEachIndexed { idx, output -> + val v = when (DataType.valueOf(output.datatype)) { + DataType.UINT8 -> { + output.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(1) + .put(it.toByte()) + .array() + .toList() + }.toByteArray() + } + DataType.UINT16 -> { + output.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(UShort.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putShort(it.toShort()) + .array() + .toList() + }.toByteArray() + } + DataType.UINT32 -> { + output.contents.uintContentsList.flatMap { + ByteBuffer + .allocate(UInt.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it) + .array() + .toList() + }.toByteArray() + } + DataType.UINT64 -> { + output.contents.uint64ContentsList.flatMap { + ByteBuffer + .allocate(ULong.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(it) + .array() + .toList() + }.toByteArray() + } + DataType.INT8 -> { + output.contents.intContentsList.flatMap { + ByteBuffer + .allocate(1) + .put(it.toByte()) + .array() + .toList() + }.toByteArray() + } + DataType.INT16 -> { + output.contents.intContentsList.flatMap { + ByteBuffer + .allocate(Short.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putShort(it.toShort()) + .array() + .toList() + }.toByteArray() + } + DataType.INT32 -> { + output.contents.intContentsList.flatMap { + ByteBuffer + .allocate(Int.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it) + .array() + .toList() + }.toByteArray() + } + DataType.INT64 -> { + output.contents.int64ContentsList.flatMap { + ByteBuffer + .allocate(Long.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(it) + .array() + .toList() + }.toByteArray() + } + DataType.BOOL -> { + output.contents.boolContentsList.flatMap { + ByteBuffer.allocate(1) + .put(if (it) {1} else {0}) + .array().toList() + }.toByteArray() + } + DataType.FP16, // may need to handle this separately in future + DataType.FP32 -> { + output.contents.fp32ContentsList.flatMap { + ByteBuffer + .allocate(Float.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putFloat(it) + .array() + .toList() + }.toByteArray() + } + DataType.FP64 -> { + output.contents.fp64ContentsList.flatMap { + ByteBuffer + .allocate(Double.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putDouble(it) + .array() + .toList() + }.toByteArray() + } + DataType.BYTES -> { + output.contents.bytesContentsList.flatMap { + ByteBuffer + .allocate(it.size() + Int.SIZE_BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(it.size()) + .put(it.toByteArray()) + .array() + .toList() + }.toByteArray() + } + } + // Add raw contents + builder.addRawOutputContents(v.toByteString()) + // Clear the contents now we have added the raw outputs + builder.getOutputsBuilder(idx).clearContents() + } + return builder.build() +} +