Mkulakow/mediapipe kfs rest test

src/http_rest_api_handler.cpp

@@ -265,6 +265,8 @@ static bool isInputEmpty(const ::KFSRequest::InferInputTensor& input) {
         return input.contents().fp64_contents_size() == 0;
     if (input.datatype() == "BYTES")
         return input.contents().bytes_contents_size() == 0;
+    if (input.datatype() == "BOOL")
+        return input.contents().bool_contents_size() == 0;
     return true;
 }
 
@@ -317,7 +319,6 @@ static Status handleBinaryInputs(::KFSRequest& grpc_request, const std::string&
                 binary_input_size = calculateBinaryDataSize(*input);
             }
         }
-
         auto status = handleBinaryInput(binary_input_size, binary_input_offset, binary_buffer_size, binary_inputs_buffer, *input, grpc_request.add_raw_input_contents());
         if (!status.ok())
             return status;

src/mediapipe_internal/mediapipegraphexecutor.cpp

@@ -383,6 +383,9 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             case TFSDataType::DT_FLOAT: {
                 COPY_INPUT_VALUE_BY_VALUE(float, fp32);
             }
+            case TFSDataType::DT_DOUBLE: {
+                COPY_INPUT_VALUE_BY_VALUE(double, fp64);
+            }
             case TFSDataType::DT_INT64: {
                 COPY_INPUT_VALUE_BY_VALUE(int64_t, int64);
             }
@@ -410,7 +413,6 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             case TFSDataType::DT_BOOL: {
                 COPY_INPUT_VALUE_BY_VALUE(bool, bool);
             }
-            case TFSDataType::DT_DOUBLE:
             case TFSDataType::DT_HALF:
             default:
                 return ovms::Status(ovms::StatusCode::NOT_IMPLEMENTED, "There is no support for types different than fp32, int64, int32, uint32, uint64, int8, uint8, bool");
@@ -455,10 +457,10 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             }
         } else {
             OVMS_RETURN_ON_FAIL(validateInputContent(*requestInputItr, expectedBytes, requestedName, request));
-            outTensor = std::make_unique<ov::Tensor>(precision, shape);
             if (expectedBytes == 0) {
                 return StatusCode::OK;
             }
+            outTensor = std::make_unique<ov::Tensor>(precision, shape);
             void* data = outTensor->data();
             switch (precision) {
             case ov::element::Type_t::f32: {
@@ -491,12 +493,14 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             case ov::element::Type_t::boolean: {
                 COPY_INPUT_VALUE_BY_VALUE(bool, bool);
             }
+            case ov::element::Type_t::f64: {
+                COPY_INPUT_VALUE_BY_VALUE(double, fp64);
+            }
             // the rest not supported by KFS
             case ov::element::Type_t::u1:
             case ov::element::Type_t::u4:
             case ov::element::Type_t::i4:
             case ov::element::Type_t::f16:
-            case ov::element::Type_t::f64:
             case ov::element::Type_t::bf16:
             case ov::element::Type_t::undefined:
             case ov::element::Type_t::dynamic:
@@ -684,25 +688,37 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
                 return StatusCode::UNKNOWN_ERROR;
             }
         } else {
-            if (formatIt == datatypeToBufferFormat.end()) {
+            if ((precision != ov::element::Type_t::string) && formatIt == datatypeToBufferFormat.end()) {
                 const std::string details = "Provided datatype is invalid, custom datatypes are allowed only when raw_input_contents is used.";
                 SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details);
                 return Status(StatusCode::INVALID_PRECISION, details);
             }
-            size_t expectedBytes = 1;
-            bool expectedBufferSizeValid = computeExpectedBufferSizeReturnFalseIfOverflow<py::ssize_t>(shape, precision.size(), expectedBytes);
-            if (!expectedBufferSizeValid) {
-                const std::string details = "Provided shape and datatype declare too large buffer.";
-                SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details);
-                return Status(StatusCode::INVALID_CONTENT_SIZE, details);
+            size_t expectedBytes;
+            if (precision == ov::element::Type_t::string) {
+                expectedBytes = 0;
+                for (auto contents : request.inputs(inputIndex).contents().bytes_contents()) {
+                    expectedBytes += contents.size() + sizeof(uint32_t);
+                }
+            } else {
+                expectedBytes = 1;
+                bool expectedBufferSizeValid = computeExpectedBufferSizeReturnFalseIfOverflow<py::ssize_t>(shape, precision.size(), expectedBytes);
+                if (!expectedBufferSizeValid) {
+                    const std::string details = "Provided shape and datatype declare too large buffer.";
+                    SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details);
+                    return Status(StatusCode::INVALID_CONTENT_SIZE, details);
+                }
+                OVMS_RETURN_ON_FAIL(validateInputContent(*requestInputItr, expectedBytes, requestedName, request));
             }
-            OVMS_RETURN_ON_FAIL(validateInputContent(*requestInputItr, expectedBytes, requestedName, request));
             auto ok = pythonBackend->createEmptyOvmsPyTensor(
                 requestedName,
                 shape,
                 requestInputItr->datatype(),
                 expectedBytes,
                 outTensor);
+            if (!ok) {
+                SPDLOG_DEBUG("Error creating empty Python tensor");
+                return StatusCode::UNKNOWN_ERROR;
+            }
             void* data;
             if (!pythonBackend->getOvmsPyTensorData(outTensor, &data)) {
                 return Status(StatusCode::INTERNAL_ERROR);
@@ -711,6 +727,9 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             case ov::element::Type_t::f32: {
                 COPY_INPUT_VALUE_BY_VALUE(float, fp32);
             }
+            case ov::element::Type_t::f64: {
+                COPY_INPUT_VALUE_BY_VALUE(double, fp64);
+            }
             case ov::element::Type_t::i64: {
                 COPY_INPUT_VALUE_BY_VALUE(int64_t, int64);
             }
@@ -738,13 +757,23 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
             case ov::element::Type_t::boolean: {
                 COPY_INPUT_VALUE_BY_VALUE(bool, bool);
             }
+            case ov::element::Type_t::string: {
+                uint32_t offset = 0;
+                for (auto contents : request.inputs(inputIndex).contents().bytes_contents()) {
+                    uint32_t size = contents.size();
+                    std::memcpy(reinterpret_cast<char*>(data) + offset, &size, sizeof(uint32_t));
+                    offset += sizeof(uint32_t);
+                    std::memcpy(reinterpret_cast<char*>(data) + offset, contents.data(), size);
+                    offset += size;
+                }
+                return StatusCode::OK;
+            }
 
             // the rest not supported by KFS
             case ov::element::Type_t::u1:
             case ov::element::Type_t::u4:
             case ov::element::Type_t::i4:
             case ov::element::Type_t::f16:
-            case ov::element::Type_t::f64:
             case ov::element::Type_t::bf16:
             case ov::element::Type_t::undefined:
             case ov::element::Type_t::dynamic:

src/python/python_backend.cpp

@@ -81,6 +81,7 @@ bool PythonBackend::createEmptyOvmsPyTensor(const std::string& name, const std::
         outTensor = std::make_unique<PyObjectWrapper<py::object>>(ovmsPyTensor);
         return true;
     } catch (const pybind11::error_already_set& e) {
+        SPDLOG_ERROR("TENSOR {}", size);
         SPDLOG_DEBUG("PythonBackend::createEmptyOvmsPyTensor - Py Error: {}", e.what());
         return false;
     } catch (std::exception& e) {

src/rest_parser.cpp

@@ -730,6 +730,10 @@ Status KFSRestParser::parseInput(rapidjson::Value& node, bool onlyOneInput) {
         if (!(dataItr->value.IsArray())) {
             return StatusCode::REST_COULD_NOT_PARSE_INPUT;
         }
+        if (std::strcmp(datatypeItr->value.GetString(), "FP16") == 0 || std::strcmp(datatypeItr->value.GetString(), "BF16") == 0) {
+            SPDLOG_DEBUG("{} datatype is supported only when data is located in raw_input_contents", datatypeItr->value.GetString());
+            return StatusCode::REST_COULD_NOT_PARSE_INPUT;
+        }
         return parseData(dataItr->value, *input);
     } else {
         auto binary_data_size_parameter = input->parameters().find("binary_data_size");

src/rest_utils.cpp

@@ -387,6 +387,8 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett
             PARSE_OUTPUT_DATA(int64_contents, int64_t, Int64)
         } else if (tensor.datatype() == "UINT64") {
             PARSE_OUTPUT_DATA(uint64_contents, uint64_t, Uint64)
+        } else if (tensor.datatype() == "BOOL") {
+            PARSE_OUTPUT_DATA(bool_contents, bool, Bool)
         } else if (tensor.datatype() == "BYTES") {
             PARSE_OUTPUT_DATA_STRING(bytes_contents, String)
         } else {

src/test/kfs_rest_test.cpp

@@ -148,6 +148,10 @@ class HttpRestApiHandlerWithMediapipe : public ::testing::TestWithParam<std::str
         handler = std::make_unique<HttpRestApiHandler>(server, 5);
     }
 
+    void SetUp() {
+        SetUpServer("/ovms/src/test/mediapipe/config_python_summator.json");
+    }
+
     void TearDown() {
         handler.reset();
         server.setShutdownRequest(1);
@@ -156,10 +160,10 @@ class HttpRestApiHandlerWithMediapipe : public ::testing::TestWithParam<std::str
     }
 };
 
-class HttpRestApiHandlerWithMediapipeForkTest : public HttpRestApiHandlerWithMediapipe {
-public:
+class HttpRestApiHandlerWithMediapipePassthrough : public HttpRestApiHandlerWithMediapipe {
+protected:
     void SetUp() {
-        SetUpServer("/ovms/src/test/mediapipe/config_summator.json");
+        SetUpServer("/ovms/src/test/mediapipe/config_mp_pytensor_passthrough.json");
     }
 };
 
@@ -224,16 +228,14 @@ class HttpRestApiHandlerWithStringModelTest : public HttpRestApiHandlerTest {
 std::unique_ptr<MockedServer> HttpRestApiHandlerTest::server = nullptr;
 std::unique_ptr<std::thread> HttpRestApiHandlerTest::thread = nullptr;
 
-static void testInference(int headerLength, std::string& request_body, std::unique_ptr<HttpRestApiHandler>& handler) {
-    std::string request = "/v2/models/mediapipeAdd/versions/1/infer";
-
+static void testInference(int headerLength, std::string& request_body, std::unique_ptr<HttpRestApiHandler>& handler, const std::string endpoint = "/v2/models/mediapipeAdd/versions/1/infer") {
     std::vector<std::pair<std::string, std::string>> headers;
     std::pair<std::string, std::string> binaryInputsHeader{"Inference-Header-Content-Length", std::to_string(headerLength)};
     headers.emplace_back(binaryInputsHeader);
 
     ovms::HttpRequestComponents comp;
 
-    ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request, headers), ovms::StatusCode::OK);
+    ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK);
 
     std::string response;
     ovms::HttpResponseComponents responseComponents;
@@ -245,21 +247,96 @@ static void testInference(int headerLength, std::string& request_body, std::uniq
 
     auto output = doc["outputs"].GetArray()[0].GetObject()["data"].GetArray();
     ASSERT_EQ(output.Size(), 10);
-
+    auto datatype = doc["outputs"].GetArray()[0].GetObject()["datatype"].GetString();
     for (auto& data : output) {
-        ASSERT_EQ(data.GetFloat(), 2);
+        if (strcmp(datatype, "BOOL") == 0) {
+            ASSERT_EQ(data.GetBool(), true);
+        } else {
+            ASSERT_EQ(data.GetFloat(), 2);
+        }
     }
 }
 
+static void testInferenceNegative(int headerLength, std::string& request_body, std::unique_ptr<HttpRestApiHandler>& handler, ovms::Status processorStatus) {
+    std::string request = "/v2/models/mediapipeAdd/versions/1/infer";
+
+    std::vector<std::pair<std::string, std::string>> headers;
+    std::pair<std::string, std::string> binaryInputsHeader{"Inference-Header-Content-Length", std::to_string(headerLength)};
+    headers.emplace_back(binaryInputsHeader);
+
+    ovms::HttpRequestComponents comp;
+
+    ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request, headers), ovms::StatusCode::OK);
+
+    std::string response;
+    ovms::HttpResponseComponents responseComponents;
+    ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp, responseComponents), processorStatus);
+}
+
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wnarrowing"
 
-TEST_F(HttpRestApiHandlerWithMediapipeForkTest, inferRequestFP32) {
+TEST_P(HttpRestApiHandlerWithMediapipe, inferRequestWithSupportedPrecision) {
+    std::string datatype = GetParam();
+    std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"" + datatype + "\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+    std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"" + datatype + "\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+
+    std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
+    int headerLength = request_body.length();
+
+    testInference(headerLength, request_body, handler);
+}
+
+TEST_F(HttpRestApiHandlerWithMediapipe, inferRequestFP16) {
+    std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"FP16\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+    std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"FP16\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+
+    std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
+    int headerLength = request_body.length();
+    // Supported only when data is in binary extension
+    testInferenceNegative(headerLength, request_body, handler, ovms::StatusCode::REST_COULD_NOT_PARSE_INPUT);
+}
+
+TEST_F(HttpRestApiHandlerWithMediapipe, inferRequestBF16) {
+    std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"BF16\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+    std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"BF16\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+
+    std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
+    int headerLength = request_body.length();
+    // Supported only when data is in binary extension
+    testInferenceNegative(headerLength, request_body, handler, ovms::StatusCode::REST_COULD_NOT_PARSE_INPUT);
+}
+
+TEST_F(HttpRestApiHandlerWithMediapipe, inferRequestBOOL) {
+    std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"BOOL\", \"data\": [true,true,true,true,true,true,true,true,true,true]}";
+    std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"BOOL\", \"data\": [true,true,true,true,true,true,true,true,true,true]}";
+
+    std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
+    int headerLength = request_body.length();
+
+    testInference(headerLength, request_body, handler);
+}
+
+TEST_F(HttpRestApiHandlerWithMediapipe, inferRequestFP32DataInJsonAndBinaryExtension) {
     // 10 element array of floats: [1,1,1,1,1,1,1,1,1,1]
     std::string binaryData{0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F};
 
     std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"FP32\",\"parameters\":{\"binary_data_size\":40}}";
-    std::string param = ",\"parameters\":{\"binary_data_output\":true}";
+    std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"FP32\", \"data\": [1,1,1,1,1,1,1,1,1,1]}";
+
+    std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
+    int headerLength = request_body.length();
+
+    request_body += binaryData;
+    request_body += binaryData;
+
+    testInferenceNegative(headerLength, request_body, handler, ovms::StatusCode::INVALID_MESSAGE_STRUCTURE);
+}
+
+TEST_F(HttpRestApiHandlerWithMediapipe, inferRequestFP32BinaryExtension) {
+    std::string binaryData{0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x80, 0x3F};
+
+    std::string tensor1 = "{\"name\":\"in1\",\"shape\":[1,10],\"datatype\":\"FP32\",\"parameters\":{\"binary_data_size\":40}}";
     std::string tensor2 = "{\"name\":\"in2\",\"shape\":[1,10],\"datatype\":\"FP32\",\"parameters\":{\"binary_data_size\":40}}";
 
     std::string request_body = "{\"inputs\":[" + tensor1 + ", " + tensor2 + "]}";
@@ -271,6 +348,40 @@ TEST_F(HttpRestApiHandlerWithMediapipeForkTest, inferRequestFP32) {
     testInference(headerLength, request_body, handler);
 }
 
+std::vector<std::string> supportedDatatypes = {"FP32", "FP64", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64"};
+
+INSTANTIATE_TEST_SUITE_P(
+    TestDeserialize,
+    HttpRestApiHandlerWithMediapipe,
+    ::testing::ValuesIn(supportedDatatypes),
+    [](const ::testing::TestParamInfo<HttpRestApiHandlerWithMediapipe::ParamType>& info) {
+        return info.param;
+    });
+
+TEST_F(HttpRestApiHandlerWithMediapipePassthrough, inferRequestBYTES) {
+    std::string request = "/v2/models/mpPytensorPassthrough/versions/1/infer";
+    std::string request_body = "{\"inputs\":[{\"name\":\"in\",\"shape\":[3],\"datatype\":\"BYTES\", \"data\": [\"abc\", \"def\", \"ghi\"]}]}";
+    ovms::HttpRequestComponents comp;
+
+    ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request), ovms::StatusCode::OK);
+    std::string response;
+    ovms::HttpResponseComponents responseComponents;
+    ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp, responseComponents), ovms::StatusCode::OK);
+
+    rapidjson::Document doc;
+    doc.Parse(response.c_str());
+    ASSERT_FALSE(doc.HasParseError());
+    ASSERT_TRUE(doc["outputs"][0].GetObject().HasMember("data"));
+    ASSERT_TRUE(doc["outputs"][0].GetObject()["data"].IsArray());
+    auto output = doc["outputs"].GetArray()[0].GetObject()["data"].GetArray();
+    std::vector<std::string> expectedStrings{"abc", "def", "ghi"};
+    ASSERT_EQ(output.Size(), expectedStrings.size());
+    for (size_t i = 0; i < expectedStrings.size(); i++) {
+        ASSERT_TRUE(output[i].IsString());
+        ASSERT_EQ(output[i].GetString(), expectedStrings[i]);
+    }
+}
+
 #pragma GCC diagnostic pop
 
 TEST_F(HttpRestApiHandlerTest, MetricsParameters) {

src/test/mediapipe/config_mp_pytensor_passthrough.json

@@ -0,0 +1,9 @@
+{
+    "model_config_list": [],
+    "mediapipe_config_list": [
+    {
+        "name":"mpPytensorPassthrough",
+        "graph_path":"/ovms/src/test/mediapipe/graphpytensorpassthrough.pbtxt"
+    }
+    ]
+}

src/test/mediapipe/config_python_summator.json

@@ -0,0 +1,9 @@
+{
+    "model_config_list": [],
+    "mediapipe_config_list": [
+    {
+        "name":"mediapipeAdd",
+        "graph_path":"/ovms/src/test/mediapipe/graphpythonsummator.pbtxt"
+    }
+    ]
+}