From 21bb5cc95e5d18e97e850906159ff0364a12f2e3 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:15:07 +0200 Subject: [PATCH 01/35] cc: matrix_frame as input to graph - "matrix_frame" as name in order to avoid confusion with matrix.cc - matrix_frame is a 2D input data modality that gets converted to Eigen::MatrixXf internally - suited for non-image input to tflite models --- mediapipe_api/BUILD | 1 + mediapipe_api/framework/formats/BUILD | 16 +++++++ .../framework/formats/matrix_frame.cc | 46 +++++++++++++++++++ .../framework/formats/matrix_frame.h | 27 +++++++++++ 4 files changed, 90 insertions(+) create mode 100644 mediapipe_api/framework/formats/matrix_frame.cc create mode 100644 mediapipe_api/framework/formats/matrix_frame.h diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index cc4ba455f..1bffc909c 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -195,6 +195,7 @@ cc_library( "//mediapipe_api/framework/formats:detection", "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", + "//mediapipe_api/framework/formats:matrix_frame", "//mediapipe_api/framework/formats:rect", "//mediapipe_api/framework/port:logging", "//mediapipe_api/graphs/object_detection_3d/calculators:model_matrix", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index 6bd9f8966..20b97a860 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -64,6 +64,22 @@ cc_library( alwayslink = True, ) +cc_library( + name = "matrix_frame", + srcs = ["matrix_frame.cc"], + hdrs = ["matrix_frame.h"], + deps = [ + "//mediapipe_api:common", + "//mediapipe_api/external/absl:status", + "//mediapipe_api/external/absl:statusor", + "//mediapipe_api/framework:packet", + "@com_google_mediapipe//mediapipe/framework/formats:matrix", + "@com_google_mediapipe//mediapipe/framework/formats:matrix_data_cc_proto", + "@com_google_mediapipe//mediapipe/framework/port:file_helpers", + ], + alwayslink = True, +) + cc_library( name = "rect", srcs = ["rect.cc"], diff --git a/mediapipe_api/framework/formats/matrix_frame.cc b/mediapipe_api/framework/formats/matrix_frame.cc new file mode 100644 index 000000000..aa0718f1b --- /dev/null +++ b/mediapipe_api/framework/formats/matrix_frame.cc @@ -0,0 +1,46 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#include "mediapipe_api/framework/formats/matrix_frame.h" + +MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out) { + TRY + mediapipe::Matrix matrix; + + // convert matrix data from char into mediapipe::MatrixData + std::string content; + mediapipe::MatrixData matrix_data; + CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); + + // fill matrix with data from matrix_data_serialized + mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); + + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out) { + TRY + mediapipe::Matrix matrix; + + // convert matrix data from char into mediapipe::MatrixData + std::string content; + mediapipe::MatrixData matrix_data; + CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); + + // fill matrix with data from matrix_data_serialized + mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); + + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix).At(*timestamp)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + + + + diff --git a/mediapipe_api/framework/formats/matrix_frame.h b/mediapipe_api/framework/formats/matrix_frame.h new file mode 100644 index 000000000..d833a69a7 --- /dev/null +++ b/mediapipe_api/framework/formats/matrix_frame.h @@ -0,0 +1,27 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ + +#include "mediapipe_api/framework/formats/matrix_frame.h" +#include "mediapipe_api/common.h" +#include "mediapipe_api/external/protobuf.h" +#include "mediapipe_api/framework/packet.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/port/file_helpers.h" + +extern "C" { + +MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out); +// MP_CAPI(MpReturnCode) mp_Packet__GetMatrixFrame(mediapipe::Packet* packet, const mediapipe::Matrix** value_out); +// MP_CAPI(void) mp_MatrixFrame__delete(mediapipe::Matrix* matrix); + +} // extern "C" + +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ From 537b83f0acbaf6d4760291ab3d1106ba8ddcc745 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:19:05 +0200 Subject: [PATCH 02/35] cc: float_vector_frame as input to graph - "float_vector_frame" as name in order to avoid confusion with FloatArrayPacket - float_vector_frame is a 1D output data modality that accepts std::vector outputs from a mediapipe / tflite graph - suited for tflite classification results packaged as vector of floats --- mediapipe_api/BUILD | 1 + mediapipe_api/framework/formats/BUILD | 13 +++++++ .../framework/formats/float_vector_frame.cc | 35 +++++++++++++++++++ .../framework/formats/float_vector_frame.h | 26 ++++++++++++++ 4 files changed, 75 insertions(+) create mode 100644 mediapipe_api/framework/formats/float_vector_frame.cc create mode 100644 mediapipe_api/framework/formats/float_vector_frame.h diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index 1bffc909c..f98e9f70c 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -196,6 +196,7 @@ cc_library( "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", "//mediapipe_api/framework/formats:matrix_frame", + "//mediapipe_api/framework/formats:float_vector_frame", "//mediapipe_api/framework/formats:rect", "//mediapipe_api/framework/port:logging", "//mediapipe_api/graphs/object_detection_3d/calculators:model_matrix", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index 20b97a860..b3d7cca10 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -80,6 +80,19 @@ cc_library( alwayslink = True, ) +cc_library( + name = "float_vector_frame", + srcs = ["float_vector_frame.cc"], + hdrs = ["float_vector_frame.h"], + deps = [ + "//mediapipe_api:common", + "//mediapipe_api/external/absl:status", + "//mediapipe_api/external/absl:statusor", + "//mediapipe_api/framework:packet", + ], + alwayslink = True, +) + cc_library( name = "rect", srcs = ["rect.cc"], diff --git a/mediapipe_api/framework/formats/float_vector_frame.cc b/mediapipe_api/framework/formats/float_vector_frame.cc new file mode 100644 index 000000000..b748ab06c --- /dev/null +++ b/mediapipe_api/framework/formats/float_vector_frame.cc @@ -0,0 +1,35 @@ +#include "mediapipe_api/framework/formats/float_vector_frame.h" + +#include + +MpReturnCode mp__MakeFloatVectorFramePacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out) { + TRY + std::vector vector{}; + for (auto i = 0; i < size; ++i) { + vector.push_back(value[i]); + } + *packet_out = new mediapipe::Packet{mediapipe::MakePacket>(vector)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp__MakeFloatVectorFramePacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out) { + TRY + std::vector vector{}; + for (auto i = 0; i < size; ++i) { + vector.push_back(value[i]); + } + *packet_out = new mediapipe::Packet{mediapipe::MakePacket>(vector).At(*timestamp)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp_Packet__GetFloatVectorFrame(mediapipe::Packet* packet, const float** value_out) { + TRY_ALL + *value_out = packet->Get>().data(); + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +void mp_FloatVectorFrame__delete(float* vector_data) { delete[] vector_data; } \ No newline at end of file diff --git a/mediapipe_api/framework/formats/float_vector_frame.h b/mediapipe_api/framework/formats/float_vector_frame.h new file mode 100644 index 000000000..be66e92d4 --- /dev/null +++ b/mediapipe_api/framework/formats/float_vector_frame.h @@ -0,0 +1,26 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ + +#include "mediapipe_api/framework/formats/float_vector_frame.h" +#include "mediapipe_api/common.h" +#include "mediapipe_api/external/protobuf.h" +#include "mediapipe_api/framework/packet.h" + + +extern "C" { + +MP_CAPI(MpReturnCode) mp__MakeFloatVectorFramePacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeFloatVectorFramePacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp_Packet__GetFloatVectorFrame(mediapipe::Packet* packet, const float** value_out); +MP_CAPI(void) mp_FloatVectorFrame__delete(float* vector_data); + +} // extern "C" + +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ From 9e1334b60a25f9805444108ebd1a3db5cb867f0f Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:25:19 +0200 Subject: [PATCH 03/35] MatrixFramePacket - c# helper functions - send MatrixData to C++ as byte array --- .../Framework/Packet/MatrixFramePacket.cs | 67 +++++++++++++++++++ .../Packet/MatrixFramePacket.cs.meta | 11 +++ .../Framework/Format/MatrixFrame_Unsafe.cs | 20 ++++++ .../Format/MatrixFrame_Unsafe.cs.meta | 11 +++ 4 files changed, 109 insertions(+) create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs new file mode 100644 index 000000000..9df27dcb5 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs @@ -0,0 +1,67 @@ + + +using System; + +namespace Mediapipe +{ + public class MatrixFramePacket : Packet + { + private int _length = -1; + + public int length + { + get => _length; + set + { + if (_length >= 0) + { + throw new InvalidOperationException("Length is already set and cannot be changed"); + } + + _length = value; + } + } + + /// + /// Creates an empty instance. + /// + public MatrixFramePacket() : base(true) { } + + [UnityEngine.Scripting.Preserve] + public MatrixFramePacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } + + public MatrixFramePacket(byte[] value) : base() + { + UnsafeNativeMethods.mp__MakeMatrixFramePacket__PKc_i(value, value.Length, out var ptr).Assert(); + this.ptr = ptr; + length = value.Length; + } + + public MatrixFramePacket(byte[] value, Timestamp timestamp) : base() + { + UnsafeNativeMethods.mp__MakeMatrixFramePacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); + GC.KeepAlive(timestamp); + this.ptr = ptr; + length = value.Length; + } + + public MatrixFramePacket At(Timestamp timestamp) + { + var packet = At(timestamp); + packet.length = length; + return packet; + } + + public override byte[] Get() + { + throw new NotImplementedException(); + } + + public override StatusOr Consume() + { + throw new NotImplementedException(); + } + + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta new file mode 100644 index 000000000..2e04b5783 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ef898503888a0ee42af28aef1fbbe794 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs new file mode 100644 index 000000000..28a58b8ce --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs @@ -0,0 +1,20 @@ + + +using System; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + internal static partial class UnsafeNativeMethods + { + + #region Packet + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeMatrixFramePacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); + + #endregion + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta new file mode 100644 index 000000000..67688a598 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: fc5345171757d2049a546634b03b8c5e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 69f705f687f23ee1f12a759e5d046181d865cc3e Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:28:23 +0200 Subject: [PATCH 04/35] FloatVectorFramePacket - c# helper functions - return std::vector from C++ to Unity as List --- .../Packet/FloatVectorFramePacket.cs | 76 +++++++++++++++++++ .../Packet/FloatVectorFramePacket.cs.meta | 11 +++ .../Format/FloatVectorFrame_Unsafe.cs | 22 ++++++ .../Format/FloatVectorFrame_Unsafe.cs.meta | 11 +++ 4 files changed, 120 insertions(+) create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs new file mode 100644 index 000000000..444d301d4 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Mediapipe +{ + public class FloatVectorFramePacket : Packet> + { + /// + /// Creates an empty instance. + /// + /// + + private int _vectorLength; + + + public FloatVectorFramePacket() : base(true) { } + + [UnityEngine.Scripting.Preserve] + public FloatVectorFramePacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } + + public FloatVectorFramePacket(float[] value) : base() + { + UnsafeNativeMethods.mp__MakeFloatVectorFramePacket__PA_i(value, value.Length, out var ptr).Assert(); + this.ptr = ptr; + _vectorLength = value.Length; + } + + public FloatVectorFramePacket(float[] value, Timestamp timestamp) : base() + { + UnsafeNativeMethods.mp__MakeFloatVectorFramePacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); + GC.KeepAlive(timestamp); + this.ptr = ptr; + } + + public FloatVectorFramePacket At(Timestamp timestamp) + { + return At(timestamp); + } + + public override List Get() + { + if (_vectorLength < 0) + { + throw new InvalidOperationException("The array's length is unknown, set Length first"); + } + + var result = new float[_vectorLength]; + + unsafe + { + var src = (float*)GetArrayPtr(); + + for (var i = 0; i < result.Length; i++) + { + result[i] = *src++; + } + } + + return result.ToList(); + } + + public IntPtr GetArrayPtr() + { + UnsafeNativeMethods.mp_Packet__GetFloatVectorFrame(mpPtr, out var floatFrameVector).Assert(); + GC.KeepAlive(this); + + return floatFrameVector; + } + + public override StatusOr> Consume() + { + throw new NotSupportedException(); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta new file mode 100644 index 000000000..23702bbfb --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8a9b77e746fd38d46a09d847f7370446 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs new file mode 100644 index 000000000..17740f5ed --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + internal static partial class UnsafeNativeMethods + { + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorFramePacket__PA_i(float[] value, int size, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorFramePacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetFloatVectorFrame(IntPtr packet, out IntPtr value); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_FloatVectorFrame__delete(IntPtr floatVectorFrame); + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta new file mode 100644 index 000000000..fe31878a4 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 43e481a62e09d8541aff1c80f92c735a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 4d434895eddfe2cbdb58c9814cb68c35ac61fd2a Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:47:49 +0200 Subject: [PATCH 05/35] Unity: Matrix Classification - Example scene --- .../Samples/Scenes/Matrix Classification.meta | 8 + .../Matrix Classification.unity | 347 ++++++++++++++++++ .../Matrix Classification.unity.meta | 7 + 3 files changed, 362 insertions(+) create mode 100644 Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta create mode 100644 Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity create mode 100644 Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta new file mode 100644 index 000000000..771ea769e --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 1d0aace15955cf94b8513eb95bd7a753 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity new file mode 100644 index 000000000..a5c93c129 --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity @@ -0,0 +1,347 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!29 &1 +OcclusionCullingSettings: + m_ObjectHideFlags: 0 + serializedVersion: 2 + m_OcclusionBakeSettings: + smallestOccluder: 5 + smallestHole: 0.25 + backfaceThreshold: 100 + m_SceneGUID: 00000000000000000000000000000000 + m_OcclusionCullingData: {fileID: 0} +--- !u!104 &2 +RenderSettings: + m_ObjectHideFlags: 0 + serializedVersion: 9 + m_Fog: 0 + m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1} + m_FogMode: 3 + m_FogDensity: 0.01 + m_LinearFogStart: 0 + m_LinearFogEnd: 300 + m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1} + m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1} + m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1} + m_AmbientIntensity: 1 + m_AmbientMode: 0 + m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1} + m_SkyboxMaterial: {fileID: 10304, guid: 0000000000000000f000000000000000, type: 0} + m_HaloStrength: 0.5 + m_FlareStrength: 1 + m_FlareFadeSpeed: 3 + m_HaloTexture: {fileID: 0} + m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0} + m_DefaultReflectionMode: 0 + m_DefaultReflectionResolution: 128 + m_ReflectionBounces: 1 + m_ReflectionIntensity: 1 + m_CustomReflection: {fileID: 0} + m_Sun: {fileID: 0} + m_IndirectSpecularColor: {r: 0.44657898, g: 0.4964133, b: 0.5748178, a: 1} + m_UseRadianceAmbientProbe: 0 +--- !u!157 &3 +LightmapSettings: + m_ObjectHideFlags: 0 + serializedVersion: 12 + m_GIWorkflowMode: 1 + m_GISettings: + serializedVersion: 2 + m_BounceScale: 1 + m_IndirectOutputScale: 1 + m_AlbedoBoost: 1 + m_EnvironmentLightingMode: 0 + m_EnableBakedLightmaps: 1 + m_EnableRealtimeLightmaps: 0 + m_LightmapEditorSettings: + serializedVersion: 12 + m_Resolution: 2 + m_BakeResolution: 40 + m_AtlasSize: 1024 + m_AO: 0 + m_AOMaxDistance: 1 + m_CompAOExponent: 1 + m_CompAOExponentDirect: 0 + m_ExtractAmbientOcclusion: 0 + m_Padding: 2 + m_LightmapParameters: {fileID: 0} + m_LightmapsBakeMode: 1 + m_TextureCompression: 1 + m_FinalGather: 0 + m_FinalGatherFiltering: 1 + m_FinalGatherRayCount: 256 + m_ReflectionCompression: 2 + m_MixedBakeMode: 2 + m_BakeBackend: 1 + m_PVRSampling: 1 + m_PVRDirectSampleCount: 32 + m_PVRSampleCount: 512 + m_PVRBounces: 2 + m_PVREnvironmentSampleCount: 256 + m_PVREnvironmentReferencePointCount: 2048 + m_PVRFilteringMode: 1 + m_PVRDenoiserTypeDirect: 1 + m_PVRDenoiserTypeIndirect: 1 + m_PVRDenoiserTypeAO: 1 + m_PVRFilterTypeDirect: 0 + m_PVRFilterTypeIndirect: 0 + m_PVRFilterTypeAO: 0 + m_PVREnvironmentMIS: 1 + m_PVRCulling: 1 + m_PVRFilteringGaussRadiusDirect: 1 + m_PVRFilteringGaussRadiusIndirect: 5 + m_PVRFilteringGaussRadiusAO: 2 + m_PVRFilteringAtrousPositionSigmaDirect: 0.5 + m_PVRFilteringAtrousPositionSigmaIndirect: 2 + m_PVRFilteringAtrousPositionSigmaAO: 1 + m_ExportTrainingData: 0 + m_TrainingDataDestination: TrainingData + m_LightProbeSampleCountMultiplier: 4 + m_LightingDataAsset: {fileID: 0} + m_LightingSettings: {fileID: 0} +--- !u!196 &4 +NavMeshSettings: + serializedVersion: 2 + m_ObjectHideFlags: 0 + m_BuildSettings: + serializedVersion: 2 + agentTypeID: 0 + agentRadius: 0.5 + agentHeight: 2 + agentSlope: 45 + agentClimb: 0.4 + ledgeDropHeight: 0 + maxJumpAcrossDistance: 0 + minRegionArea: 2 + manualCellSize: 0 + cellSize: 0.16666667 + manualTileSize: 0 + tileSize: 256 + accuratePlacement: 0 + maxJobWorkers: 0 + preserveTilesOutsideBounds: 0 + debug: + m_Flags: 0 + m_NavMeshData: {fileID: 0} +--- !u!1 &938836659 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 938836661} + - component: {fileID: 938836662} + m_Layer: 0 + m_Name: Matrix Classification + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &938836661 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 938836659} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &938836662 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 938836659} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 87ae56d766402f849ba1ad498f5f2f67, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!1 &984312839 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 984312842} + - component: {fileID: 984312841} + - component: {fileID: 984312840} + m_Layer: 0 + m_Name: Main Camera + m_TagString: MainCamera + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!81 &984312840 +AudioListener: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 984312839} + m_Enabled: 1 +--- !u!20 &984312841 +Camera: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 984312839} + m_Enabled: 1 + serializedVersion: 2 + m_ClearFlags: 1 + m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0} + m_projectionMatrixMode: 1 + m_GateFitMode: 2 + m_FOVAxisMode: 0 + m_SensorSize: {x: 36, y: 24} + m_LensShift: {x: 0, y: 0} + m_FocalLength: 50 + m_NormalizedViewPortRect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1 + height: 1 + near clip plane: 0.3 + far clip plane: 1000 + field of view: 60 + orthographic: 0 + orthographic size: 5 + m_Depth: -1 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingPath: -1 + m_TargetTexture: {fileID: 0} + m_TargetDisplay: 0 + m_TargetEye: 3 + m_HDR: 1 + m_AllowMSAA: 1 + m_AllowDynamicResolution: 0 + m_ForceIntoRT: 0 + m_OcclusionCulling: 1 + m_StereoConvergence: 10 + m_StereoSeparation: 0.022 +--- !u!4 &984312842 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 984312839} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 1, z: -10} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &1144986087 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1144986089} + - component: {fileID: 1144986088} + m_Layer: 0 + m_Name: Directional Light + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!108 &1144986088 +Light: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1144986087} + m_Enabled: 1 + serializedVersion: 10 + m_Type: 1 + m_Shape: 0 + m_Color: {r: 1, g: 0.95686275, b: 0.8392157, a: 1} + m_Intensity: 1 + m_Range: 10 + m_SpotAngle: 30 + m_InnerSpotAngle: 21.80208 + m_CookieSize: 10 + m_Shadows: + m_Type: 2 + m_Resolution: -1 + m_CustomResolution: -1 + m_Strength: 1 + m_Bias: 0.05 + m_NormalBias: 0.4 + m_NearPlane: 0.2 + m_CullingMatrixOverride: + e00: 1 + e01: 0 + e02: 0 + e03: 0 + e10: 0 + e11: 1 + e12: 0 + e13: 0 + e20: 0 + e21: 0 + e22: 1 + e23: 0 + e30: 0 + e31: 0 + e32: 0 + e33: 1 + m_UseCullingMatrixOverride: 0 + m_Cookie: {fileID: 0} + m_DrawHalo: 0 + m_Flare: {fileID: 0} + m_RenderMode: 0 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingLayerMask: 1 + m_Lightmapping: 4 + m_LightShadowCasterMode: 0 + m_AreaSize: {x: 1, y: 1} + m_BounceIntensity: 1 + m_ColorTemperature: 6570 + m_UseColorTemperature: 0 + m_BoundingSphereOverride: {x: 0, y: 0, z: 0, w: 0} + m_UseBoundingSphereOverride: 0 + m_UseViewFrustumForShadowCasterCull: 1 + m_ShadowRadius: 0 + m_ShadowAngle: 0 +--- !u!4 &1144986089 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1144986087} + m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261} + m_LocalPosition: {x: 0, y: 3, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0} diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta new file mode 100644 index 000000000..afa52ba3e --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 2f27ce48d99be3841b0d4e4f6c4fb0c9 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: From 8a14d3be3cfe48b374bee15e74dcdc4f7b26fa46 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 15 Jul 2022 13:48:43 +0200 Subject: [PATCH 06/35] Matrix Classification.cs - driver code for the newly added MatrixFramePacket and FloatVectorFramePacket - feeds an example matrix of size [ 2 x 3 ] into a mediapipe graph - the graph runs a simple tflite model (adds +1 to every input) - then the graph returns the result back to Unity as List - only tested on Unity-Editor-Mode on Windows 10 Pro --- .../MatrixClassification.cs | 200 ++++++++++++++++++ .../MatrixClassification.cs.meta | 11 + 2 files changed, 211 insertions(+) create mode 100644 Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs create mode 100644 Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs new file mode 100644 index 000000000..4ac649cd6 --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs @@ -0,0 +1,200 @@ +// TODO: +// THIS class needs the following binary to work: "adder_model_single_input_2x3.bytes" +// You can download it via this Google Drive link: +// https://drive.google.com/file/d/1vVDn80eZClBDJ8dJUk3szb9rLaE9p60q/view?usp=sharing +// You need to copy the file to +// Packages\com.github.homuler.mediapipe\Runtime\Resources + +// Issues: +// - Make sure TfliteConverterCalculator is compiled along with the plugin + + +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using Google.Protobuf; + +namespace Mediapipe.Unity +{ + /// + /// Toy example on how to use tflite model for matrix classification. + /// In contrast to other mediapipe graphs, this graph does not expect an image as input data, + /// but deals with matrix input data. Internally the input (MatrixData) will be converted + /// into an Eigen::MatrixXf which is then fed to the first calculator node of the graph + /// (TfLiteConverterCalculator). From there the input is converted to a tflite tensor, forwarded + /// throught the neural network (tflite model). The output of the neural network is converted into + /// a std::vector and and finally passed back into Unity as a List + /// + /// Example: This code could be used for a pose based action classifier. + /// Input: Matrix containing values of an input pose + /// Output: Vector of action probabilities + /// + /// Since matrix can only store 2D input data, but tflite models generally expect 4D input (NHWC), + /// the trick is to collapse the exceeding dimensions such that tflite can parse the 4D data from a + /// 2D matrix. Since the input matrix (Eigen::MatrixXf) is stored in column major order, the input 2D + /// matrix has to be transposed befored feeding it into the graph. + /// + /// Example: matrix of dimension [num_frames = 10, num_landmarks = 33, num_coordinates = 3] + /// - collapse into matrix of shape [10 x 99] + /// - now transpose to make "coordinate" dimension "fastest" -> output shape is now [99x10] + /// - feed the data into the graph + /// + /// TODO: + /// In theory mediapipe can also handle matrices stored in row-major order + /// -> write code for row-major matrix input to mediapipe graph + /// + public class MatrixClassification : MonoBehaviour + { + + private readonly string TAG = "MatrixClassificationToyExample"; + + private void OnEnable() + { + var _ = StartCoroutine(Init()); + } + + private void Start() + { + Debug.Log("Setup Protobuf Logging"); + Protobuf.SetLogHandler(Protobuf.DefaultLogHandler); + + Debug.Log("Start"); + var configText = @" + input_stream: ""MATRIX:in"" + output_stream: ""FLOATS:out"" + + node + { + calculator: ""TfLiteConverterCalculator"" + input_stream: ""MATRIX:in"" + output_stream: ""TENSORS:image_tensor"" + options: + { + [mediapipe.TfLiteConverterCalculatorOptions.ext] + { + zero_center: false + } + } + } + + node + { + calculator: ""TfLiteInferenceCalculator"" + input_stream: ""TENSORS:image_tensor"" + output_stream: ""TENSORS:tensor_features"" + options: + { + [mediapipe.TfLiteInferenceCalculatorOptions.ext] + { + model_path: ""mediapipe/models/adder_model_single_input_2x3.tflite"" + } + } + } + + node + { + calculator: ""TfLiteTensorsToFloatsCalculator"" + input_stream: ""TENSORS:tensor_features"" + output_stream: ""FLOATS:out"" + } + "; + var graph = new CalculatorGraph(configText); + + // Specify expected output of tflite model + var poller = graph.AddOutputStreamPoller>("out").Value(); + + Debug.Log("StartRun"); + graph.StartRun().AssertOk(); + for (var i = 0; i < 10; i++) + { + var matrix = CreateInputData(); + + // feed data into graph + var input = new MatrixFramePacket(matrix.ToByteArray(), new Timestamp(i)); + graph.AddPacketToInputStream("in", input).AssertOk(); + } + graph.CloseInputStream("in").AssertOk(); + + Debug.Log("Poll output"); + // Create output container with suitable size + // -> size should correspond to tflite model output size + var outputFloatArray = new float[6] { 10, 11, 12, 13, 14, 15 }; + var output = new FloatVectorFramePacket(outputFloatArray); + + while (poller.Next(output)) + { + var result = output.Get(); + foreach (var item in result) + { + Debug.Log("result array: " + item); + } + } + + + graph.WaitUntilDone().AssertOk(); + graph.Dispose(); + + Debug.Log("Done"); + } + + private static MatrixData CreateInputData() + { + var matrix = new MatrixData(); + matrix.PackedData.Add(0); + matrix.PackedData.Add(1); + matrix.PackedData.Add(2); + matrix.PackedData.Add(3); + matrix.PackedData.Add(4); + matrix.PackedData.Add(5); + + matrix.Rows = 2; + matrix.Cols = 3; + return matrix; + } + + private void OnApplicationQuit() + { + Protobuf.ResetLogHandler(); + } + + // Load tflite model from assets + private IList RequestDependentAssets() + { + return new List { + WaitForAsset("adder_model_single_input_2x3.bytes"), + }; + } + + private WaitForResult WaitForAsset(string assetName, bool overwrite = false) + { + return WaitForAsset(assetName, assetName, overwrite); + } + + private WaitForResult WaitForAsset(string assetName, string uniqueKey, bool overwrite = false) + { + return new WaitForResult(this, AssetLoader.PrepareAssetAsync(assetName, uniqueKey, overwrite)); + } + + private IEnumerator Init() + { + Logger.LogInfo(TAG, "Loading dependent assets..."); + var assetRequests = RequestDependentAssets(); + yield return new WaitWhile(() => assetRequests.Any((request) => request.keepWaiting)); + + var errors = assetRequests.Where((request) => request.isError).Select((request) => request.error).ToList(); + if (errors.Count > 0) + { + foreach (var error in errors) + { + Logger.LogError(TAG, error); + } + throw new InternalException("Failed to prepare dependent assets"); + } + } + + } +} + + + diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta new file mode 100644 index 000000000..eb475807b --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 87ae56d766402f849ba1ad498f5f2f67 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 0bf959dacc88f0fa4c3fff0686c362b57a193bde Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 16:27:15 +0200 Subject: [PATCH 07/35] refactor: rename FloatVectorFrame -> FloatVector - rename variables - rename cs files - rename cc files --- .../MatrixClassification.cs | 2 +- ...torFramePacket.cs => FloatVectorPacket.cs} | 22 ++++++++-------- ...cket.cs.meta => FloatVectorPacket.cs.meta} | 0 .../Format/FloatVectorFrame_Unsafe.cs | 22 ---------------- .../Framework/Format/FloatVector_Unsafe.cs | 22 ++++++++++++++++ ...afe.cs.meta => FloatVector_Unsafe.cs.meta} | 0 mediapipe_api/BUILD | 2 +- mediapipe_api/framework/formats/BUILD | 6 ++--- ...{float_vector_frame.cc => float_vector.cc} | 10 +++---- .../framework/formats/float_vector.h | 26 +++++++++++++++++++ .../framework/formats/float_vector_frame.h | 26 ------------------- 11 files changed, 69 insertions(+), 69 deletions(-) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/{FloatVectorFramePacket.cs => FloatVectorPacket.cs} (54%) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/{FloatVectorFramePacket.cs.meta => FloatVectorPacket.cs.meta} (100%) delete mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/{FloatVectorFrame_Unsafe.cs.meta => FloatVector_Unsafe.cs.meta} (100%) rename mediapipe_api/framework/formats/{float_vector_frame.cc => float_vector.cc} (62%) create mode 100644 mediapipe_api/framework/formats/float_vector.h delete mode 100644 mediapipe_api/framework/formats/float_vector_frame.h diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs index 4ac649cd6..3af57270c 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs @@ -120,7 +120,7 @@ private void Start() // Create output container with suitable size // -> size should correspond to tflite model output size var outputFloatArray = new float[6] { 10, 11, 12, 13, 14, 15 }; - var output = new FloatVectorFramePacket(outputFloatArray); + var output = new FloatVectorPacket(outputFloatArray); while (poller.Next(output)) { diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs similarity index 54% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index 444d301d4..aa74105ad 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -4,38 +4,38 @@ namespace Mediapipe { - public class FloatVectorFramePacket : Packet> + public class FloatVectorPacket : Packet> { /// - /// Creates an empty instance. + /// Creates an empty instance. /// /// private int _vectorLength; - public FloatVectorFramePacket() : base(true) { } + public FloatVectorPacket() : base(true) { } [UnityEngine.Scripting.Preserve] - public FloatVectorFramePacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } + public FloatVectorPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } - public FloatVectorFramePacket(float[] value) : base() + public FloatVectorPacket(float[] value) : base() { - UnsafeNativeMethods.mp__MakeFloatVectorFramePacket__PA_i(value, value.Length, out var ptr).Assert(); + UnsafeNativeMethods.mp__MakeFloatVectorPacket__PA_i(value, value.Length, out var ptr).Assert(); this.ptr = ptr; _vectorLength = value.Length; } - public FloatVectorFramePacket(float[] value, Timestamp timestamp) : base() + public FloatVectorPacket(float[] value, Timestamp timestamp) : base() { - UnsafeNativeMethods.mp__MakeFloatVectorFramePacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); + UnsafeNativeMethods.mp__MakeFloatVectorPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; } - public FloatVectorFramePacket At(Timestamp timestamp) + public FloatVectorPacket At(Timestamp timestamp) { - return At(timestamp); + return At(timestamp); } public override List Get() @@ -62,7 +62,7 @@ public override List Get() public IntPtr GetArrayPtr() { - UnsafeNativeMethods.mp_Packet__GetFloatVectorFrame(mpPtr, out var floatFrameVector).Assert(); + UnsafeNativeMethods.mp_Packet__GetFloatVector(mpPtr, out var floatFrameVector).Assert(); GC.KeepAlive(this); return floatFrameVector; diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs.meta similarity index 100% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorFramePacket.cs.meta rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs deleted file mode 100644 index 17740f5ed..000000000 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace Mediapipe -{ - internal static partial class UnsafeNativeMethods - { - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeFloatVectorFramePacket__PA_i(float[] value, int size, out IntPtr packet); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeFloatVectorFramePacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_Packet__GetFloatVectorFrame(IntPtr packet, out IntPtr value); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern void mp_FloatVectorFrame__delete(IntPtr floatVectorFrame); - } -} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs new file mode 100644 index 000000000..5b2e33901 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + internal static partial class UnsafeNativeMethods + { + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorPacket__PA_i(float[] value, int size, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_FloatVector__delete(IntPtr floatVector); + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta similarity index 100% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVectorFrame_Unsafe.cs.meta rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index f98e9f70c..26be4b36a 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -196,7 +196,7 @@ cc_library( "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", "//mediapipe_api/framework/formats:matrix_frame", - "//mediapipe_api/framework/formats:float_vector_frame", + "//mediapipe_api/framework/formats:float_vector", "//mediapipe_api/framework/formats:rect", "//mediapipe_api/framework/port:logging", "//mediapipe_api/graphs/object_detection_3d/calculators:model_matrix", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index b3d7cca10..4bfab3a34 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -81,9 +81,9 @@ cc_library( ) cc_library( - name = "float_vector_frame", - srcs = ["float_vector_frame.cc"], - hdrs = ["float_vector_frame.h"], + name = "float_vector", + srcs = ["float_vector.cc"], + hdrs = ["float_vector.h"], deps = [ "//mediapipe_api:common", "//mediapipe_api/external/absl:status", diff --git a/mediapipe_api/framework/formats/float_vector_frame.cc b/mediapipe_api/framework/formats/float_vector.cc similarity index 62% rename from mediapipe_api/framework/formats/float_vector_frame.cc rename to mediapipe_api/framework/formats/float_vector.cc index b748ab06c..54aa703e3 100644 --- a/mediapipe_api/framework/formats/float_vector_frame.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -1,8 +1,8 @@ -#include "mediapipe_api/framework/formats/float_vector_frame.h" +#include "mediapipe_api/framework/formats/float_vector.h" #include -MpReturnCode mp__MakeFloatVectorFramePacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out) { TRY std::vector vector{}; for (auto i = 0; i < size; ++i) { @@ -13,7 +13,7 @@ MpReturnCode mp__MakeFloatVectorFramePacket__PA_i(const float* value, int size, CATCH_EXCEPTION } -MpReturnCode mp__MakeFloatVectorFramePacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, +MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY std::vector vector{}; @@ -25,11 +25,11 @@ MpReturnCode mp__MakeFloatVectorFramePacket_At__PA_i_Rt(const float* value, int CATCH_EXCEPTION } -MpReturnCode mp_Packet__GetFloatVectorFrame(mediapipe::Packet* packet, const float** value_out) { +MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out) { TRY_ALL *value_out = packet->Get>().data(); RETURN_CODE(MpReturnCode::Success); CATCH_ALL } -void mp_FloatVectorFrame__delete(float* vector_data) { delete[] vector_data; } \ No newline at end of file +void mp_FloatVector__delete(float* vector_data) { delete[] vector_data; } \ No newline at end of file diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h new file mode 100644 index 000000000..45688c976 --- /dev/null +++ b/mediapipe_api/framework/formats/float_vector.h @@ -0,0 +1,26 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ + +#include "mediapipe_api/framework/formats/float_vector.h" +#include "mediapipe_api/common.h" +#include "mediapipe_api/external/protobuf.h" +#include "mediapipe_api/framework/packet.h" + + +extern "C" { + +MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); +MP_CAPI(void) mp_FloatVector__delete(float* vector_data); + +} // extern "C" + +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ diff --git a/mediapipe_api/framework/formats/float_vector_frame.h b/mediapipe_api/framework/formats/float_vector_frame.h deleted file mode 100644 index be66e92d4..000000000 --- a/mediapipe_api/framework/formats/float_vector_frame.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ -#define MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ - -#include "mediapipe_api/framework/formats/float_vector_frame.h" -#include "mediapipe_api/common.h" -#include "mediapipe_api/external/protobuf.h" -#include "mediapipe_api/framework/packet.h" - - -extern "C" { - -MP_CAPI(MpReturnCode) mp__MakeFloatVectorFramePacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp__MakeFloatVectorFramePacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp_Packet__GetFloatVectorFrame(mediapipe::Packet* packet, const float** value_out); -MP_CAPI(void) mp_FloatVectorFrame__delete(float* vector_data); - -} // extern "C" - -#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_FRAME_H_ From 3a444c924ec2bd72a111ffd051648e4971ceb2c8 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 17:09:33 +0200 Subject: [PATCH 08/35] refactor: rename MatrixFrame -> Matrix - rename variables - rename cs files - rename cc files matrix_frame -> matrix_data -> avoid matrix.cc as it is already used as a name in mediapipe --- .../MatrixClassification.cs | 2 +- .../{MatrixFramePacket.cs => MatrixPacket.cs} | 20 +++++++++---------- ...amePacket.cs.meta => MatrixPacket.cs.meta} | 0 ...MatrixFrame_Unsafe.cs => Matrix_Unsafe.cs} | 4 ++-- ...e_Unsafe.cs.meta => Matrix_Unsafe.cs.meta} | 0 mediapipe_api/BUILD | 2 +- mediapipe_api/framework/formats/BUILD | 7 +++---- .../{matrix_frame.cc => matrix_data.cc} | 2 +- .../formats/{matrix_frame.h => matrix_data.h} | 8 ++++---- 9 files changed, 22 insertions(+), 23 deletions(-) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/{MatrixFramePacket.cs => MatrixPacket.cs} (54%) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/{MatrixFramePacket.cs.meta => MatrixPacket.cs.meta} (100%) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/{MatrixFrame_Unsafe.cs => Matrix_Unsafe.cs} (50%) rename Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/{MatrixFrame_Unsafe.cs.meta => Matrix_Unsafe.cs.meta} (100%) rename mediapipe_api/framework/formats/{matrix_frame.cc => matrix_data.cc} (96%) rename mediapipe_api/framework/formats/{matrix_frame.h => matrix_data.h} (81%) diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs index 3af57270c..d0aa629e3 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs @@ -111,7 +111,7 @@ private void Start() var matrix = CreateInputData(); // feed data into graph - var input = new MatrixFramePacket(matrix.ToByteArray(), new Timestamp(i)); + var input = new MatrixPacket(matrix.ToByteArray(), new Timestamp(i)); graph.AddPacketToInputStream("in", input).AssertOk(); } graph.CloseInputStream("in").AssertOk(); diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs similarity index 54% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs index 9df27dcb5..81cc07979 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs @@ -4,7 +4,7 @@ namespace Mediapipe { - public class MatrixFramePacket : Packet + public class MatrixPacket : Packet { private int _length = -1; @@ -23,32 +23,32 @@ public int length } /// - /// Creates an empty instance. /// - public MatrixFramePacket() : base(true) { } + public MatrixPacket() : base(true) { } [UnityEngine.Scripting.Preserve] - public MatrixFramePacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } + public MatrixPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } - public MatrixFramePacket(byte[] value) : base() + public MatrixPacket(byte[] value) : base() { - UnsafeNativeMethods.mp__MakeMatrixFramePacket__PKc_i(value, value.Length, out var ptr).Assert(); + UnsafeNativeMethods.mp__MakeMatrixPacket__PKc_i(value, value.Length, out var ptr).Assert(); this.ptr = ptr; length = value.Length; } - public MatrixFramePacket(byte[] value, Timestamp timestamp) : base() + public MatrixPacket(byte[] value, Timestamp timestamp) : base() { - UnsafeNativeMethods.mp__MakeMatrixFramePacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); + UnsafeNativeMethods.mp__MakeMatrixPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; length = value.Length; } - public MatrixFramePacket At(Timestamp timestamp) + public MatrixPacket At(Timestamp timestamp) { - var packet = At(timestamp); + var packet = At(timestamp); packet.length = length; return packet; } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs.meta similarity index 100% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixFramePacket.cs.meta rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs similarity index 50% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index 28a58b8ce..f3ded97a3 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -10,10 +10,10 @@ internal static partial class UnsafeNativeMethods #region Packet [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeMatrixFramePacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); + public static extern MpReturnCode mp__MakeMatrixPacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); + public static extern MpReturnCode mp__MakeMatrixPacket_At__PA_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); #endregion } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs.meta similarity index 100% rename from Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/MatrixFrame_Unsafe.cs.meta rename to Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs.meta diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index 26be4b36a..5e313382f 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -195,7 +195,7 @@ cc_library( "//mediapipe_api/framework/formats:detection", "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", - "//mediapipe_api/framework/formats:matrix_frame", + "//mediapipe_api/framework/formats:matrix_data", "//mediapipe_api/framework/formats:float_vector", "//mediapipe_api/framework/formats:rect", "//mediapipe_api/framework/port:logging", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index 4bfab3a34..bc98f3f85 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -65,9 +65,9 @@ cc_library( ) cc_library( - name = "matrix_frame", - srcs = ["matrix_frame.cc"], - hdrs = ["matrix_frame.h"], + name = "matrix_data", + srcs = ["matrix_data.cc"], + hdrs = ["matrix_data.h"], deps = [ "//mediapipe_api:common", "//mediapipe_api/external/absl:status", @@ -75,7 +75,6 @@ cc_library( "//mediapipe_api/framework:packet", "@com_google_mediapipe//mediapipe/framework/formats:matrix", "@com_google_mediapipe//mediapipe/framework/formats:matrix_data_cc_proto", - "@com_google_mediapipe//mediapipe/framework/port:file_helpers", ], alwayslink = True, ) diff --git a/mediapipe_api/framework/formats/matrix_frame.cc b/mediapipe_api/framework/formats/matrix_data.cc similarity index 96% rename from mediapipe_api/framework/formats/matrix_frame.cc rename to mediapipe_api/framework/formats/matrix_data.cc index aa0718f1b..f91d6200d 100644 --- a/mediapipe_api/framework/formats/matrix_frame.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -4,7 +4,7 @@ // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT. -#include "mediapipe_api/framework/formats/matrix_frame.h" +#include "mediapipe_api/framework/formats/matrix_data.h" MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out) { TRY diff --git a/mediapipe_api/framework/formats/matrix_frame.h b/mediapipe_api/framework/formats/matrix_data.h similarity index 81% rename from mediapipe_api/framework/formats/matrix_frame.h rename to mediapipe_api/framework/formats/matrix_data.h index d833a69a7..e618b082e 100644 --- a/mediapipe_api/framework/formats/matrix_frame.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -4,10 +4,10 @@ // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT. -#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ -#define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ -#include "mediapipe_api/framework/formats/matrix_frame.h" +#include "mediapipe_api/framework/formats/matrix_data.h" #include "mediapipe_api/common.h" #include "mediapipe_api/external/protobuf.h" #include "mediapipe_api/framework/packet.h" @@ -24,4 +24,4 @@ MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_d } // extern "C" -#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_FRAME_H_ +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From ffd978e33271f1c8e1044d6c3db84692931104ef Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 17:36:14 +0200 Subject: [PATCH 09/35] move MatrixClassification example scene to Tutorials - does not represent an official solution - not sure where else to place this - MatrixClassification.cs is an important example for showcasing the usage of a tflite model with a matrix data input --- .../{Samples/Scenes => Tutorial}/Matrix Classification.meta | 0 .../Matrix Classification/Matrix Classification.unity | 0 .../Matrix Classification/Matrix Classification.unity.meta | 0 .../Matrix Classification/MatrixClassification.cs | 0 .../Matrix Classification/MatrixClassification.cs.meta | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename Assets/MediaPipeUnity/{Samples/Scenes => Tutorial}/Matrix Classification.meta (100%) rename Assets/MediaPipeUnity/{Samples/Scenes => Tutorial}/Matrix Classification/Matrix Classification.unity (100%) rename Assets/MediaPipeUnity/{Samples/Scenes => Tutorial}/Matrix Classification/Matrix Classification.unity.meta (100%) rename Assets/MediaPipeUnity/{Samples/Scenes => Tutorial}/Matrix Classification/MatrixClassification.cs (100%) rename Assets/MediaPipeUnity/{Samples/Scenes => Tutorial}/Matrix Classification/MatrixClassification.cs.meta (100%) diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta similarity index 100% rename from Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification.meta rename to Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity similarity index 100% rename from Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity rename to Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta similarity index 100% rename from Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/Matrix Classification.unity.meta rename to Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs similarity index 100% rename from Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs rename to Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta similarity index 100% rename from Assets/MediaPipeUnity/Samples/Scenes/Matrix Classification/MatrixClassification.cs.meta rename to Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta From a723eecf28139cc9f5f350c60eb58fa0a26ff9dc Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 18:10:29 +0200 Subject: [PATCH 10/35] GetArrayPtr() - change access to private --- .../Runtime/Scripts/Framework/Packet/FloatArrayPacket.cs | 2 +- .../Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatArrayPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatArrayPacket.cs index 1b7f4bd80..f9afa6521 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatArrayPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatArrayPacket.cs @@ -78,7 +78,7 @@ public override float[] Get() return result; } - public IntPtr GetArrayPtr() + private IntPtr GetArrayPtr() { UnsafeNativeMethods.mp_Packet__GetFloatArray(mpPtr, out var value).Assert(); GC.KeepAlive(this); diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index aa74105ad..b810a3fc3 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -60,7 +60,7 @@ public override List Get() return result.ToList(); } - public IntPtr GetArrayPtr() + private IntPtr GetArrayPtr() { UnsafeNativeMethods.mp_Packet__GetFloatVector(mpPtr, out var floatFrameVector).Assert(); GC.KeepAlive(this); From 6622fcb86eb9695d4ed25d57d354cff817cdc766 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 18:19:02 +0200 Subject: [PATCH 11/35] MatrixPacket: accept MatrixData as input - before it was byte[] --- .../Matrix Classification/MatrixClassification.cs | 3 +-- .../Scripts/Framework/Packet/MatrixPacket.cs | 13 ++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs index d0aa629e3..a7d9c0213 100644 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs @@ -13,7 +13,6 @@ using System.Collections.Generic; using System.Linq; using UnityEngine; -using Google.Protobuf; namespace Mediapipe.Unity { @@ -111,7 +110,7 @@ private void Start() var matrix = CreateInputData(); // feed data into graph - var input = new MatrixPacket(matrix.ToByteArray(), new Timestamp(i)); + var input = new MatrixPacket(matrix, new Timestamp(i)); graph.AddPacketToInputStream("in", input).AssertOk(); } graph.CloseInputStream("in").AssertOk(); diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs index 81cc07979..d42e7d243 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs @@ -1,10 +1,11 @@ +using Google.Protobuf; using System; namespace Mediapipe { - public class MatrixPacket : Packet + public class MatrixPacket : Packet { private int _length = -1; @@ -31,15 +32,17 @@ public MatrixPacket() : base(true) { } [UnityEngine.Scripting.Preserve] public MatrixPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } - public MatrixPacket(byte[] value) : base() + public MatrixPacket(MatrixData matrixData) : base() { + var value = matrixData.ToByteArray(); UnsafeNativeMethods.mp__MakeMatrixPacket__PKc_i(value, value.Length, out var ptr).Assert(); this.ptr = ptr; length = value.Length; } - public MatrixPacket(byte[] value, Timestamp timestamp) : base() + public MatrixPacket(MatrixData matrixData, Timestamp timestamp) : base() { + var value = matrixData.ToByteArray(); UnsafeNativeMethods.mp__MakeMatrixPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; @@ -53,12 +56,12 @@ public MatrixPacket At(Timestamp timestamp) return packet; } - public override byte[] Get() + public override MatrixData Get() { throw new NotImplementedException(); } - public override StatusOr Consume() + public override StatusOr Consume() { throw new NotImplementedException(); } From 44202abf6710e809a1d36b262f85ccd3615fbf57 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 18:47:42 +0200 Subject: [PATCH 12/35] add license --- .../NativeMethods/Framework/Format/Matrix_Unsafe.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index f3ded97a3..9cd7e8d95 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -1,4 +1,8 @@ - +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. using System; using System.Runtime.InteropServices; @@ -7,14 +11,12 @@ namespace Mediapipe { internal static partial class UnsafeNativeMethods { - #region Packet [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp__MakeMatrixPacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp__MakeMatrixPacket_At__PA_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); - #endregion } } From 69caf3cf926a308b0f86f8a5932d6e6008f2cc07 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 18:49:51 +0200 Subject: [PATCH 13/35] move native functions to Packet_Unsafe - delete FloatVector_Unsafe.cs --- .../Framework/Format/FloatVector_Unsafe.cs | 22 ------------------- .../Format/FloatVector_Unsafe.cs.meta | 11 ---------- .../NativeMethods/Framework/Packet_Unsafe.cs | 14 ++++++++++++ 3 files changed, 14 insertions(+), 33 deletions(-) delete mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs delete mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs deleted file mode 100644 index 5b2e33901..000000000 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; - -namespace Mediapipe -{ - internal static partial class UnsafeNativeMethods - { - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeFloatVectorPacket__PA_i(float[] value, int size, out IntPtr packet); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern void mp_FloatVector__delete(IntPtr floatVector); - } -} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta deleted file mode 100644 index fe31878a4..000000000 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/FloatVector_Unsafe.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 43e481a62e09d8541aff1c80f92c735a -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs index 0e5d03392..ecccc2641 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs @@ -65,6 +65,20 @@ internal static partial class UnsafeNativeMethods public static extern MpReturnCode mp_Packet__ValidateAsFloat(IntPtr packet, out IntPtr status); #endregion + #region FloatVector + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorPacket__PA_i(float[] value, int size, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_FloatVector__delete(IntPtr floatVector); + #endregion + #region Int [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp__MakeIntPacket__i(int value, out IntPtr packet); From 3669c86ddcf9989bb42e497bd1d37c2531e92bec Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:07:15 +0200 Subject: [PATCH 14/35] float_vector.cc -> faster vector allocation --- mediapipe_api/framework/formats/float_vector.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 54aa703e3..a7b53f388 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -4,10 +4,7 @@ MpReturnCode mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out) { TRY - std::vector vector{}; - for (auto i = 0; i < size; ++i) { - vector.push_back(value[i]); - } + std::vector vector(value, value + size); *packet_out = new mediapipe::Packet{mediapipe::MakePacket>(vector)}; RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION @@ -16,10 +13,7 @@ MpReturnCode mp__MakeFloatVectorPacket__PA_i(const float* value, int size, media MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY - std::vector vector{}; - for (auto i = 0; i < size; ++i) { - vector.push_back(value[i]); - } + std::vector vector(value, value + size); *packet_out = new mediapipe::Packet{mediapipe::MakePacket>(vector).At(*timestamp)}; RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION From ebbdddc3b9ba0bd1ed7834e3548484b63c781f96 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:18:39 +0200 Subject: [PATCH 15/35] float_vector.cc remove unused function - delete(...) --- .../Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs | 3 --- mediapipe_api/framework/formats/float_vector.cc | 1 - mediapipe_api/framework/formats/float_vector.h | 1 - 3 files changed, 5 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs index ecccc2641..ad6e9d1d9 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs @@ -74,9 +74,6 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); - - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern void mp_FloatVector__delete(IntPtr floatVector); #endregion #region Int diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index a7b53f388..7a9e7a051 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -26,4 +26,3 @@ MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** CATCH_ALL } -void mp_FloatVector__delete(float* vector_data) { delete[] vector_data; } \ No newline at end of file diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index 45688c976..73645f961 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -19,7 +19,6 @@ MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int si MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); -MP_CAPI(void) mp_FloatVector__delete(float* vector_data); } // extern "C" From 9328084034e3461207c44bf305bd98049581a473 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:19:12 +0200 Subject: [PATCH 16/35] float_vector.h - remove unused headers --- mediapipe_api/framework/formats/float_vector.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index 73645f961..d999909fb 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -7,9 +7,7 @@ #ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ #define MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ -#include "mediapipe_api/framework/formats/float_vector.h" #include "mediapipe_api/common.h" -#include "mediapipe_api/external/protobuf.h" #include "mediapipe_api/framework/packet.h" From b8384bf4fda8d0ac941a4ece607404133ffebf4d Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:20:17 +0200 Subject: [PATCH 17/35] refactor: float_vector.cc TODO: - implement GetFloatVector with vector size as argument --- mediapipe_api/framework/formats/float_vector.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 7a9e7a051..5f02dc590 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -19,10 +19,19 @@ MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, CATCH_EXCEPTION } -MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out) { +MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out) { TRY_ALL - *value_out = packet->Get>().data(); + *data_out = packet->Get>().data(); RETURN_CODE(MpReturnCode::Success); CATCH_ALL } +// TODO: Return with vector size +//MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out, int* size_out) { +// TRY_ALL +// *data_out = packet->Get>().data(); +// RETURN_CODE(MpReturnCode::Success); +// CATCH_ALL +//} + + From 315034068811421355806ad3d371054a3b695972 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:43:11 +0200 Subject: [PATCH 18/35] removed unused headers --- mediapipe_api/framework/formats/matrix_data.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index e618b082e..924806857 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -7,21 +7,18 @@ #ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ #define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ -#include "mediapipe_api/framework/formats/matrix_data.h" #include "mediapipe_api/common.h" #include "mediapipe_api/external/protobuf.h" #include "mediapipe_api/framework/packet.h" #include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/file_helpers.h" -extern "C" { -MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out); -// MP_CAPI(MpReturnCode) mp_Packet__GetMatrixFrame(mediapipe::Packet* packet, const mediapipe::Matrix** value_out); -// MP_CAPI(void) mp_MatrixFrame__delete(mediapipe::Matrix* matrix); + +extern "C" { + MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); + MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From 66f197bc9c328bedb9b98d45cd3a30b815bceaff Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 29 Jul 2022 19:43:43 +0200 Subject: [PATCH 19/35] refactor: float_vector.h --- mediapipe_api/framework/formats/float_vector.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index d999909fb..d28f1d0a4 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -10,14 +10,11 @@ #include "mediapipe_api/common.h" #include "mediapipe_api/framework/packet.h" - extern "C" { - -MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, + MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); + MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); - + MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ From 9c0d9ba336f42c1ab962695ffa4e650310486414 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Mon, 1 Aug 2022 13:04:14 +0200 Subject: [PATCH 20/35] refactor: apply autoformatter on cc files - using format file ".clang-format" in project root --- mediapipe_api/framework/formats/float_vector.cc | 7 ++----- mediapipe_api/framework/formats/float_vector.h | 7 +++---- mediapipe_api/framework/formats/matrix_data.cc | 6 +----- mediapipe_api/framework/formats/matrix_data.h | 12 +++++------- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 5f02dc590..71d3c3c0a 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -10,8 +10,7 @@ MpReturnCode mp__MakeFloatVectorPacket__PA_i(const float* value, int size, media CATCH_EXCEPTION } -MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY std::vector vector(value, value + size); *packet_out = new mediapipe::Packet{mediapipe::MakePacket>(vector).At(*timestamp)}; @@ -27,11 +26,9 @@ MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** } // TODO: Return with vector size -//MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out, int* size_out) { +// MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out, int* size_out) { // TRY_ALL // *data_out = packet->Get>().data(); // RETURN_CODE(MpReturnCode::Success); // CATCH_ALL //} - - diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index d28f1d0a4..7c38f050e 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -11,10 +11,9 @@ #include "mediapipe_api/framework/packet.h" extern "C" { - MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); - MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out); - MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); +MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index f91d6200d..ceadeeb98 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -24,7 +24,7 @@ MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized } MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out) { + mediapipe::Packet** packet_out) { TRY mediapipe::Matrix matrix; @@ -40,7 +40,3 @@ MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_seria RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION } - - - - diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index 924806857..81823cc65 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -7,18 +7,16 @@ #ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ #define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ +#include "mediapipe/framework/formats/matrix.h" #include "mediapipe_api/common.h" #include "mediapipe_api/external/protobuf.h" #include "mediapipe_api/framework/packet.h" -#include "mediapipe/framework/formats/matrix.h" - - - extern "C" { - MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); - MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) +mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) +mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From f9ca7e15670bf57f71ff34abfa16946dc05c70cd Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 11:06:23 +0200 Subject: [PATCH 21/35] refactor: mp__MakeMatrixFramePacket_At__PA_i_Rt -> mp__MakeMatrixPacket_At__PKc_i_Rt --- .../PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs | 2 +- mediapipe_api/framework/formats/matrix_data.cc | 2 +- mediapipe_api/framework/formats/matrix_data.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index 9cd7e8d95..912f225e0 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -16,7 +16,7 @@ internal static partial class UnsafeNativeMethods public static extern MpReturnCode mp__MakeMatrixPacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeMatrixPacket_At__PA_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); + public static extern MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); #endregion } } diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index ceadeeb98..788b4ed70 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -23,7 +23,7 @@ MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized CATCH_EXCEPTION } -MpReturnCode mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, +MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY mediapipe::Matrix matrix; diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index 81823cc65..a3666bd03 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -16,7 +16,7 @@ extern "C" { MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) -mp__MakeMatrixFramePacket_At__PA_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); +mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From 6f3d1b7e2fecf80c7acf4f4d98bd6fbb8080773c Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 12:21:24 +0200 Subject: [PATCH 22/35] FloatVectorPacketTest added - build similar to FloatArrayPacketTest - not yet tested --- .../MatrixClassification.cs | 4 +- .../Framework/Packet/FloatVectorPacket.cs | 14 +- .../NativeMethods/Framework/Packet_Unsafe.cs | 3 + .../Framework/Packet/FloatVectorPacketTest.cs | 124 ++++++++++++++++++ .../Packet/FloatVectorPacketTest.cs.meta | 11 ++ .../framework/formats/float_vector.cc | 7 + .../framework/formats/float_vector.h | 2 + 7 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs create mode 100644 Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs.meta diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs index a7d9c0213..55f00caa8 100644 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs @@ -118,8 +118,8 @@ private void Start() Debug.Log("Poll output"); // Create output container with suitable size // -> size should correspond to tflite model output size - var outputFloatArray = new float[6] { 10, 11, 12, 13, 14, 15 }; - var output = new FloatVectorPacket(outputFloatArray); + var outputFloatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); + var output = new FloatVectorPacket(outputFloatVector); while (poller.Next(output)) { diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index b810a3fc3..be6d6866e 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -19,15 +19,17 @@ public FloatVectorPacket() : base(true) { } [UnityEngine.Scripting.Preserve] public FloatVectorPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } - public FloatVectorPacket(float[] value) : base() + public FloatVectorPacket(List valuesAsList) : base() { + var value = valuesAsList.ToArray(); UnsafeNativeMethods.mp__MakeFloatVectorPacket__PA_i(value, value.Length, out var ptr).Assert(); this.ptr = ptr; _vectorLength = value.Length; } - public FloatVectorPacket(float[] value, Timestamp timestamp) : base() + public FloatVectorPacket(List valuesAsList, Timestamp timestamp) : base() { + var value = valuesAsList.ToArray(); UnsafeNativeMethods.mp__MakeFloatVectorPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; @@ -72,5 +74,13 @@ public override StatusOr> Consume() { throw new NotSupportedException(); } + + public override Status ValidateAsType() + { + UnsafeNativeMethods.mp_Packet__ValidateAsFloatVector(mpPtr, out var statusPtr).Assert(); + + GC.KeepAlive(this); + return new Status(statusPtr); + } } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs index ad6e9d1d9..7caafaa76 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs @@ -74,6 +74,9 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__ValidateAsFloatVector(IntPtr packet, out IntPtr status); #endregion #region Int diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs new file mode 100644 index 000000000..f1fed5ef2 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs @@ -0,0 +1,124 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections.Generic; +using NUnit.Framework; +using System; + +namespace Mediapipe.Tests +{ + public class FloatVectorPacketTest + { + #region Constructor + // [Test, SignalAbort] + // public void Ctor_ShouldInstantiatePacket_When_CalledWithNoArguments() + // { + // using (var packet = new FloatPacket()) + // { + //#pragma warning disable IDE0058 + // Assert.AreEqual(Status.StatusCode.Internal, packet.ValidateAsType().Code()); + // Assert.Throws(() => { packet.Get(); }); + // Assert.AreEqual(Timestamp.Unset(), packet.Timestamp()); + //#pragma warning restore IDE0058 + // } + // } + + [Test] + public void Ctor_ShouldInstantiatePacket_When_CalledWithValue() + { + var floatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); + using (var packet = new FloatVectorPacket(floatVector)) + { + Assert.True(packet.ValidateAsType().Ok()); + Assert.AreEqual(floatVector, packet.Get()); + Assert.AreEqual(Timestamp.Unset(), packet.Timestamp()); + } + } + + //[Test] + //public void Ctor_ShouldInstantiatePacket_When_CalledWithValueAndTimestamp() + //{ + // using (var timestamp = new Timestamp(1)) + // { + // var floatArray = new float[6] { 10, 11, 12, 13, 14, 15 }; + // using (var packet = new FloatPacket(floatArray, timestamp)) + // { + // Assert.True(packet.ValidateAsType().Ok()); + // Assert.AreEqual(0.01f, packet.Get()); + // Assert.AreEqual(timestamp, packet.Timestamp()); + // } + // } + //} + #endregion + + #region #isDisposed + [Test] + public void IsDisposed_ShouldReturnFalse_When_NotDisposedYet() + { + using (var packet = new FloatVectorPacket()) + { + Assert.False(packet.isDisposed); + } + } + + [Test] + public void IsDisposed_ShouldReturnTrue_When_AlreadyDisposed() + { + var packet = new FloatVectorPacket(); + packet.Dispose(); + + Assert.True(packet.isDisposed); + } + #endregion + + #region #At + [Test] + public void At_ShouldReturnNewPacketWithTimestamp() + { + using (var timestamp = new Timestamp(1)) + { + var floatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); + var packet = new FloatVectorPacket(floatVector).At(timestamp); + Assert.AreEqual(floatVector, packet.Get()); + Assert.AreEqual(timestamp, packet.Timestamp()); + + using (var newTimestamp = new Timestamp(2)) + { + var newPacket = packet.At(newTimestamp); + Assert.AreEqual(floatVector, newPacket.Get()); + Assert.AreEqual(newTimestamp, newPacket.Timestamp()); + } + + Assert.AreEqual(timestamp, packet.Timestamp()); + } + } + #endregion + + #region #Consume + [Test] + public void Consume_ShouldThrowNotSupportedException() + { + using (var packet = new FloatVectorPacket()) + { +#pragma warning disable IDE0058 + Assert.Throws(() => { packet.Consume(); }); +#pragma warning restore IDE0058 + } + } + #endregion + + // #region #DebugTypeName + // [Test] + // public void DebugTypeName_ShouldReturnFloat_When_ValueIsSet() + // { + // using (var packet = new FloatPacket(0.01f)) + // { + // Assert.AreEqual("float", packet.DebugTypeName()); + // } + // } + // #endregion + } +} diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs.meta b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs.meta new file mode 100644 index 000000000..15a7a320d --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8a567519c65348c47ae2c5784431179e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 71d3c3c0a..e9db640a8 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -32,3 +32,10 @@ MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** // RETURN_CODE(MpReturnCode::Success); // CATCH_ALL //} + +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out) { + TRY + *status_out = new absl::Status{packet->ValidateAsType>()}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index 7c38f050e..1eac6cf74 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -14,6 +14,8 @@ extern "C" { MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out); + } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_FLOAT_VECTOR_H_ From 4e60769c878c0e079ea66afb1ef724e13749734c Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 15:59:54 +0200 Subject: [PATCH 23/35] fix: float_vector.cc --- mediapipe_api/framework/formats/float_vector.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index e9db640a8..0c40ec728 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -35,7 +35,7 @@ MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out) { TRY - *status_out = new absl::Status{packet->ValidateAsType>()}; + *status_out = new absl::Status{packet->ValidateAsType>()}; RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION } From 8a018c3dea77bc929a608c67bf0d8b422f653380 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 16:00:35 +0200 Subject: [PATCH 24/35] fix: MatrixPacket.cs --- .../Runtime/Scripts/Framework/Packet/MatrixPacket.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs index d42e7d243..3b8bea782 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs @@ -43,7 +43,7 @@ public MatrixPacket(MatrixData matrixData) : base() public MatrixPacket(MatrixData matrixData, Timestamp timestamp) : base() { var value = matrixData.ToByteArray(); - UnsafeNativeMethods.mp__MakeMatrixPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); + UnsafeNativeMethods.mp__MakeMatrixPacket_At__PKc_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; length = value.Length; From 08133c775d6db4de461692eccf97385572fcd7a2 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 17:35:51 +0200 Subject: [PATCH 25/35] fix: Test: FloatVectorPacketTest - Consume_ShouldThrowNotSupportedException --- .../Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index be6d6866e..3b3ad789f 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -11,7 +11,7 @@ public class FloatVectorPacket : Packet> /// /// - private int _vectorLength; + private int _vectorLength = -1; public FloatVectorPacket() : base(true) { } @@ -37,7 +37,9 @@ public FloatVectorPacket(List valuesAsList, Timestamp timestamp) : base() public FloatVectorPacket At(Timestamp timestamp) { - return At(timestamp); + var packet = At(timestamp); + packet._vectorLength = _vectorLength; + return packet; } public override List Get() From 8edacb84404eaaecdb17fcea8a6207542936e029 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 10 Aug 2022 17:58:34 +0200 Subject: [PATCH 26/35] MatrixPacketTest - add - all tests involving packet.Get() do not work - function is not yet implemented --- .../MatrixClassification.cs | 4 +- .../Framework/Format/Matrix_Unsafe.cs | 3 + .../Framework/Packet/MatrixPacketTest.cs | 101 ++++++++++++++++++ .../Framework/Packet/MatrixPacketTest.cs.meta | 11 ++ .../framework/formats/matrix_data.cc | 10 +- mediapipe_api/framework/formats/matrix_data.h | 1 + 6 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs create mode 100644 Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs.meta diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs index 55f00caa8..19c8866cb 100644 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs @@ -107,7 +107,7 @@ private void Start() graph.StartRun().AssertOk(); for (var i = 0; i < 10; i++) { - var matrix = CreateInputData(); + var matrix = CreateMatrixInputData(); // feed data into graph var input = new MatrixPacket(matrix, new Timestamp(i)); @@ -137,7 +137,7 @@ private void Start() Debug.Log("Done"); } - private static MatrixData CreateInputData() + private static MatrixData CreateMatrixInputData() { var matrix = new MatrixData(); matrix.PackedData.Add(0); diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index 912f225e0..8ccd17fed 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -17,6 +17,9 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__ValidateAsMatrix(IntPtr packet, out IntPtr status); #endregion } } diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs new file mode 100644 index 000000000..1e10b29c9 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs @@ -0,0 +1,101 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections.Generic; +using NUnit.Framework; +using System; + +namespace Mediapipe.Tests +{ + public class MatrixPacketTest + { + #region Constructor + [Test] + public void Ctor_ShouldInstantiatePacket_When_CalledWithValue() + { + var matrix = CreateMatrixInputData(); + using (var packet = new MatrixPacket(matrix)) + { + Assert.True(packet.ValidateAsType().Ok()); + Assert.AreEqual(matrix, packet.Get()); + Assert.AreEqual(Timestamp.Unset(), packet.Timestamp()); + } + } + #endregion + + #region #isDisposed + [Test] + public void IsDisposed_ShouldReturnFalse_When_NotDisposedYet() + { + using (var packet = new MatrixPacket()) + { + Assert.False(packet.isDisposed); + } + } + + [Test] + public void IsDisposed_ShouldReturnTrue_When_AlreadyDisposed() + { + var packet = new MatrixPacket(); + packet.Dispose(); + + Assert.True(packet.isDisposed); + } + #endregion + + #region #At + [Test] + public void At_ShouldReturnNewPacketWithTimestamp() + { + using (var timestamp = new Timestamp(1)) + { + var matrix = CreateMatrixInputData(); + var packet = new MatrixPacket(matrix).At(timestamp); + Assert.AreEqual(matrix, packet.Get()); + Assert.AreEqual(timestamp, packet.Timestamp()); + + using (var newTimestamp = new Timestamp(2)) + { + var newPacket = packet.At(newTimestamp); + Assert.AreEqual(matrix, newPacket.Get()); + Assert.AreEqual(newTimestamp, newPacket.Timestamp()); + } + + Assert.AreEqual(timestamp, packet.Timestamp()); + } + } + #endregion + + #region #Consume + [Test] + public void Consume_ShouldThrowNotSupportedException() + { + using (var packet = new MatrixPacket()) + { +#pragma warning disable IDE0058 + Assert.Throws(() => { packet.Consume(); }); +#pragma warning restore IDE0058 + } + } + #endregion + + private static MatrixData CreateMatrixInputData() + { + var matrix = new MatrixData(); + matrix.PackedData.Add(0); + matrix.PackedData.Add(1); + matrix.PackedData.Add(2); + matrix.PackedData.Add(3); + matrix.PackedData.Add(4); + matrix.PackedData.Add(5); + + matrix.Rows = 2; + matrix.Cols = 3; + return matrix; + } + + } +} diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs.meta b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs.meta new file mode 100644 index 000000000..b349e3721 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/MatrixPacketTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a3a4963f37dcdcc43a8292c2493c19bc +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index 788b4ed70..1babca352 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -23,8 +23,7 @@ MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized CATCH_EXCEPTION } -MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY mediapipe::Matrix matrix; @@ -40,3 +39,10 @@ MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialize RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION } + +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out) { + TRY + *status_out = new absl::Status{packet->ValidateAsType()}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} \ No newline at end of file diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index a3666bd03..7e591beb2 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -17,6 +17,7 @@ MP_CAPI(MpReturnCode) mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out); } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From e19cbad31e283edaffdd0f646efe67687f8dff08 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Thu, 11 Aug 2022 18:15:29 +0200 Subject: [PATCH 27/35] fix: Make MatrixClassification.cs run on Android - adding StreamingAssets to ResourceManager [skip actions] --- .../Matrix Classification/MatrixClassification.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs index 19c8866cb..715118233 100644 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs @@ -3,7 +3,10 @@ // You can download it via this Google Drive link: // https://drive.google.com/file/d/1vVDn80eZClBDJ8dJUk3szb9rLaE9p60q/view?usp=sharing // You need to copy the file to -// Packages\com.github.homuler.mediapipe\Runtime\Resources +// Packages\com.github.homuler.mediapipe\Runtime\Resources +// for running in Editor mode and to +// Assets\StreamingAssets +// for Android // Issues: // - Make sure TfliteConverterCalculator is compiled along with the plugin @@ -176,7 +179,14 @@ private WaitForResult WaitForAsset(string assetName, string uniqueKey, bool over } private IEnumerator Init() - { + { + Logger.LogInfo(TAG, "Initializing AssetLoader..."); +#if UNITY_EDITOR + AssetLoader.Provide(new LocalResourceManager()); +#else + AssetLoader.Provide(new StreamingAssetsResourceManager()); +#endif + Logger.LogInfo(TAG, "Loading dependent assets..."); var assetRequests = RequestDependentAssets(); yield return new WaitWhile(() => assetRequests.Any((request) => request.keepWaiting)); From 81f752038248a79790f6d9803ca674d43485b595 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Thu, 22 Sep 2022 17:34:17 +0200 Subject: [PATCH 28/35] Update mediapipe_api/framework/formats/matrix_data.h Co-authored-by: Junrou Nishida --- mediapipe_api/framework/formats/matrix_data.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index 7e591beb2..b11c4441f 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -13,11 +13,12 @@ #include "mediapipe_api/framework/packet.h" extern "C" { -MP_CAPI(MpReturnCode) -mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) -mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); + +MP_CAPI(MpReturnCode) mp__MakeMatrixPacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, + mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out); + } // extern "C" #endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ From ff378edc8cb85e54a9d48d6dd4ad606f224b5641 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Fri, 23 Sep 2022 18:12:06 +0200 Subject: [PATCH 29/35] Apply suggestions from code review Co-authored-by: Junrou Nishida --- .../framework/formats/matrix_data.cc | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index 1babca352..43909a5d2 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -6,16 +6,18 @@ #include "mediapipe_api/framework/formats/matrix_data.h" -MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out) { - TRY - mediapipe::Matrix matrix; +inline mediapipe::MatrixData& ParseFromStringAsMatrixData(const char* serialized_matrix_data, int size) { + mediapipe::MatrixData matrix_data; + CHECK(matrix_data.ParseFromString(std::string(serialized_matrix_data, size))); - // convert matrix data from char into mediapipe::MatrixData - std::string content; - mediapipe::MatrixData matrix_data; - CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); + return matrix_data; +} + +MpReturnCode mp__MakeMatrixPacket__PKc_i(const char* serialized_matrix_data, int size, mediapipe::Packet** packet_out) { + TRY + auto matrix_data = ParseFromStringAsMatrixData(serialized_matrix_data, size); - // fill matrix with data from matrix_data_serialized + mediapipe::Matrix matrix; mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix)}; @@ -23,16 +25,11 @@ MpReturnCode mp__MakeMatrixFramePacket__PKc_i(const char* matrix_data_serialized CATCH_EXCEPTION } -MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* serialized_matrix_data, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY - mediapipe::Matrix matrix; + auto matrix_data = ParseFromStringAsMatrixData(serialized_matrix_data, size); - // convert matrix data from char into mediapipe::MatrixData - std::string content; - mediapipe::MatrixData matrix_data; - CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); - - // fill matrix with data from matrix_data_serialized + mediapipe::Matrix matrix; mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix).At(*timestamp)}; From 9fa813e32b9929d836ed01a2a34af0aad57f57b1 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Mon, 26 Sep 2022 08:00:42 +0200 Subject: [PATCH 30/35] float_vector - return vector size (+2 squashed commit) Squashed commit: [e409b05] refactor: vector_float.cc - naming aligns with files like packet.cc [bad3cd6] float_vector - return vector size --- .../NativeMethods/Framework/Packet_Unsafe.cs | 3 +++ .../framework/formats/float_vector.cc | 21 +++++++++++-------- .../framework/formats/float_vector.h | 1 + 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs index 7caafaa76..3b3d453ec 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs @@ -75,6 +75,9 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value, out int size); + [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__ValidateAsFloatVector(IntPtr packet, out IntPtr status); #endregion diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 0c40ec728..bc27f56dd 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -18,20 +18,23 @@ MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, CATCH_EXCEPTION } -MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out) { +MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out) { TRY_ALL - *data_out = packet->Get>().data(); + *value_out = packet->Get>().data(); RETURN_CODE(MpReturnCode::Success); CATCH_ALL } -// TODO: Return with vector size -// MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** data_out, int* size_out) { -// TRY_ALL -// *data_out = packet->Get>().data(); -// RETURN_CODE(MpReturnCode::Success); -// CATCH_ALL -//} +MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out, int* size_out) { + TRY_ALL + auto& vector_float = packet->Get>(); + auto length = vector_float.size(); + + *value_out = vector_float.data(); + *size_out = length; + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out) { TRY diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index 1eac6cf74..fa6a38e4f 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -14,6 +14,7 @@ extern "C" { MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); +MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out, int* size_out) MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out); } // extern "C" From 14dba0120ef352de4f2dc15048ad4eb06f8c8dc8 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Mon, 26 Sep 2022 11:40:26 +0200 Subject: [PATCH 31/35] fix: matrix_data.cc - wrong func name (+3 squashed commit) Squashed commit: [9245a37] fix: Revert "Apply suggestions from code review" - the below mentioned commit is not working - return value of inline function is invalid -> probably due to inline function This reverts commit c374d617b4d39024153958a5971d746c516a8202. [f597e83] fix: remove duplicate cpp func [6def10c] fix: semicolon omitted --- .../framework/formats/float_vector.cc | 7 ----- .../framework/formats/float_vector.h | 3 +- .../framework/formats/matrix_data.cc | 29 ++++++++++--------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index bc27f56dd..5abe9fd46 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -18,13 +18,6 @@ MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, CATCH_EXCEPTION } -MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out) { - TRY_ALL - *value_out = packet->Get>().data(); - RETURN_CODE(MpReturnCode::Success); - CATCH_ALL -} - MpReturnCode mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out, int* size_out) { TRY_ALL auto& vector_float = packet->Get>(); diff --git a/mediapipe_api/framework/formats/float_vector.h b/mediapipe_api/framework/formats/float_vector.h index fa6a38e4f..8adb1c513 100644 --- a/mediapipe_api/framework/formats/float_vector.h +++ b/mediapipe_api/framework/formats/float_vector.h @@ -13,8 +13,7 @@ extern "C" { MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket__PA_i(const float* value, int size, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp__MakeFloatVectorPacket_At__PA_i_Rt(const float* value, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out); -MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out, int* size_out) +MP_CAPI(MpReturnCode) mp_Packet__GetFloatVector(mediapipe::Packet* packet, const float** value_out, int* size_out); MP_CAPI(MpReturnCode) mp_Packet__ValidateAsFloatVector(mediapipe::Packet* packet, absl::Status** status_out); } // extern "C" diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index 43909a5d2..a255563d7 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -6,18 +6,16 @@ #include "mediapipe_api/framework/formats/matrix_data.h" -inline mediapipe::MatrixData& ParseFromStringAsMatrixData(const char* serialized_matrix_data, int size) { - mediapipe::MatrixData matrix_data; - CHECK(matrix_data.ParseFromString(std::string(serialized_matrix_data, size))); - - return matrix_data; -} - -MpReturnCode mp__MakeMatrixPacket__PKc_i(const char* serialized_matrix_data, int size, mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeMatrixPacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out) { TRY - auto matrix_data = ParseFromStringAsMatrixData(serialized_matrix_data, size); - mediapipe::Matrix matrix; + + // convert matrix data from char into mediapipe::MatrixData + std::string content; + mediapipe::MatrixData matrix_data; + CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); + + // fill matrix with data from matrix_data_serialized mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix)}; @@ -25,11 +23,16 @@ MpReturnCode mp__MakeMatrixPacket__PKc_i(const char* serialized_matrix_data, int CATCH_EXCEPTION } -MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* serialized_matrix_data, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { +MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { TRY - auto matrix_data = ParseFromStringAsMatrixData(serialized_matrix_data, size); - mediapipe::Matrix matrix; + + // convert matrix data from char into mediapipe::MatrixData + std::string content; + mediapipe::MatrixData matrix_data; + CHECK(matrix_data.ParseFromString(std::string(matrix_data_serialized, size))); + + // fill matrix with data from matrix_data_serialized mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix).At(*timestamp)}; From a8ec2a3ec4f13ca5c49b9ca37981e3e7dc9ce488 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Tue, 27 Sep 2022 09:21:42 +0200 Subject: [PATCH 32/35] FloatVectorPacket - replace list by array fix: FloatVectorPacket [skip actions] (+1 squashed commits) Squashed commits: [69302b1] FloatVectorPacket - replace list by array - list is slow --- .../MatrixClassification.cs | 8 ++--- .../Framework/Packet/FloatVectorPacket.cs | 30 +++++++------------ .../Framework/Packet/FloatVectorPacketTest.cs | 4 +-- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs index 715118233..b80def633 100644 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs @@ -104,7 +104,7 @@ private void Start() var graph = new CalculatorGraph(configText); // Specify expected output of tflite model - var poller = graph.AddOutputStreamPoller>("out").Value(); + var poller = graph.AddOutputStreamPoller("out").Value(); Debug.Log("StartRun"); graph.StartRun().AssertOk(); @@ -121,8 +121,8 @@ private void Start() Debug.Log("Poll output"); // Create output container with suitable size // -> size should correspond to tflite model output size - var outputFloatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); - var output = new FloatVectorPacket(outputFloatVector); + + var output = new FloatVectorPacket(); while (poller.Next(output)) { @@ -179,7 +179,7 @@ private WaitForResult WaitForAsset(string assetName, string uniqueKey, bool over } private IEnumerator Init() - { + { Logger.LogInfo(TAG, "Initializing AssetLoader..."); #if UNITY_EDITOR AssetLoader.Provide(new LocalResourceManager()); diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index 3b3ad789f..2495fa2df 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -4,7 +4,7 @@ namespace Mediapipe { - public class FloatVectorPacket : Packet> + public class FloatVectorPacket : Packet { /// /// Creates an empty instance. @@ -19,17 +19,15 @@ public FloatVectorPacket() : base(true) { } [UnityEngine.Scripting.Preserve] public FloatVectorPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } - public FloatVectorPacket(List valuesAsList) : base() + public FloatVectorPacket(float[] value) : base() { - var value = valuesAsList.ToArray(); UnsafeNativeMethods.mp__MakeFloatVectorPacket__PA_i(value, value.Length, out var ptr).Assert(); this.ptr = ptr; _vectorLength = value.Length; } - public FloatVectorPacket(List valuesAsList, Timestamp timestamp) : base() + public FloatVectorPacket(float[] value, Timestamp timestamp) : base() { - var value = valuesAsList.ToArray(); UnsafeNativeMethods.mp__MakeFloatVectorPacket_At__PA_i_Rt(value, value.Length, timestamp.mpPtr, out var ptr).Assert(); GC.KeepAlive(timestamp); this.ptr = ptr; @@ -42,18 +40,20 @@ public FloatVectorPacket At(Timestamp timestamp) return packet; } - public override List Get() + public override float[] Get() { - if (_vectorLength < 0) + UnsafeNativeMethods.mp_Packet__GetFloatVector(mpPtr, out var floatFrameVector, out var size).Assert(); + GC.KeepAlive(this); + if (size < 0) { throw new InvalidOperationException("The array's length is unknown, set Length first"); } - var result = new float[_vectorLength]; + var result = new float[size]; unsafe { - var src = (float*)GetArrayPtr(); + var src = (float*)floatFrameVector; for (var i = 0; i < result.Length; i++) { @@ -61,18 +61,10 @@ public override List Get() } } - return result.ToList(); - } - - private IntPtr GetArrayPtr() - { - UnsafeNativeMethods.mp_Packet__GetFloatVector(mpPtr, out var floatFrameVector).Assert(); - GC.KeepAlive(this); - - return floatFrameVector; + return result; } - public override StatusOr> Consume() + public override StatusOr Consume() { throw new NotSupportedException(); } diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs index f1fed5ef2..dcce600bf 100644 --- a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/FloatVectorPacketTest.cs @@ -29,7 +29,7 @@ public class FloatVectorPacketTest [Test] public void Ctor_ShouldInstantiatePacket_When_CalledWithValue() { - var floatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); + var floatVector = new float[6] { 10, 11, 12, 13, 14, 15 }; using (var packet = new FloatVectorPacket(floatVector)) { Assert.True(packet.ValidateAsType().Ok()); @@ -80,7 +80,7 @@ public void At_ShouldReturnNewPacketWithTimestamp() { using (var timestamp = new Timestamp(1)) { - var floatVector = new List(new float[6] { 10, 11, 12, 13, 14, 15 }); + var floatVector = new float[6] { 10, 11, 12, 13, 14, 15 }; var packet = new FloatVectorPacket(floatVector).At(timestamp); Assert.AreEqual(floatVector, packet.Get()); Assert.AreEqual(timestamp, packet.Timestamp()); From 8e7771d8fa05e9439d86d98785f82e8d6220eb93 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Tue, 27 Sep 2022 11:36:33 +0200 Subject: [PATCH 33/35] Add license headers [skip actions] --- .../Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs | 6 ++++++ .../Runtime/Scripts/Framework/Packet/MatrixPacket.cs | 8 ++++++-- mediapipe_api/framework/formats/float_vector.cc | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs index 2495fa2df..26bbdae5f 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/FloatVectorPacket.cs @@ -1,3 +1,9 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + using System; using System.Collections.Generic; using System.Linq; diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs index 3b8bea782..4bbe84443 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs @@ -1,6 +1,10 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. - -using Google.Protobuf; +using Google.Protobuf; using System; namespace Mediapipe diff --git a/mediapipe_api/framework/formats/float_vector.cc b/mediapipe_api/framework/formats/float_vector.cc index 5abe9fd46..d4e6f3328 100644 --- a/mediapipe_api/framework/formats/float_vector.cc +++ b/mediapipe_api/framework/formats/float_vector.cc @@ -1,3 +1,9 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + #include "mediapipe_api/framework/formats/float_vector.h" #include From 829a3650cb7efe1979a26783119d0a3198d8d904 Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Tue, 27 Sep 2022 11:42:20 +0200 Subject: [PATCH 34/35] Remove Tutorial Scene: MatrixClassification as per request: - deleted demo / tutorial scene that showcasts a simple tflite graph [skip actions] --- .../Tutorial/Matrix Classification.meta | 8 - .../Matrix Classification.unity | 347 ------------------ .../Matrix Classification.unity.meta | 7 - .../MatrixClassification.cs | 209 ----------- .../MatrixClassification.cs.meta | 11 - 5 files changed, 582 deletions(-) delete mode 100644 Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta delete mode 100644 Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity delete mode 100644 Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta delete mode 100644 Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs delete mode 100644 Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta deleted file mode 100644 index 771ea769e..000000000 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 1d0aace15955cf94b8513eb95bd7a753 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity deleted file mode 100644 index a5c93c129..000000000 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity +++ /dev/null @@ -1,347 +0,0 @@ -%YAML 1.1 -%TAG !u! tag:unity3d.com,2011: ---- !u!29 &1 -OcclusionCullingSettings: - m_ObjectHideFlags: 0 - serializedVersion: 2 - m_OcclusionBakeSettings: - smallestOccluder: 5 - smallestHole: 0.25 - backfaceThreshold: 100 - m_SceneGUID: 00000000000000000000000000000000 - m_OcclusionCullingData: {fileID: 0} ---- !u!104 &2 -RenderSettings: - m_ObjectHideFlags: 0 - serializedVersion: 9 - m_Fog: 0 - m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1} - m_FogMode: 3 - m_FogDensity: 0.01 - m_LinearFogStart: 0 - m_LinearFogEnd: 300 - m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1} - m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1} - m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1} - m_AmbientIntensity: 1 - m_AmbientMode: 0 - m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1} - m_SkyboxMaterial: {fileID: 10304, guid: 0000000000000000f000000000000000, type: 0} - m_HaloStrength: 0.5 - m_FlareStrength: 1 - m_FlareFadeSpeed: 3 - m_HaloTexture: {fileID: 0} - m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0} - m_DefaultReflectionMode: 0 - m_DefaultReflectionResolution: 128 - m_ReflectionBounces: 1 - m_ReflectionIntensity: 1 - m_CustomReflection: {fileID: 0} - m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.44657898, g: 0.4964133, b: 0.5748178, a: 1} - m_UseRadianceAmbientProbe: 0 ---- !u!157 &3 -LightmapSettings: - m_ObjectHideFlags: 0 - serializedVersion: 12 - m_GIWorkflowMode: 1 - m_GISettings: - serializedVersion: 2 - m_BounceScale: 1 - m_IndirectOutputScale: 1 - m_AlbedoBoost: 1 - m_EnvironmentLightingMode: 0 - m_EnableBakedLightmaps: 1 - m_EnableRealtimeLightmaps: 0 - m_LightmapEditorSettings: - serializedVersion: 12 - m_Resolution: 2 - m_BakeResolution: 40 - m_AtlasSize: 1024 - m_AO: 0 - m_AOMaxDistance: 1 - m_CompAOExponent: 1 - m_CompAOExponentDirect: 0 - m_ExtractAmbientOcclusion: 0 - m_Padding: 2 - m_LightmapParameters: {fileID: 0} - m_LightmapsBakeMode: 1 - m_TextureCompression: 1 - m_FinalGather: 0 - m_FinalGatherFiltering: 1 - m_FinalGatherRayCount: 256 - m_ReflectionCompression: 2 - m_MixedBakeMode: 2 - m_BakeBackend: 1 - m_PVRSampling: 1 - m_PVRDirectSampleCount: 32 - m_PVRSampleCount: 512 - m_PVRBounces: 2 - m_PVREnvironmentSampleCount: 256 - m_PVREnvironmentReferencePointCount: 2048 - m_PVRFilteringMode: 1 - m_PVRDenoiserTypeDirect: 1 - m_PVRDenoiserTypeIndirect: 1 - m_PVRDenoiserTypeAO: 1 - m_PVRFilterTypeDirect: 0 - m_PVRFilterTypeIndirect: 0 - m_PVRFilterTypeAO: 0 - m_PVREnvironmentMIS: 1 - m_PVRCulling: 1 - m_PVRFilteringGaussRadiusDirect: 1 - m_PVRFilteringGaussRadiusIndirect: 5 - m_PVRFilteringGaussRadiusAO: 2 - m_PVRFilteringAtrousPositionSigmaDirect: 0.5 - m_PVRFilteringAtrousPositionSigmaIndirect: 2 - m_PVRFilteringAtrousPositionSigmaAO: 1 - m_ExportTrainingData: 0 - m_TrainingDataDestination: TrainingData - m_LightProbeSampleCountMultiplier: 4 - m_LightingDataAsset: {fileID: 0} - m_LightingSettings: {fileID: 0} ---- !u!196 &4 -NavMeshSettings: - serializedVersion: 2 - m_ObjectHideFlags: 0 - m_BuildSettings: - serializedVersion: 2 - agentTypeID: 0 - agentRadius: 0.5 - agentHeight: 2 - agentSlope: 45 - agentClimb: 0.4 - ledgeDropHeight: 0 - maxJumpAcrossDistance: 0 - minRegionArea: 2 - manualCellSize: 0 - cellSize: 0.16666667 - manualTileSize: 0 - tileSize: 256 - accuratePlacement: 0 - maxJobWorkers: 0 - preserveTilesOutsideBounds: 0 - debug: - m_Flags: 0 - m_NavMeshData: {fileID: 0} ---- !u!1 &938836659 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 938836661} - - component: {fileID: 938836662} - m_Layer: 0 - m_Name: Matrix Classification - m_TagString: Untagged - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!4 &938836661 -Transform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 938836659} - m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: 0, y: 0, z: 0} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 0} - m_RootOrder: 2 - m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &938836662 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 938836659} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 87ae56d766402f849ba1ad498f5f2f67, type: 3} - m_Name: - m_EditorClassIdentifier: ---- !u!1 &984312839 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 984312842} - - component: {fileID: 984312841} - - component: {fileID: 984312840} - m_Layer: 0 - m_Name: Main Camera - m_TagString: MainCamera - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!81 &984312840 -AudioListener: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 984312839} - m_Enabled: 1 ---- !u!20 &984312841 -Camera: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 984312839} - m_Enabled: 1 - serializedVersion: 2 - m_ClearFlags: 1 - m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0} - m_projectionMatrixMode: 1 - m_GateFitMode: 2 - m_FOVAxisMode: 0 - m_SensorSize: {x: 36, y: 24} - m_LensShift: {x: 0, y: 0} - m_FocalLength: 50 - m_NormalizedViewPortRect: - serializedVersion: 2 - x: 0 - y: 0 - width: 1 - height: 1 - near clip plane: 0.3 - far clip plane: 1000 - field of view: 60 - orthographic: 0 - orthographic size: 5 - m_Depth: -1 - m_CullingMask: - serializedVersion: 2 - m_Bits: 4294967295 - m_RenderingPath: -1 - m_TargetTexture: {fileID: 0} - m_TargetDisplay: 0 - m_TargetEye: 3 - m_HDR: 1 - m_AllowMSAA: 1 - m_AllowDynamicResolution: 0 - m_ForceIntoRT: 0 - m_OcclusionCulling: 1 - m_StereoConvergence: 10 - m_StereoSeparation: 0.022 ---- !u!4 &984312842 -Transform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 984312839} - m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: 0, y: 1, z: -10} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 0} - m_RootOrder: 0 - m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!1 &1144986087 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 1144986089} - - component: {fileID: 1144986088} - m_Layer: 0 - m_Name: Directional Light - m_TagString: Untagged - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!108 &1144986088 -Light: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1144986087} - m_Enabled: 1 - serializedVersion: 10 - m_Type: 1 - m_Shape: 0 - m_Color: {r: 1, g: 0.95686275, b: 0.8392157, a: 1} - m_Intensity: 1 - m_Range: 10 - m_SpotAngle: 30 - m_InnerSpotAngle: 21.80208 - m_CookieSize: 10 - m_Shadows: - m_Type: 2 - m_Resolution: -1 - m_CustomResolution: -1 - m_Strength: 1 - m_Bias: 0.05 - m_NormalBias: 0.4 - m_NearPlane: 0.2 - m_CullingMatrixOverride: - e00: 1 - e01: 0 - e02: 0 - e03: 0 - e10: 0 - e11: 1 - e12: 0 - e13: 0 - e20: 0 - e21: 0 - e22: 1 - e23: 0 - e30: 0 - e31: 0 - e32: 0 - e33: 1 - m_UseCullingMatrixOverride: 0 - m_Cookie: {fileID: 0} - m_DrawHalo: 0 - m_Flare: {fileID: 0} - m_RenderMode: 0 - m_CullingMask: - serializedVersion: 2 - m_Bits: 4294967295 - m_RenderingLayerMask: 1 - m_Lightmapping: 4 - m_LightShadowCasterMode: 0 - m_AreaSize: {x: 1, y: 1} - m_BounceIntensity: 1 - m_ColorTemperature: 6570 - m_UseColorTemperature: 0 - m_BoundingSphereOverride: {x: 0, y: 0, z: 0, w: 0} - m_UseBoundingSphereOverride: 0 - m_UseViewFrustumForShadowCasterCull: 1 - m_ShadowRadius: 0 - m_ShadowAngle: 0 ---- !u!4 &1144986089 -Transform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1144986087} - m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261} - m_LocalPosition: {x: 0, y: 3, z: 0} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 0} - m_RootOrder: 1 - m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0} diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta deleted file mode 100644 index afa52ba3e..000000000 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/Matrix Classification.unity.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 2f27ce48d99be3841b0d4e4f6c4fb0c9 -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs deleted file mode 100644 index b80def633..000000000 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs +++ /dev/null @@ -1,209 +0,0 @@ -// TODO: -// THIS class needs the following binary to work: "adder_model_single_input_2x3.bytes" -// You can download it via this Google Drive link: -// https://drive.google.com/file/d/1vVDn80eZClBDJ8dJUk3szb9rLaE9p60q/view?usp=sharing -// You need to copy the file to -// Packages\com.github.homuler.mediapipe\Runtime\Resources -// for running in Editor mode and to -// Assets\StreamingAssets -// for Android - -// Issues: -// - Make sure TfliteConverterCalculator is compiled along with the plugin - - -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using UnityEngine; - -namespace Mediapipe.Unity -{ - /// - /// Toy example on how to use tflite model for matrix classification. - /// In contrast to other mediapipe graphs, this graph does not expect an image as input data, - /// but deals with matrix input data. Internally the input (MatrixData) will be converted - /// into an Eigen::MatrixXf which is then fed to the first calculator node of the graph - /// (TfLiteConverterCalculator). From there the input is converted to a tflite tensor, forwarded - /// throught the neural network (tflite model). The output of the neural network is converted into - /// a std::vector and and finally passed back into Unity as a List - /// - /// Example: This code could be used for a pose based action classifier. - /// Input: Matrix containing values of an input pose - /// Output: Vector of action probabilities - /// - /// Since matrix can only store 2D input data, but tflite models generally expect 4D input (NHWC), - /// the trick is to collapse the exceeding dimensions such that tflite can parse the 4D data from a - /// 2D matrix. Since the input matrix (Eigen::MatrixXf) is stored in column major order, the input 2D - /// matrix has to be transposed befored feeding it into the graph. - /// - /// Example: matrix of dimension [num_frames = 10, num_landmarks = 33, num_coordinates = 3] - /// - collapse into matrix of shape [10 x 99] - /// - now transpose to make "coordinate" dimension "fastest" -> output shape is now [99x10] - /// - feed the data into the graph - /// - /// TODO: - /// In theory mediapipe can also handle matrices stored in row-major order - /// -> write code for row-major matrix input to mediapipe graph - /// - public class MatrixClassification : MonoBehaviour - { - - private readonly string TAG = "MatrixClassificationToyExample"; - - private void OnEnable() - { - var _ = StartCoroutine(Init()); - } - - private void Start() - { - Debug.Log("Setup Protobuf Logging"); - Protobuf.SetLogHandler(Protobuf.DefaultLogHandler); - - Debug.Log("Start"); - var configText = @" - input_stream: ""MATRIX:in"" - output_stream: ""FLOATS:out"" - - node - { - calculator: ""TfLiteConverterCalculator"" - input_stream: ""MATRIX:in"" - output_stream: ""TENSORS:image_tensor"" - options: - { - [mediapipe.TfLiteConverterCalculatorOptions.ext] - { - zero_center: false - } - } - } - - node - { - calculator: ""TfLiteInferenceCalculator"" - input_stream: ""TENSORS:image_tensor"" - output_stream: ""TENSORS:tensor_features"" - options: - { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] - { - model_path: ""mediapipe/models/adder_model_single_input_2x3.tflite"" - } - } - } - - node - { - calculator: ""TfLiteTensorsToFloatsCalculator"" - input_stream: ""TENSORS:tensor_features"" - output_stream: ""FLOATS:out"" - } - "; - var graph = new CalculatorGraph(configText); - - // Specify expected output of tflite model - var poller = graph.AddOutputStreamPoller("out").Value(); - - Debug.Log("StartRun"); - graph.StartRun().AssertOk(); - for (var i = 0; i < 10; i++) - { - var matrix = CreateMatrixInputData(); - - // feed data into graph - var input = new MatrixPacket(matrix, new Timestamp(i)); - graph.AddPacketToInputStream("in", input).AssertOk(); - } - graph.CloseInputStream("in").AssertOk(); - - Debug.Log("Poll output"); - // Create output container with suitable size - // -> size should correspond to tflite model output size - - var output = new FloatVectorPacket(); - - while (poller.Next(output)) - { - var result = output.Get(); - foreach (var item in result) - { - Debug.Log("result array: " + item); - } - } - - - graph.WaitUntilDone().AssertOk(); - graph.Dispose(); - - Debug.Log("Done"); - } - - private static MatrixData CreateMatrixInputData() - { - var matrix = new MatrixData(); - matrix.PackedData.Add(0); - matrix.PackedData.Add(1); - matrix.PackedData.Add(2); - matrix.PackedData.Add(3); - matrix.PackedData.Add(4); - matrix.PackedData.Add(5); - - matrix.Rows = 2; - matrix.Cols = 3; - return matrix; - } - - private void OnApplicationQuit() - { - Protobuf.ResetLogHandler(); - } - - // Load tflite model from assets - private IList RequestDependentAssets() - { - return new List { - WaitForAsset("adder_model_single_input_2x3.bytes"), - }; - } - - private WaitForResult WaitForAsset(string assetName, bool overwrite = false) - { - return WaitForAsset(assetName, assetName, overwrite); - } - - private WaitForResult WaitForAsset(string assetName, string uniqueKey, bool overwrite = false) - { - return new WaitForResult(this, AssetLoader.PrepareAssetAsync(assetName, uniqueKey, overwrite)); - } - - private IEnumerator Init() - { - Logger.LogInfo(TAG, "Initializing AssetLoader..."); -#if UNITY_EDITOR - AssetLoader.Provide(new LocalResourceManager()); -#else - AssetLoader.Provide(new StreamingAssetsResourceManager()); -#endif - - Logger.LogInfo(TAG, "Loading dependent assets..."); - var assetRequests = RequestDependentAssets(); - yield return new WaitWhile(() => assetRequests.Any((request) => request.keepWaiting)); - - var errors = assetRequests.Where((request) => request.isError).Select((request) => request.error).ToList(); - if (errors.Count > 0) - { - foreach (var error in errors) - { - Logger.LogError(TAG, error); - } - throw new InternalException("Failed to prepare dependent assets"); - } - } - - } -} - - - diff --git a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta b/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta deleted file mode 100644 index eb475807b..000000000 --- a/Assets/MediaPipeUnity/Tutorial/Matrix Classification/MatrixClassification.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 87ae56d766402f849ba1ad498f5f2f67 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: From 861752f9fe93cdc8504c8c896f58f38810f9e9cd Mon Sep 17 00:00:00 2001 From: Martin Garbade Date: Wed, 12 Oct 2022 12:10:51 +0200 Subject: [PATCH 35/35] fix: MatrixPacket tests - new GetMatrix function Caveat: - MatrixPacket: Consume throws NotSupportedException() -> not sure if this is a useful test, but such tests exists in similar classes as well (cherry picked from commit 707af5f454e87312a86b60deaebec18463e47ded) --- .../Scripts/Framework/Packet/MatrixPacket.cs | 18 ++++++++++++++++-- .../Framework/Format/Matrix_Unsafe.cs | 4 ++++ .../NativeMethods/Framework/Packet_Unsafe.cs | 3 --- .../framework/formats/matrix_data.cc | 19 ++++++++++++++++++- mediapipe_api/framework/formats/matrix_data.h | 1 + 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs index 4bbe84443..04fce4cdc 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/MatrixPacket.cs @@ -62,12 +62,26 @@ public MatrixPacket At(Timestamp timestamp) public override MatrixData Get() { - throw new NotImplementedException(); + UnsafeNativeMethods.mp_Packet__GetMatrix(mpPtr, out var serializedMatrixData).Assert(); + GC.KeepAlive(this); + + var matrixData = serializedMatrixData.Deserialize(MatrixData.Parser); + serializedMatrixData.Dispose(); + + return matrixData; } public override StatusOr Consume() { - throw new NotImplementedException(); + throw new NotSupportedException(); + } + + public override Status ValidateAsType() + { + UnsafeNativeMethods.mp_Packet__ValidateAsMatrix(mpPtr, out var statusPtr).Assert(); + + GC.KeepAlive(this); + return new Status(statusPtr); } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index 8ccd17fed..bfc16aec5 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -20,6 +20,10 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__ValidateAsMatrix(IntPtr packet, out IntPtr status); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetMatrix(IntPtr packet, out SerializedProto serializedProto); + #endregion } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs index 3b3d453ec..db505c43b 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Packet_Unsafe.cs @@ -72,9 +72,6 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp__MakeFloatVectorPacket_At__PA_i_Rt(float[] value, int size, IntPtr timestamp, out IntPtr packet); - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value); - [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_Packet__GetFloatVector(IntPtr packet, out IntPtr value, out int size); diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc index a255563d7..441bf6345 100644 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ b/mediapipe_api/framework/formats/matrix_data.cc @@ -45,4 +45,21 @@ MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, abs *status_out = new absl::Status{packet->ValidateAsType()}; RETURN_CODE(MpReturnCode::Success); CATCH_EXCEPTION -} \ No newline at end of file +} + +MP_CAPI(MpReturnCode) mp_Packet__GetMatrix(mediapipe::Packet* packet, mp_api::SerializedProto* value_out) { + TRY + // Get Eigen::Matrix from packet + mediapipe::Matrix matrix; + matrix = packet->Get(); + + // Convert to format that can be send to Unity + mediapipe::MatrixData matrix_data; + mediapipe::MatrixDataProtoFromMatrix(matrix, &matrix_data); + + // auto matrix_data_serialized = new mp_api::SerializedProto(); + SerializeProto(matrix_data, value_out); + + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h index b11c4441f..e56e953be 100644 --- a/mediapipe_api/framework/formats/matrix_data.h +++ b/mediapipe_api/framework/formats/matrix_data.h @@ -18,6 +18,7 @@ MP_CAPI(MpReturnCode) mp__MakeMatrixPacket__PKc_i(const char* matrix_data_serial MP_CAPI(MpReturnCode) mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out); +MP_CAPI(MpReturnCode) mp_Packet__GetMatrix(mediapipe::Packet* packet, mp_api::SerializedProto* value_out); } // extern "C"