Skip to content
Permalink
Browse files

Project import generated by Copybara.

GitOrigin-RevId: b137378673f7d66d41bcd46e4fc3a0d9ef254894
  • Loading branch information
MediaPipe Team jqtang
MediaPipe Team authored and jqtang committed Oct 25, 2019
1 parent a2a63e3 commit 259b48e08263a9d363b6d7d589489ccb2dbeab8a
Showing with 8,564 additions and 398 deletions.
  1. +6 −1 README.md
  2. +2 −1 WORKSPACE
  3. +67 −0 mediapipe/calculators/core/BUILD
  4. +90 −0 mediapipe/calculators/core/dequantize_byte_array_calculator.cc
  5. +28 −0 mediapipe/calculators/core/dequantize_byte_array_calculator.proto
  6. +137 −0 mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc
  7. +6 −0 mediapipe/calculators/core/previous_loopback_calculator.cc
  8. +91 −0 mediapipe/calculators/core/previous_loopback_calculator_test.cc
  9. +83 −0 mediapipe/calculators/core/side_packet_to_stream_calculator.cc
  10. +66 −29 mediapipe/calculators/core/split_vector_calculator.h
  11. +3 −0 mediapipe/calculators/core/split_vector_calculator.proto
  12. +134 −0 mediapipe/calculators/core/split_vector_calculator_test.cc
  13. +79 −0 mediapipe/calculators/core/string_to_int_calculator.cc
  14. +80 −0 mediapipe/calculators/tensorflow/BUILD
  15. +50 −11 mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc
  16. +126 −0 mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc
  17. +192 −0 mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc
  18. +4 −2 mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc
  19. +203 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc
  20. +43 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto
  21. +202 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc
  22. +22 −13 mediapipe/calculators/tflite/tflite_converter_calculator.cc
  23. +37 −17 mediapipe/calculators/tflite/tflite_inference_calculator.cc
  24. +15 −11 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc
  25. +33 −14 mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
  26. +2 −4 mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
  27. +26 −13 mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc
  28. +99 −2 mediapipe/calculators/util/BUILD
  29. +40 −6 mediapipe/calculators/util/annotation_overlay_calculator.cc
  30. +2 −2 mediapipe/calculators/util/detection_label_id_to_text_calculator.cc
  31. +17 −78 mediapipe/calculators/util/detections_to_rects_calculator.cc
  32. +105 −0 mediapipe/calculators/util/detections_to_rects_calculator.h
  33. +181 −0 mediapipe/calculators/util/labels_to_render_data_calculator.cc
  34. +62 −0 mediapipe/calculators/util/labels_to_render_data_calculator.proto
  35. +138 −0 mediapipe/calculators/util/landmarks_to_floats_calculator.cc
  36. +28 −0 mediapipe/calculators/util/landmarks_to_floats_calculator.proto
  37. +57 −0 mediapipe/calculators/util/local_file_contents_calculator.cc
  38. +46 −4 mediapipe/calculators/util/top_k_scores_calculator.cc
  39. +130 −0 mediapipe/docs/android_archive_library.md
  40. +3 −2 mediapipe/docs/examples.md
  41. +2 −4 mediapipe/docs/face_detection_desktop.md
  42. +1 −2 mediapipe/docs/hair_segmentation_desktop.md
  43. +2 −4 mediapipe/docs/hand_tracking_desktop.md
  44. BIN mediapipe/docs/images/mobile/aar_location.png
  45. BIN mediapipe/docs/images/mobile/android_studio_opencv_location.png
  46. BIN mediapipe/docs/images/mobile/assets_location.png
  47. +40 −32 mediapipe/docs/install.md
  48. +3 −6 mediapipe/docs/object_detection_desktop.md
  49. +91 −10 mediapipe/docs/youtube_8m.md
  50. +2 −0 mediapipe/examples/desktop/BUILD
  51. +86 −6 mediapipe/examples/desktop/simple_run_graph_main.cc
  52. +11 −0 mediapipe/examples/desktop/youtube8m/BUILD
  53. +96 −8 mediapipe/examples/desktop/youtube8m/README.md
  54. +262 −0 mediapipe/examples/desktop/youtube8m/viewer/server.py
  55. +96 −0 mediapipe/examples/desktop/youtube8m/viewer/static/index.html
  56. +217 −0 mediapipe/examples/desktop/youtube8m/viewer/static/main.js
  57. +13 −0 mediapipe/framework/BUILD
  58. +89 −1 mediapipe/framework/calculator_graph_bounds_test.cc
  59. +61 −0 mediapipe/framework/calculator_graph_side_packet_test.cc
  60. +43 −4 mediapipe/framework/calculator_node.cc
  61. +3 −0 mediapipe/framework/calculator_node.h
  62. +16 −0 mediapipe/framework/collection.h
  63. +8 −6 mediapipe/framework/demangle.h
  64. +2 −3 mediapipe/framework/deps/BUILD
  65. +4 −0 mediapipe/framework/formats/image_format.proto
  66. +6 −0 mediapipe/framework/formats/image_frame.cc
  67. +3 −0 mediapipe/framework/formats/image_frame_opencv.cc
  68. +5 −0 mediapipe/framework/formats/landmark.proto
  69. +7 −0 mediapipe/framework/input_side_packet_handler.cc
  70. +5 −0 mediapipe/framework/input_side_packet_handler.h
  71. +3 −2 mediapipe/framework/output_side_packet_impl.cc
  72. +1 −0 mediapipe/framework/output_side_packet_impl.h
  73. +8 −0 mediapipe/framework/packet.h
  74. +18 −0 mediapipe/framework/port.h
  75. +169 −77 mediapipe/framework/profiler/graph_profiler_test.cc
  76. +14 −3 mediapipe/framework/profiler/trace_buffer.h
  77. +16 −16 mediapipe/framework/profiler/trace_builder.cc
  78. +5 −0 mediapipe/framework/tool/tag_map.h
  79. +4 −0 mediapipe/framework/tool/template_expander.cc
  80. +2 −2 mediapipe/framework/tool/template_parser.cc
  81. +4 −0 mediapipe/gpu/gl_simple_shaders.h
  82. +27 −0 mediapipe/graphs/youtube8m/BUILD
  83. +3,862 −0 mediapipe/graphs/youtube8m/label_map.txt
  84. +178 −0 mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt
  85. +139 −0 mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt
  86. +15 −0 mediapipe/java/com/google/mediapipe/BUILD
  87. +7 −0 mediapipe/java/com/google/mediapipe/components/BUILD
  88. +3 −1 mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
  89. +7 −0 mediapipe/java/com/google/mediapipe/framework/BUILD
  90. +7 −0 mediapipe/java/com/google/mediapipe/glutil/BUILD
  91. +157 −0 mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl
  92. +2 −0 mediapipe/util/sequence/README.md
  93. +3 −1 mediapipe/util/sequence/media_sequence.py
  94. +4 −0 mediapipe/util/sequence/media_sequence_test.py
@@ -37,10 +37,15 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe * [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe


## Publications ## Publications
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172) * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)


## Events ## Events
[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA * [ML Conference, Berlin 9-11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
* [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA


## Alpha Disclaimer ## Alpha Disclaimer
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0. MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
@@ -10,7 +10,8 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
) )
load("@bazel_skylib//lib:versions.bzl", "versions") load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1") versions.check(minimum_bazel_version = "0.24.1",
maximum_bazel_version = "0.29.1")


# ABSL cpp library. # ABSL cpp library.
http_archive( http_archive(
@@ -26,6 +26,13 @@ proto_library(
deps = ["//mediapipe/framework:calculator_proto"], deps = ["//mediapipe/framework:calculator_proto"],
) )


proto_library(
name = "dequantize_byte_array_calculator_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)

proto_library( proto_library(
name = "packet_cloner_calculator_proto", name = "packet_cloner_calculator_proto",
srcs = ["packet_cloner_calculator.proto"], srcs = ["packet_cloner_calculator.proto"],
@@ -104,6 +111,14 @@ mediapipe_cc_proto_library(
deps = [":concatenate_vector_calculator_proto"], deps = [":concatenate_vector_calculator_proto"],
) )


mediapipe_cc_proto_library(
name = "dequantize_byte_array_calculator_cc_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":dequantize_byte_array_calculator_proto"],
)

mediapipe_cc_proto_library( mediapipe_cc_proto_library(
name = "quantize_float_vector_calculator_cc_proto", name = "quantize_float_vector_calculator_cc_proto",
srcs = ["quantize_float_vector_calculator.proto"], srcs = ["quantize_float_vector_calculator.proto"],
@@ -387,6 +402,32 @@ cc_library(
alwayslink = 1, alwayslink = 1,
) )


cc_library(
name = "string_to_int_calculator",
srcs = ["string_to_int_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)

cc_library(
name = "side_packet_to_stream_calculator",
srcs = ["side_packet_to_stream_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test( cc_test(
name = "immediate_mux_calculator_test", name = "immediate_mux_calculator_test",
srcs = ["immediate_mux_calculator_test.cc"], srcs = ["immediate_mux_calculator_test.cc"],
@@ -558,6 +599,32 @@ cc_test(
], ],
) )


cc_library(
name = "dequantize_byte_array_calculator",
srcs = ["dequantize_byte_array_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":dequantize_byte_array_calculator_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test(
name = "dequantize_byte_array_calculator_test",
srcs = ["dequantize_byte_array_calculator_test.cc"],
deps = [
":dequantize_byte_array_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
],
)

cc_library( cc_library(
name = "quantize_float_vector_calculator", name = "quantize_float_vector_calculator",
srcs = ["quantize_float_vector_calculator.cc"], srcs = ["quantize_float_vector_calculator.cc"],
@@ -0,0 +1,90 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cfloat>

#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/status.h"

// Dequantizes a byte array to a vector of floats.
//
// Example config:
// node {
// calculator: "DequantizeByteArrayCalculator"
// input_stream: "ENCODED:encoded"
// output_stream: "FLOAT_VECTOR:float_vector"
// options {
// [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
// max_quantized_value: 2
// min_quantized_value: -2
// }
// }
// }
namespace mediapipe {

class DequantizeByteArrayCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag("ENCODED").Set<std::string>();
cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
return ::mediapipe::OkStatus();
}

::mediapipe::Status Open(CalculatorContext* cc) final {
const auto options =
cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
if (!options.has_max_quantized_value() ||
!options.has_min_quantized_value()) {
return ::mediapipe::InvalidArgumentError(
"Both max_quantized_value and min_quantized_value must be provided "
"in DequantizeByteArrayCalculatorOptions.");
}
float max_quantized_value = options.max_quantized_value();
float min_quantized_value = options.min_quantized_value();
if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
return ::mediapipe::InvalidArgumentError(
"max_quantized_value must be greater than min_quantized_value.");
}
float range = max_quantized_value - min_quantized_value;
scalar_ = range / 255.0;
bias_ = (range / 512.0) + min_quantized_value;
return ::mediapipe::OkStatus();
}

::mediapipe::Status Process(CalculatorContext* cc) final {
const std::string& encoded =
cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
std::vector<float> float_vector;
float_vector.reserve(encoded.length());
for (int i = 0; i < encoded.length(); ++i) {
float_vector.push_back(
static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
}
cc->Outputs()
.Tag("FLOAT_VECTOR")
.AddPacket(MakePacket<std::vector<float>>(float_vector)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}

private:
float scalar_;
float bias_;
};

REGISTER_CALCULATOR(DequantizeByteArrayCalculator);

} // namespace mediapipe
@@ -0,0 +1,28 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";

package mediapipe;

import "mediapipe/framework/calculator.proto";

message DequantizeByteArrayCalculatorOptions {
extend CalculatorOptions {
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
}

optional float max_quantized_value = 1;
optional float min_quantized_value = 2;
}
@@ -0,0 +1,137 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <string>
#include <vector>

#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h" // NOLINT

namespace mediapipe {

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 2
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"Both max_quantized_value and min_quantized_value must be provided"));
}

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: -2
min_quantized_value: 2
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"max_quantized_value must be greater than min_quantized_value"));
}

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 1
min_quantized_value: 1
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"max_quantized_value must be greater than min_quantized_value"));
}

TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 2
min_quantized_value: -2
}
}
)");
CalculatorRunner runner(node_config);
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(
std::string(reinterpret_cast<char const*>(input), 4))
.At(Timestamp(0)));
auto status = runner.Run();
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs =
runner.Outputs().Tag("FLOAT_VECTOR").packets;
EXPECT_EQ(1, outputs.size());
const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
ASSERT_FALSE(result.empty());
EXPECT_EQ(4, result.size());
EXPECT_NEAR(0, result[0], 0.01);
EXPECT_NEAR(2, result[1], 0.01);
EXPECT_NEAR(-2, result[2], 0.01);
EXPECT_NEAR(-1.976, result[3], 0.01);

EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
}

} // namespace mediapipe
@@ -102,6 +102,12 @@ class PreviousLoopbackCalculator : public CalculatorBase {
cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback)); cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
} }
} }
if (!main_ts_.empty()) {
cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
}
if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
cc->Outputs().Get(loop_out_id_).Close();
}
return ::mediapipe::OkStatus(); return ::mediapipe::OkStatus();
} }


0 comments on commit 259b48e

Please sign in to comment.
You can’t perform that action at this time.