Skip to content
Permalink
Browse files

Project import generated by Copybara.

GitOrigin-RevId: b137378673f7d66d41bcd46e4fc3a0d9ef254894
  • Loading branch information...
MediaPipe Team jqtang
MediaPipe Team authored and jqtang committed Oct 25, 2019
1 parent a2a63e3 commit 259b48e08263a9d363b6d7d589489ccb2dbeab8a
Showing with 8,564 additions and 398 deletions.
  1. +6 −1 README.md
  2. +2 −1 WORKSPACE
  3. +67 −0 mediapipe/calculators/core/BUILD
  4. +90 −0 mediapipe/calculators/core/dequantize_byte_array_calculator.cc
  5. +28 −0 mediapipe/calculators/core/dequantize_byte_array_calculator.proto
  6. +137 −0 mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc
  7. +6 −0 mediapipe/calculators/core/previous_loopback_calculator.cc
  8. +91 −0 mediapipe/calculators/core/previous_loopback_calculator_test.cc
  9. +83 −0 mediapipe/calculators/core/side_packet_to_stream_calculator.cc
  10. +66 −29 mediapipe/calculators/core/split_vector_calculator.h
  11. +3 −0 mediapipe/calculators/core/split_vector_calculator.proto
  12. +134 −0 mediapipe/calculators/core/split_vector_calculator_test.cc
  13. +79 −0 mediapipe/calculators/core/string_to_int_calculator.cc
  14. +80 −0 mediapipe/calculators/tensorflow/BUILD
  15. +50 −11 mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc
  16. +126 −0 mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc
  17. +192 −0 mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc
  18. +4 −2 mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc
  19. +203 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc
  20. +43 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto
  21. +202 −0 mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc
  22. +22 −13 mediapipe/calculators/tflite/tflite_converter_calculator.cc
  23. +37 −17 mediapipe/calculators/tflite/tflite_inference_calculator.cc
  24. +15 −11 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc
  25. +33 −14 mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
  26. +2 −4 mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
  27. +26 −13 mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc
  28. +99 −2 mediapipe/calculators/util/BUILD
  29. +40 −6 mediapipe/calculators/util/annotation_overlay_calculator.cc
  30. +2 −2 mediapipe/calculators/util/detection_label_id_to_text_calculator.cc
  31. +17 −78 mediapipe/calculators/util/detections_to_rects_calculator.cc
  32. +105 −0 mediapipe/calculators/util/detections_to_rects_calculator.h
  33. +181 −0 mediapipe/calculators/util/labels_to_render_data_calculator.cc
  34. +62 −0 mediapipe/calculators/util/labels_to_render_data_calculator.proto
  35. +138 −0 mediapipe/calculators/util/landmarks_to_floats_calculator.cc
  36. +28 −0 mediapipe/calculators/util/landmarks_to_floats_calculator.proto
  37. +57 −0 mediapipe/calculators/util/local_file_contents_calculator.cc
  38. +46 −4 mediapipe/calculators/util/top_k_scores_calculator.cc
  39. +130 −0 mediapipe/docs/android_archive_library.md
  40. +3 −2 mediapipe/docs/examples.md
  41. +2 −4 mediapipe/docs/face_detection_desktop.md
  42. +1 −2 mediapipe/docs/hair_segmentation_desktop.md
  43. +2 −4 mediapipe/docs/hand_tracking_desktop.md
  44. BIN mediapipe/docs/images/mobile/aar_location.png
  45. BIN mediapipe/docs/images/mobile/android_studio_opencv_location.png
  46. BIN mediapipe/docs/images/mobile/assets_location.png
  47. +40 −32 mediapipe/docs/install.md
  48. +3 −6 mediapipe/docs/object_detection_desktop.md
  49. +91 −10 mediapipe/docs/youtube_8m.md
  50. +2 −0 mediapipe/examples/desktop/BUILD
  51. +86 −6 mediapipe/examples/desktop/simple_run_graph_main.cc
  52. +11 −0 mediapipe/examples/desktop/youtube8m/BUILD
  53. +96 −8 mediapipe/examples/desktop/youtube8m/README.md
  54. +262 −0 mediapipe/examples/desktop/youtube8m/viewer/server.py
  55. +96 −0 mediapipe/examples/desktop/youtube8m/viewer/static/index.html
  56. +217 −0 mediapipe/examples/desktop/youtube8m/viewer/static/main.js
  57. +13 −0 mediapipe/framework/BUILD
  58. +89 −1 mediapipe/framework/calculator_graph_bounds_test.cc
  59. +61 −0 mediapipe/framework/calculator_graph_side_packet_test.cc
  60. +43 −4 mediapipe/framework/calculator_node.cc
  61. +3 −0 mediapipe/framework/calculator_node.h
  62. +16 −0 mediapipe/framework/collection.h
  63. +8 −6 mediapipe/framework/demangle.h
  64. +2 −3 mediapipe/framework/deps/BUILD
  65. +4 −0 mediapipe/framework/formats/image_format.proto
  66. +6 −0 mediapipe/framework/formats/image_frame.cc
  67. +3 −0 mediapipe/framework/formats/image_frame_opencv.cc
  68. +5 −0 mediapipe/framework/formats/landmark.proto
  69. +7 −0 mediapipe/framework/input_side_packet_handler.cc
  70. +5 −0 mediapipe/framework/input_side_packet_handler.h
  71. +3 −2 mediapipe/framework/output_side_packet_impl.cc
  72. +1 −0 mediapipe/framework/output_side_packet_impl.h
  73. +8 −0 mediapipe/framework/packet.h
  74. +18 −0 mediapipe/framework/port.h
  75. +169 −77 mediapipe/framework/profiler/graph_profiler_test.cc
  76. +14 −3 mediapipe/framework/profiler/trace_buffer.h
  77. +16 −16 mediapipe/framework/profiler/trace_builder.cc
  78. +5 −0 mediapipe/framework/tool/tag_map.h
  79. +4 −0 mediapipe/framework/tool/template_expander.cc
  80. +2 −2 mediapipe/framework/tool/template_parser.cc
  81. +4 −0 mediapipe/gpu/gl_simple_shaders.h
  82. +27 −0 mediapipe/graphs/youtube8m/BUILD
  83. +3,862 −0 mediapipe/graphs/youtube8m/label_map.txt
  84. +178 −0 mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt
  85. +139 −0 mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt
  86. +15 −0 mediapipe/java/com/google/mediapipe/BUILD
  87. +7 −0 mediapipe/java/com/google/mediapipe/components/BUILD
  88. +3 −1 mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
  89. +7 −0 mediapipe/java/com/google/mediapipe/framework/BUILD
  90. +7 −0 mediapipe/java/com/google/mediapipe/glutil/BUILD
  91. +157 −0 mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl
  92. +2 −0 mediapipe/util/sequence/README.md
  93. +3 −1 mediapipe/util/sequence/media_sequence.py
  94. +4 −0 mediapipe/util/sequence/media_sequence_test.py
@@ -37,10 +37,15 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe

## Publications
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)

## Events
[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
* [ML Conference, Berlin 9-11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
* [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA

## Alpha Disclaimer
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
@@ -10,7 +10,8 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1")
versions.check(minimum_bazel_version = "0.24.1",
maximum_bazel_version = "0.29.1")

# ABSL cpp library.
http_archive(
@@ -26,6 +26,13 @@ proto_library(
deps = ["//mediapipe/framework:calculator_proto"],
)

proto_library(
name = "dequantize_byte_array_calculator_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)

proto_library(
name = "packet_cloner_calculator_proto",
srcs = ["packet_cloner_calculator.proto"],
@@ -104,6 +111,14 @@ mediapipe_cc_proto_library(
deps = [":concatenate_vector_calculator_proto"],
)

mediapipe_cc_proto_library(
name = "dequantize_byte_array_calculator_cc_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":dequantize_byte_array_calculator_proto"],
)

mediapipe_cc_proto_library(
name = "quantize_float_vector_calculator_cc_proto",
srcs = ["quantize_float_vector_calculator.proto"],
@@ -387,6 +402,32 @@ cc_library(
alwayslink = 1,
)

cc_library(
name = "string_to_int_calculator",
srcs = ["string_to_int_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)

cc_library(
name = "side_packet_to_stream_calculator",
srcs = ["side_packet_to_stream_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test(
name = "immediate_mux_calculator_test",
srcs = ["immediate_mux_calculator_test.cc"],
@@ -558,6 +599,32 @@ cc_test(
],
)

cc_library(
name = "dequantize_byte_array_calculator",
srcs = ["dequantize_byte_array_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":dequantize_byte_array_calculator_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test(
name = "dequantize_byte_array_calculator_test",
srcs = ["dequantize_byte_array_calculator_test.cc"],
deps = [
":dequantize_byte_array_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
],
)

cc_library(
name = "quantize_float_vector_calculator",
srcs = ["quantize_float_vector_calculator.cc"],
@@ -0,0 +1,90 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cfloat>

#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/status.h"

// Dequantizes a byte array to a vector of floats.
//
// Example config:
// node {
// calculator: "DequantizeByteArrayCalculator"
// input_stream: "ENCODED:encoded"
// output_stream: "FLOAT_VECTOR:float_vector"
// options {
// [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
// max_quantized_value: 2
// min_quantized_value: -2
// }
// }
// }
namespace mediapipe {

class DequantizeByteArrayCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag("ENCODED").Set<std::string>();
cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
return ::mediapipe::OkStatus();
}

::mediapipe::Status Open(CalculatorContext* cc) final {
const auto options =
cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
if (!options.has_max_quantized_value() ||
!options.has_min_quantized_value()) {
return ::mediapipe::InvalidArgumentError(
"Both max_quantized_value and min_quantized_value must be provided "
"in DequantizeByteArrayCalculatorOptions.");
}
float max_quantized_value = options.max_quantized_value();
float min_quantized_value = options.min_quantized_value();
if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
return ::mediapipe::InvalidArgumentError(
"max_quantized_value must be greater than min_quantized_value.");
}
float range = max_quantized_value - min_quantized_value;
scalar_ = range / 255.0;
bias_ = (range / 512.0) + min_quantized_value;
return ::mediapipe::OkStatus();
}

::mediapipe::Status Process(CalculatorContext* cc) final {
const std::string& encoded =
cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
std::vector<float> float_vector;
float_vector.reserve(encoded.length());
for (int i = 0; i < encoded.length(); ++i) {
float_vector.push_back(
static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
}
cc->Outputs()
.Tag("FLOAT_VECTOR")
.AddPacket(MakePacket<std::vector<float>>(float_vector)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}

private:
float scalar_;
float bias_;
};

REGISTER_CALCULATOR(DequantizeByteArrayCalculator);

} // namespace mediapipe
@@ -0,0 +1,28 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";

package mediapipe;

import "mediapipe/framework/calculator.proto";

message DequantizeByteArrayCalculatorOptions {
extend CalculatorOptions {
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
}

optional float max_quantized_value = 1;
optional float min_quantized_value = 2;
}
@@ -0,0 +1,137 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <string>
#include <vector>

#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h" // NOLINT

namespace mediapipe {

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 2
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"Both max_quantized_value and min_quantized_value must be provided"));
}

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: -2
min_quantized_value: 2
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"max_quantized_value must be greater than min_quantized_value"));
}

TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 1
min_quantized_value: 1
}
}
)");
CalculatorRunner runner(node_config);
std::string empty_string;
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(empty_string).At(Timestamp(0)));
auto status = runner.Run();
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
testing::HasSubstr(
"max_quantized_value must be greater than min_quantized_value"));
}

TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
calculator: "DequantizeByteArrayCalculator"
input_stream: "ENCODED:encoded"
output_stream: "FLOAT_VECTOR:float_vector"
options {
[mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
max_quantized_value: 2
min_quantized_value: -2
}
}
)");
CalculatorRunner runner(node_config);
unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
runner.MutableInputs()->Tag("ENCODED").packets.push_back(
MakePacket<std::string>(
std::string(reinterpret_cast<char const*>(input), 4))
.At(Timestamp(0)));
auto status = runner.Run();
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs =
runner.Outputs().Tag("FLOAT_VECTOR").packets;
EXPECT_EQ(1, outputs.size());
const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
ASSERT_FALSE(result.empty());
EXPECT_EQ(4, result.size());
EXPECT_NEAR(0, result[0], 0.01);
EXPECT_NEAR(2, result[1], 0.01);
EXPECT_NEAR(-2, result[2], 0.01);
EXPECT_NEAR(-1.976, result[3], 0.01);

EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
}

} // namespace mediapipe
@@ -102,6 +102,12 @@ class PreviousLoopbackCalculator : public CalculatorBase {
cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
}
}
if (!main_ts_.empty()) {
cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
}
if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
cc->Outputs().Get(loop_out_id_).Close();
}
return ::mediapipe::OkStatus();
}

0 comments on commit 259b48e

Please sign in to comment.
You can’t perform that action at this time.