Skip to content
Permalink
Browse files

Project import generated by Copybara.

GitOrigin-RevId: 852dfb05d450167899c0dd5ef7c45622a12e865b
  • Loading branch information
MediaPipe Team Hadon Nash
MediaPipe Team authored and Hadon Nash committed Feb 10, 2020
1 parent d144e56 commit de4fbc10e62001795741cb6d0200a17a94a29b0f
Showing with 1,664 additions and 628 deletions.
  1. +15 −8 README.md
  2. +8 −0 WORKSPACE
  3. +44 −0 mediapipe/calculators/core/BUILD
  4. +116 −0 mediapipe/calculators/core/constant_side_packet_calculator.cc
  5. +36 −0 mediapipe/calculators/core/constant_side_packet_calculator.proto
  6. +196 −0 mediapipe/calculators/core/constant_side_packet_calculator_test.cc
  7. +29 −0 mediapipe/calculators/core/packet_resampler_calculator.cc
  8. +37 −0 mediapipe/calculators/core/packet_resampler_calculator.h
  9. +1 −0 mediapipe/calculators/core/packet_resampler_calculator.proto
  10. +17 −0 mediapipe/calculators/image/BUILD
  11. +89 −135 mediapipe/calculators/image/image_cropping_calculator.cc
  12. +86 −0 mediapipe/calculators/image/image_cropping_calculator.h
  13. +10 −0 mediapipe/calculators/image/image_cropping_calculator.proto
  14. +216 −0 mediapipe/calculators/image/image_cropping_calculator_test.cc
  15. +1 −0 mediapipe/calculators/image/testdata/BUILD
  16. BIN mediapipe/calculators/image/testdata/front_camera_pixel2.jpg
  17. +1 −0 mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc
  18. +2 −1 mediapipe/calculators/video/tvl1_optical_flow_calculator.cc
  19. +17 −1 mediapipe/docs/examples.md
  20. +5 −2 mediapipe/docs/face_detection_desktop.md
  21. +5 −2 mediapipe/docs/hair_segmentation_desktop.md
  22. +5 −2 mediapipe/docs/hand_tracking_desktop.md
  23. BIN mediapipe/docs/images/faviconv2.ico
  24. BIN mediapipe/docs/images/iconv2.png
  25. BIN mediapipe/docs/images/iris_tracking_desktop.png
  26. BIN mediapipe/docs/images/logov2.png
  27. +4 −2 mediapipe/docs/multi_hand_tracking_desktop.md
  28. +2 −3 mediapipe/docs/web.md
  29. +2 −2 mediapipe/examples/coral/BUILD
  30. +17 −17 mediapipe/examples/coral/README.md
  31. +5 −5 mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt
  32. +5 −5 mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
  33. +1 −1 mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py
  34. +2 −0 mediapipe/framework/BUILD
  35. +7 −7 mediapipe/framework/calculator_context_manager.h
  36. +7 −1 mediapipe/framework/calculator_graph.cc
  37. +9 −8 mediapipe/framework/calculator_graph.h
  38. +1 −1 mediapipe/framework/calculator_graph_event_loop_test.cc
  39. +2 −1 mediapipe/framework/calculator_node.cc
  40. +26 −25 mediapipe/framework/calculator_node.h
  41. +1 −1 mediapipe/framework/calculator_parallel_execution_test.cc
  42. +9 −9 mediapipe/framework/counter_factory.cc
  43. +4 −3 mediapipe/framework/counter_factory.h
  44. +1 −0 mediapipe/framework/deps/BUILD
  45. +4 −4 mediapipe/framework/deps/monotonic_clock.cc
  46. +1 −1 mediapipe/framework/deps/monotonic_clock_test.cc
  47. +4 −1 mediapipe/framework/deps/random_base.h
  48. +7 −7 mediapipe/framework/deps/registration.h
  49. +2 −2 mediapipe/framework/deps/threadpool.h
  50. +2 −0 mediapipe/framework/encode_binary_proto.bzl
  51. +2 −2 mediapipe/framework/graph_output_stream.h
  52. +4 −1 mediapipe/framework/graph_validation_test.cc
  53. +1 −1 mediapipe/framework/input_stream_manager.cc
  54. +22 −22 mediapipe/framework/input_stream_manager.h
  55. +10 −7 mediapipe/framework/output_stream_handler.h
  56. +2 −2 mediapipe/framework/output_stream_manager.h
  57. +8 −8 mediapipe/framework/packet_generator_graph.cc
  58. +1 −0 mediapipe/framework/profiler/BUILD
  59. +2 −1 mediapipe/framework/profiler/graph_profiler.cc
  60. +10 −10 mediapipe/framework/profiler/graph_profiler.h
  61. +28 −0 mediapipe/framework/profiler/graph_tracer_test.cc
  62. +5 −5 mediapipe/framework/profiler/sharded_map.h
  63. +3 −3 mediapipe/framework/scheduler.cc
  64. +32 −28 mediapipe/framework/scheduler.h
  65. +13 −13 mediapipe/framework/scheduler_queue.h
  66. +6 −5 mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc
  67. +5 −5 mediapipe/framework/stream_handler/fixed_size_input_stream_handler_test.cc
  68. +3 −3 mediapipe/framework/stream_handler/in_order_output_stream_handler.h
  69. +4 −4 mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc
  70. +1 −1 mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.cc
  71. +1 −0 mediapipe/framework/tool/BUILD
  72. +25 −0 mediapipe/framework/tool/name_util.cc
  73. +46 −0 mediapipe/framework/tool/name_util.h
  74. +6 −0 mediapipe/framework/tool/simulation_clock.cc
  75. +6 −6 mediapipe/framework/tool/simulation_clock.h
  76. +54 −0 mediapipe/framework/tool/simulation_clock_test.cc
  77. +20 −17 mediapipe/framework/tool/subgraph_expansion.cc
  78. +25 −23 mediapipe/framework/tool/subgraph_expansion_test.cc
  79. +1 −25 mediapipe/framework/validated_graph_config.cc
  80. +0 −42 mediapipe/framework/validated_graph_config.h
  81. +1 −0 mediapipe/gpu/BUILD
  82. +2 −2 mediapipe/gpu/egl_surface_holder.h
  83. +1 −1 mediapipe/gpu/gl_context.cc
  84. +1 −1 mediapipe/gpu/gl_context.h
  85. +3 −3 mediapipe/gpu/gl_context_internal.h
  86. +4 −3 mediapipe/gpu/gl_texture_buffer_pool.h
  87. +1 −1 mediapipe/gpu/gl_thread_collector.h
  88. +1 −1 mediapipe/gpu/gpu_buffer_multi_pool.h
  89. +1 −1 mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt
  90. +2 −1 mediapipe/java/com/google/mediapipe/components/AudioDataConsumer.java
  91. +89 −19 mediapipe/java/com/google/mediapipe/components/CameraXPreviewHelper.java
  92. +23 −7 mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
  93. +68 −50 mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java
  94. +1 −1 mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc
  95. +3 −0 mediapipe/objc/MPPCameraInputSource.h
  96. +22 −28 mediapipe/objc/MPPCameraInputSource.m
  97. +4 −0 mediapipe/objc/util.cc
  98. +18 −17 mediapipe/util/tracking/box_tracker.h
  99. +1 −1 mediapipe/util/tracking/parallel_invoker.h
  100. +11 −0 third_party/easyexif.BUILD
@@ -1,25 +1,25 @@
![MediaPipe](mediapipe/docs/images/mediapipe_small.png?raw=true "MediaPipe logo")
=======================================================================

[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data), cross platform (i.e Android, iOS, web, edge devices) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.

![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)

> "<em>MediaPipe has made it extremely easy to build our 3D person pose reconstruction demo app, facilitating accelerated neural network inference on device and synchronization of our result visualization with the video capture stream. Highly recommended!</em>" - George Papandreou, CTO, [Ariel AI](https://arielai.com)
## ML Solutions in MediaPipe

* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
* [AutoFlip](mediapipe/docs/autoflip.md)

![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)

@@ -29,6 +29,8 @@ Follow these [instructions](mediapipe/docs/install.md).
## Getting started
See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).

Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)

## Documentation
[MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)

@@ -37,10 +39,12 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
## Visualizing MediaPipe graphs
A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).

## Community forum
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
## Videos
* [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)

## Publications
* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)

@@ -55,6 +59,9 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA

## Community forum
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe

## Alpha Disclaimer
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.

@@ -78,6 +78,14 @@ http_archive(
],
)

# easyexif
http_archive(
name = "easyexif",
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
strip_prefix = "easyexif-master",
build_file = "@//third_party:easyexif.BUILD",
)

# libyuv
http_archive(
name = "libyuv",
@@ -86,6 +86,15 @@ proto_library(
],
)

proto_library(
name = "constant_side_packet_calculator_proto",
srcs = ["constant_side_packet_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)

proto_library(
name = "clip_vector_size_calculator_proto",
srcs = ["clip_vector_size_calculator.proto"],
@@ -173,6 +182,14 @@ mediapipe_cc_proto_library(
deps = [":gate_calculator_proto"],
)

mediapipe_cc_proto_library(
name = "constant_side_packet_calculator_cc_proto",
srcs = ["constant_side_packet_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":constant_side_packet_calculator_proto"],
)

cc_library(
name = "add_header_calculator",
srcs = ["add_header_calculator.cc"],
@@ -960,3 +977,30 @@ cc_test(
"@com_google_absl//absl/memory",
],
)

cc_library(
name = "constant_side_packet_calculator",
srcs = ["constant_side_packet_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":constant_side_packet_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test(
name = "constant_side_packet_calculator_test",
srcs = ["constant_side_packet_calculator_test.cc"],
deps = [
":constant_side_packet_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
)
@@ -0,0 +1,116 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <string>

#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"

namespace mediapipe {

// Generates an output side packet or multiple output side packets according to
// the specified options.
//
// Example configs:
// node {
// calculator: "ConstantSidePacketCalculator"
// output_side_packet: "PACKET:packet"
// options: {
// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
// packet { int_value: 2 }
// }
// }
// }
//
// node {
// calculator: "ConstantSidePacketCalculator"
// output_side_packet: "PACKET:0:int_packet"
// output_side_packet: "PACKET:1:bool_packet"
// options: {
// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
// packet { int_value: 2 }
// packet { bool_value: true }
// }
// }
// }
class ConstantSidePacketCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
const auto& options = cc->Options().GetExtension(
::mediapipe::ConstantSidePacketCalculatorOptions::ext);
RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
options.packet_size())
<< "Number of output side packets has to be same as number of packets "
"configured in options.";

int index = 0;
for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
const auto& packet_options = options.packet(index);
auto& packet = cc->OutputSidePackets().Get(id);
if (packet_options.has_int_value()) {
packet.Set<int>();
} else if (packet_options.has_float_value()) {
packet.Set<float>();
} else if (packet_options.has_bool_value()) {
packet.Set<bool>();
} else if (packet_options.has_string_value()) {
packet.Set<std::string>();
} else {
return ::mediapipe::InvalidArgumentError(
"None of supported values were specified in options.");
}
}
return ::mediapipe::OkStatus();
}

::mediapipe::Status Open(CalculatorContext* cc) override {
const auto& options = cc->Options().GetExtension(
::mediapipe::ConstantSidePacketCalculatorOptions::ext);
int index = 0;
for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
auto& packet = cc->OutputSidePackets().Get(id);
const auto& packet_options = options.packet(index);
if (packet_options.has_int_value()) {
packet.Set(MakePacket<int>(packet_options.int_value()));
} else if (packet_options.has_float_value()) {
packet.Set(MakePacket<float>(packet_options.float_value()));
} else if (packet_options.has_bool_value()) {
packet.Set(MakePacket<bool>(packet_options.bool_value()));
} else if (packet_options.has_string_value()) {
packet.Set(MakePacket<std::string>(packet_options.string_value()));
} else {
return ::mediapipe::InvalidArgumentError(
"None of supported values were specified in options.");
}
}
return ::mediapipe::OkStatus();
}

::mediapipe::Status Process(CalculatorContext* cc) override {
return ::mediapipe::OkStatus();
}

private:
static constexpr const char* kPacketTag = "PACKET";
};

REGISTER_CALCULATOR(ConstantSidePacketCalculator);

} // namespace mediapipe
@@ -0,0 +1,36 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";

package mediapipe;

import "mediapipe/framework/calculator.proto";

message ConstantSidePacketCalculatorOptions {
extend CalculatorOptions {
optional ConstantSidePacketCalculatorOptions ext = 291214597;
}

message ConstantSidePacket {
oneof value {
int32 int_value = 1;
float float_value = 2;
bool bool_value = 3;
string string_value = 4;
}
}

repeated ConstantSidePacket packet = 1;
}

0 comments on commit de4fbc1

Please sign in to comment.
You can’t perform that action at this time.