Skip to content
Permalink
Browse files

Project import generated by Copybara.

GitOrigin-RevId: e3566e5029af25b0fc4b1071a49e49ae20aa5df6
  • Loading branch information
MediaPipe Team jqtang
MediaPipe Team authored and jqtang committed Dec 3, 2019
1 parent 446d7cf commit 137867d0888a7e5a02089829360b3d32857cdad9
Showing with 908 additions and 503 deletions.
  1. +3 −1 Dockerfile
  2. +14 −10 WORKSPACE
  3. +1 −0 mediapipe/calculators/core/BUILD
  4. +4 −10 mediapipe/calculators/core/begin_loop_calculator.cc
  5. +2 −2 mediapipe/calculators/core/concatenate_vector_calculator.cc
  6. +3 −8 mediapipe/calculators/core/end_loop_calculator.cc
  7. +4 −17 mediapipe/calculators/core/previous_loopback_calculator.cc
  8. +15 −65 mediapipe/calculators/core/previous_loopback_calculator_test.cc
  9. +5 −0 mediapipe/calculators/core/split_vector_calculator.cc
  10. +1 −1 mediapipe/calculators/image/BUILD
  11. +1 −1 mediapipe/calculators/image/image_transformation_calculator.cc
  12. +2 −2 mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
  13. +1 −2 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc
  14. +7 −2 mediapipe/calculators/tflite/BUILD
  15. +21 −28 mediapipe/calculators/tflite/tflite_converter_calculator.cc
  16. +58 −48 mediapipe/calculators/tflite/tflite_inference_calculator.cc
  17. +5 −0 mediapipe/calculators/tflite/tflite_inference_calculator.proto
  18. +1 −2 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc
  19. +19 −29 mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
  20. +25 −24 mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
  21. +13 −26 mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc
  22. +1 −2 mediapipe/calculators/util/detection_label_id_to_text_calculator.cc
  23. +3 −3 mediapipe/calculators/util/filter_collection_calculator.cc
  24. +15 −16 mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
  25. +27 −26 mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc
  26. +15 −16 mediapipe/calculators/util/landmark_projection_calculator.cc
  27. +15 −16 mediapipe/calculators/util/landmarks_to_detection_calculator.cc
  28. +10 −9 mediapipe/calculators/util/landmarks_to_floats_calculator.cc
  29. +39 −34 mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
  30. +1 −2 mediapipe/calculators/util/top_k_scores_calculator.cc
  31. +38 −0 mediapipe/calculators/video/opencv_video_decoder_calculator.cc
  32. +46 −1 mediapipe/calculators/video/opencv_video_encoder_calculator.cc
  33. +2 −2 mediapipe/docs/android_archive_library.md
  34. +26 −2 mediapipe/docs/gpu.md
  35. +11 −7 mediapipe/docs/install.md
  36. +14 −3 mediapipe/docs/mediapipe_ios_setup.md
  37. +3 −3 mediapipe/docs/multi_hand_tracking_mobile_gpu.md
  38. +1 −0 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD
  39. +61 −0 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java
  40. +1 −0 mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD
  41. +48 −0 mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java
  42. +5 −0 mediapipe/examples/desktop/demo_run_graph_main.cc
  43. +5 −0 mediapipe/examples/desktop/demo_run_graph_main_gpu.cc
  44. +5 −4 mediapipe/examples/ios/handtrackinggpu/BUILD
  45. +23 −0 mediapipe/examples/ios/handtrackinggpu/ViewController.mm
  46. +5 −4 mediapipe/examples/ios/multihandtrackinggpu/BUILD
  47. +27 −0 mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm
  48. +20 −15 mediapipe/framework/calculator_graph.cc
  49. +18 −4 mediapipe/framework/formats/BUILD
  50. +1 −1 mediapipe/framework/formats/annotation/rasterization.proto
  51. +3 −0 mediapipe/framework/formats/landmark.proto
  52. +8 −0 mediapipe/framework/packet.cc
  53. +52 −0 mediapipe/framework/packet.h
  54. +7 −3 mediapipe/framework/port.h
  55. +2 −0 mediapipe/framework/port/opencv_core_inc.h
  56. +4 −2 mediapipe/framework/scheduler.cc
  57. +4 −4 mediapipe/framework/tool/BUILD
  58. +1 −1 mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt
  59. +1 −1 mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt
  60. +2 −2 mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt
  61. +1 −1 mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt
  62. +1 −1 mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt
  63. +5 −0 mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
  64. +22 −0 mediapipe/java/com/google/mediapipe/framework/PacketGetter.java
  65. +1 −0 mediapipe/java/com/google/mediapipe/framework/jni/BUILD
  66. +1 −15 mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc
  67. +13 −0 mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc
  68. +5 −0 mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h
  69. +33 −0 mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc
  70. +3 −0 mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.h
  71. +33 −18 mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl
  72. +1 −1 mediapipe/util/sequence/media_sequence.py
  73. +0 −6 third_party/com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff
  74. +14 −0 third_party/com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff
@@ -30,6 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
unzip \
python \
python-pip \
python3-pip \
libopencv-core-dev \
libopencv-highgui-dev \
libopencv-imgproc-dev \
@@ -42,9 +43,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \

RUN pip install --upgrade setuptools
RUN pip install future
RUN pip3 install six

# Install bazel
ARG BAZEL_VERSION=0.26.1
ARG BAZEL_VERSION=1.1.0
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
@@ -12,17 +12,21 @@ http_archive(
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1")

# ABSL cpp library.
# ABSL cpp library lts_2019_08_08.
http_archive(
name = "com_google_absl",
# Head commit on 2019-04-12.
# TODO: Switch to the latest absl version when the problem gets
# fixed.
urls = [
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
"https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
],
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "abseil-cpp-20190808",
sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
)

http_archive(
@@ -103,9 +107,9 @@ http_archive(
],
)

# 2019-11-12
_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
# 2019-11-21
_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
http_archive(
name = "org_tensorflow",
urls = [
@@ -691,6 +691,7 @@ cc_library(
":split_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:resource_util",
@@ -21,16 +21,10 @@

namespace mediapipe {

// A calculator to process std::vector<NormalizedLandmark>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
BeginLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);

// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
typedef BeginLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
BeginLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
// A calculator to process std::vector<NormalizedLandmarkList>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
BeginLoopNormalizedLandmarkListVectorCalculator;
REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator);

// A calculator to process std::vector<NormalizedRect>.
typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
@@ -19,7 +19,7 @@
#include "mediapipe/framework/formats/landmark.pb.h"
#include "tensorflow/lite/interpreter.h"

#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
#endif // !MEDIAPIPE_DISABLE_GPU

@@ -50,7 +50,7 @@ typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
ConcatenateLandmarkVectorCalculator;
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);

#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
ConcatenateGlBufferVectorCalculator;
REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
@@ -26,14 +26,9 @@ typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
EndLoopNormalizedRectCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);

typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
EndLoopNormalizedLandmarkCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);

typedef EndLoopCalculator<
std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
EndLoopNormalizedLandmarksVectorCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
EndLoopNormalizedLandmarkListVectorCalculator;
REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);

typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
REGISTER_CALCULATOR(EndLoopBooleanCalculator);
@@ -74,6 +74,10 @@ class PreviousLoopbackCalculator : public CalculatorBase {
}

::mediapipe::Status Process(CalculatorContext* cc) final {
Packet& main_packet = cc->Inputs().Get(main_id_).Value();
if (!main_packet.IsEmpty()) {
main_ts_.push_back(main_packet.Timestamp());
}
Packet& loopback_packet = cc->Inputs().Get(loop_id_).Value();
if (!loopback_packet.IsEmpty()) {
loopback_packets_.push_back(loopback_packet);
@@ -83,23 +87,6 @@ class PreviousLoopbackCalculator : public CalculatorBase {
}
}

Packet& main_packet = cc->Inputs().Get(main_id_).Value();
if (!main_packet.IsEmpty()) {
main_ts_.push_back(main_packet.Timestamp());

// In case of an empty "LOOP" input, truncate timestamp is set to the
// lowest possible timestamp for a successive non-empty "LOOP" input. This
// truncates main_ts_ as soon as possible, and produces the highest legal
// output timestamp bound.
if (loopback_packet.IsEmpty() &&
loopback_packet.Timestamp() != Timestamp::Unstarted()) {
while (!main_ts_.empty() &&
main_ts_.front() <= loopback_packet.Timestamp() + 1) {
main_ts_.pop_front();
}
}
}

while (!main_ts_.empty() && !loopback_packets_.empty()) {
Timestamp main_timestamp = main_ts_.front();
main_ts_.pop_front();
@@ -93,14 +93,19 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));

send_packet("in", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(2, 1));

send_packet("in", 5);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 2));

send_packet("in", 15);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5, 15}));
EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));

MP_EXPECT_OK(graph_.CloseAllInputStreams());
@@ -182,78 +187,23 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));

send_packet("in", 2);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2}));

send_packet("in", 5);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5}));
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5}));

send_packet("in", 15);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5, 15}));
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5, 15}));

MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs),
(std::vector<int64>{1, 5, 15, Timestamp::Max().Value()}));

MP_EXPECT_OK(graph_.WaitUntilDone());
}
(std::vector<int64>{1, 2, 5, 15, Timestamp::Max().Value()}));

// Demonstrates that downstream calculators won't be blocked by
// always-empty-LOOP-stream.
TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
std::vector<Packet> outputs;
CalculatorGraphConfig graph_config_ =
ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_stream: 'in'
node {
calculator: 'PreviousLoopbackCalculator'
input_stream: 'MAIN:in'
input_stream: 'LOOP:previous'
input_stream_info: { tag_index: 'LOOP' back_edge: true }
output_stream: 'PREV_LOOP:previous'
}
# This calculator synchronizes its inputs as normal, so it is used
# to check that both "in" and "previous" are ready.
node {
calculator: 'PassThroughCalculator'
input_stream: 'in'
input_stream: 'previous'
output_stream: 'out'
output_stream: 'previous2'
}
node {
calculator: 'PacketOnCloseCalculator'
input_stream: 'out'
output_stream: 'close_out'
}
)");
tool::AddVectorSink("close_out", &graph_config_, &outputs);

CalculatorGraph graph_;
MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
MP_ASSERT_OK(graph_.StartRun({}));

auto send_packet = [&graph_](const std::string& input_name, int n) {
MP_EXPECT_OK(graph_.AddPacketToInputStream(
input_name, MakePacket<int>(n).At(Timestamp(n))));
};

send_packet("in", 0);
MP_EXPECT_OK(graph_.WaitUntilIdle());
EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));

for (int main_ts = 1; main_ts < 50; ++main_ts) {
send_packet("in", main_ts);
MP_EXPECT_OK(graph_.WaitUntilIdle());
std::vector<int64> ts_values = TimestampValues(outputs);
EXPECT_EQ(ts_values.size(), main_ts);
for (int j = 0; j < main_ts; ++j) {
CHECK_EQ(ts_values[j], j);
}
}

MP_EXPECT_OK(graph_.CloseAllInputStreams());
MP_EXPECT_OK(graph_.WaitUntilIdle());
MP_EXPECT_OK(graph_.WaitUntilDone());
}

@@ -17,6 +17,7 @@
#include <vector>

#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "tensorflow/lite/interpreter.h"

namespace mediapipe {
@@ -41,4 +42,8 @@ REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
SplitLandmarkVectorCalculator;
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);

typedef SplitVectorCalculator<::mediapipe::NormalizedRect>
SplitNormalizedRectVectorCalculator;
REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
} // namespace mediapipe
@@ -356,13 +356,13 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gpu_buffer",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:shader_util",
],
}),
@@ -400,7 +400,7 @@ ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
QuadRenderer* renderer = nullptr;
GlTexture src1;

#if defined(__APPLE__) && !TARGET_OS_OSX
#if defined(MEDIAPIPE_IOS)
if (input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange ||
input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) {
if (!yuv_renderer_) {
@@ -34,7 +34,7 @@
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_util.h"

#if !defined(__ANDROID__) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
#include "tensorflow/core/profiler/lib/traceme.h"
#endif

@@ -441,7 +441,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
tf::Status tf_status;
{
#if !defined(__ANDROID__) && !defined(__APPLE__)
#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
#endif
tf_status = session_->Run(input_tensors, output_tensor_names,
@@ -31,8 +31,7 @@
#include "mediapipe/framework/tool/status_util.h"
#include "tensorflow/core/public/session_options.h"

#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
defined(__APPLE__) && !TARGET_OS_OSX
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/helpers.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
@@ -13,12 +13,12 @@
# limitations under the License.
#

load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")

licenses(["notice"]) # Apache 2.0

package(default_visibility = ["//visibility:private"])

load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")

proto_library(
name = "ssd_anchors_calculator_proto",
srcs = ["ssd_anchors_calculator.proto"],
@@ -249,6 +249,11 @@ cc_library(
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
],
}) + select({
"//conditions:default": [],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate",
],
}),
alwayslink = 1,
)

0 comments on commit 137867d

Please sign in to comment.
You can’t perform that action at this time.