Skip to content
Permalink
Browse files

Project import generated by Copybara.

GitOrigin-RevId: 0517756260533d374df93679965ca662d0ec6943
  • Loading branch information
MediaPipe Team Hadon Nash
MediaPipe Team authored and Hadon Nash committed Jan 10, 2020
1 parent 38ee260 commit ae6be10afe59a6a99d8a68007784706ac98720dd
Showing with 16,410 additions and 592 deletions.
  1. +1 −1 .bazelrc
  2. +1 −0 README.md
  3. +3 −1 WORKSPACE
  4. +57 −5 mediapipe/calculators/core/BUILD
  5. +21 −131 mediapipe/calculators/core/packet_resampler_calculator.cc
  6. +168 −0 mediapipe/calculators/core/packet_resampler_calculator.h
  7. +2 −1 mediapipe/calculators/core/packet_resampler_calculator_test.cc
  8. +304 −0 mediapipe/calculators/core/packet_thinner_calculator.cc
  9. +66 −0 mediapipe/calculators/core/packet_thinner_calculator.proto
  10. +357 −0 mediapipe/calculators/core/packet_thinner_calculator_test.cc
  11. +21 −6 mediapipe/calculators/core/previous_loopback_calculator.cc
  12. +59 −0 mediapipe/calculators/core/previous_loopback_calculator_test.cc
  13. +14 −3 mediapipe/calculators/core/split_vector_calculator.cc
  14. +112 −20 mediapipe/calculators/core/split_vector_calculator.h
  15. +239 −0 mediapipe/calculators/core/split_vector_calculator_test.cc
  16. +3 −1 mediapipe/calculators/image/BUILD
  17. +10 −3 mediapipe/calculators/image/scale_image_calculator.cc
  18. +1 −1 mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
  19. +1 −1 mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
  20. +2 −2 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc
  21. +2 −2 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc
  22. +1 −1 mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc
  23. +1 −1 mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc
  24. +2 −2 mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc
  25. +29 −30 mediapipe/calculators/tflite/tflite_converter_calculator.cc
  26. +111 −56 mediapipe/calculators/tflite/tflite_inference_calculator.cc
  27. +26 −37 mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
  28. +46 −0 mediapipe/calculators/video/BUILD
  29. +19 −7 mediapipe/calculators/video/opencv_video_decoder_calculator.cc
  30. +12 −4 mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc
  31. +14 −8 mediapipe/calculators/video/opencv_video_encoder_calculator.cc
  32. +2 −2 mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc
  33. +142 −0 mediapipe/calculators/video/video_pre_stream_calculator.cc
  34. +43 −0 mediapipe/calculators/video/video_pre_stream_calculator.proto
  35. +186 −0 mediapipe/calculators/video/video_pre_stream_calculator_test.cc
  36. +321 −0 mediapipe/docs/autoflip.md
  37. +0 −1 mediapipe/docs/calculator.md
  38. +7 −2 mediapipe/docs/examples.md
  39. +10 −7 mediapipe/docs/face_detection_coral_devboard.md
  40. +2 −2 mediapipe/docs/framework_concepts.md
  41. BIN mediapipe/docs/images/autoflip_edited_example.gif
  42. BIN mediapipe/docs/images/autoflip_graph.png
  43. BIN mediapipe/docs/images/autoflip_is_required.gif
  44. BIN mediapipe/docs/images/face_detection_demo_coral.gif
  45. BIN mediapipe/docs/images/face_detection_demo_coral.jpg
  46. BIN mediapipe/docs/images/object_detection_demo_coral.jpg
  47. +11 −14 mediapipe/docs/install.md
  48. +8 −5 mediapipe/docs/object_detection_coral_devboard.md
  49. +19 −3 mediapipe/docs/object_tracking_mobile_gpu.md
  50. +4 −2 mediapipe/docs/youtube_8m.md
  51. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD
  52. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD
  53. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD
  54. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD
  55. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD
  56. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD
  57. +0 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD
  58. +1 −1 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD
  59. +31 −0 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java
  60. +6 −2 mediapipe/examples/coral/README.md
  61. +6 −3 mediapipe/examples/coral/demo_run_graph_main.cc
  62. +56 −0 mediapipe/examples/desktop/autoflip/BUILD
  63. +25 −0 mediapipe/examples/desktop/autoflip/README.md
  64. +202 −0 mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt
  65. +252 −0 mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt
  66. +153 −0 mediapipe/examples/desktop/autoflip/autoflip_messages.proto
  67. +426 −0 mediapipe/examples/desktop/autoflip/calculators/BUILD
  68. +302 −0 mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc
  69. +44 −0 mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto
  70. +397 −0 mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc
  71. +269 −0 mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc
  72. +50 −0 mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto
  73. +247 −0 mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc
  74. +126 −0 mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc
  75. +33 −0 mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto
  76. +164 −0 mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc
  77. +589 −0 mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc
  78. +249 −0 mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h
  79. +101 −0 mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto
  80. +621 −0 mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc
  81. +190 −0 mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc
  82. +48 −0 mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto
  83. +163 −0 mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc
  84. +231 −0 mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc
  85. +54 −0 mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto
  86. +438 −0 mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc
  87. +23 −0 mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD
  88. BIN mediapipe/examples/desktop/autoflip/calculators/testdata/dino.jpg
  89. +121 −0 mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc
  90. +56 −0 mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto
  91. +177 −0 mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc
  92. +331 −0 mediapipe/examples/desktop/autoflip/quality/BUILD
  93. +217 −0 mediapipe/examples/desktop/autoflip/quality/cropping.proto
  94. +79 −0 mediapipe/examples/desktop/autoflip/quality/focus_point.proto
  95. +259 −0 mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc
  96. +110 −0 mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h
  97. +579 −0 mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc
  98. +40 −0 mediapipe/examples/desktop/autoflip/quality/math_utils.h
  99. +177 −0 mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc
  100. +70 −0 mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h
  101. +187 −0 mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc
  102. +69 −0 mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc
  103. +82 −0 mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h
  104. +73 −0 mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc
  105. +160 −0 mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc
  106. +69 −0 mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h
  107. +246 −0 mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc
  108. +428 −0 mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc
  109. +142 −0 mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h
  110. +814 −0 mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc
  111. +84 −0 mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc
  112. +65 −0 mediapipe/examples/desktop/autoflip/quality/scene_cropper.h
  113. +164 −0 mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc
  114. +186 −0 mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc
  115. +61 −0 mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h
  116. +19 −0 mediapipe/examples/desktop/autoflip/quality/testdata/BUILD
  117. +30 −0 mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv
  118. BIN mediapipe/examples/desktop/autoflip/quality/testdata/google.jpg
  119. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3.jpg
  120. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3_solid_background.jpg
  121. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6.jpg
  122. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6_solid_background.jpg
  123. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6.jpg
  124. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6_solid_background.jpg
  125. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_1.jpg
  126. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_1_solid_background.jpg
  127. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5.jpg
  128. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5_solid_background.jpg
  129. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4.jpg
  130. BIN mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4_solid_background.jpg
  131. +451 −0 mediapipe/examples/desktop/autoflip/quality/utils.cc
  132. +119 −0 mediapipe/examples/desktop/autoflip/quality/utils.h
  133. +1,072 −0 mediapipe/examples/desktop/autoflip/quality/utils_test.cc
  134. +182 −0 mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc
  135. +47 −0 mediapipe/examples/desktop/autoflip/quality/visual_scorer.h
  136. +28 −0 mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto
  137. +80 −0 mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc
  138. +39 −0 mediapipe/examples/desktop/autoflip/subgraph/BUILD
  139. +126 −0 mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt
  140. +136 −0 mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt
  141. +14 −13 mediapipe/examples/desktop/demo_run_graph_main.cc
  142. +15 −14 mediapipe/examples/desktop/demo_run_graph_main_gpu.cc
  143. +23 −10 mediapipe/examples/desktop/media_sequence/kinetics_dataset.py
  144. +3 −1 mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc
  145. +25 −0 mediapipe/examples/desktop/object_tracking/BUILD
  146. +3 −1 mediapipe/examples/desktop/simple_run_graph_main.cc
  147. +3 −1 mediapipe/examples/desktop/youtube8m/README.md
  148. +3 −1 mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
  149. +1 −1 mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py
  150. +1 −1 mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py
  151. +5 −3 mediapipe/framework/deps/file_helpers.cc
  152. +1 −0 mediapipe/framework/deps/random_base.h
  153. +7 −0 mediapipe/framework/formats/BUILD
  154. +3 −0 mediapipe/framework/formats/detection.proto
  155. +77 −0 mediapipe/framework/graph_validation_test.cc
  156. +6 −4 mediapipe/framework/port/BUILD
  157. +12 −4 mediapipe/framework/tool/subgraph_expansion.cc
  158. +1 −0 mediapipe/gpu/BUILD
  159. +6 −0 mediapipe/gpu/MPPMetalUtil.h
  160. +37 −1 mediapipe/gpu/MPPMetalUtil.mm
  161. +46 −2 mediapipe/gpu/gl_context_webgl.cc
  162. +11 −1 mediapipe/graphs/tracking/BUILD
  163. +45 −0 mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt
  164. +4 −4 mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt
  165. +62 −9 mediapipe/graphs/tracking/subgraphs/BUILD
  166. +119 −0 mediapipe/graphs/tracking/subgraphs/box_tracking_cpu.pbtxt
  167. +1 −1 mediapipe/graphs/tracking/subgraphs/{box_tracking.pbtxt → box_tracking_gpu.pbtxt}
  168. +128 −0 mediapipe/graphs/tracking/subgraphs/object_detection_cpu.pbtxt
  169. +1 −1 mediapipe/graphs/tracking/subgraphs/object_detection_gpu.pbtxt
  170. +3 −3 mediapipe/graphs/tracking/subgraphs/{object_tracking.pbtxt → object_tracking_cpu.pbtxt}
  171. +56 −0 mediapipe/graphs/tracking/subgraphs/object_tracking_gpu.pbtxt
  172. +29 −0 mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt
  173. +1 −1 mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt
  174. +24 −0 mediapipe/java/com/google/mediapipe/components/AudioDataConsumer.java
  175. +21 −0 mediapipe/java/com/google/mediapipe/components/AudioDataProcessor.java
  176. +21 −0 mediapipe/java/com/google/mediapipe/components/AudioDataProducer.java
  177. +3 −0 mediapipe/java/com/google/mediapipe/components/BUILD
  178. +9 −2 mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
  179. +121 −12 mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
  180. +24 −29 mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java
  181. +0 −24 mediapipe/java/com/google/mediapipe/framework/Graph.java
  182. +48 −5 mediapipe/java/com/google/mediapipe/framework/PacketCreator.java
  183. +0 −20 mediapipe/java/com/google/mediapipe/framework/jni/graph.cc
  184. +0 −4 mediapipe/java/com/google/mediapipe/framework/jni/graph.h
  185. +0 −19 mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc
  186. +0 −4 mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.h
  187. +25 −14 mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc
  188. +5 −1 mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h
  189. +22 −3 mediapipe/objc/MPPCameraInputSource.m
  190. +1 −1 mediapipe/util/sequence/media_sequence_test.py
  191. +1 −1 mediapipe/util/sequence/media_sequence_util.py
  192. +1 −1 mediapipe/util/sequence/media_sequence_util_test.py
@@ -12,7 +12,7 @@ build --copt='-Wno-comment'
build --copt='-Wno-return-type'
build --copt='-Wno-unused-local-typedefs'
build --copt='-Wno-ignored-attributes'
# Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
build --incompatible_disable_deprecated_attr_params=false
build --incompatible_depset_is_not_iterable=false

@@ -15,6 +15,7 @@
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
* [AutoFlip](mediapipe/docs/autoflip.md)

![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
@@ -10,7 +10,9 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1")
versions.check(minimum_bazel_version = "0.24.1",
maximum_bazel_version = "1.2.1")


# ABSL cpp library lts_2019_08_08.
http_archive(
@@ -47,6 +47,13 @@ proto_library(
deps = ["//mediapipe/framework:calculator_proto"],
)

proto_library(
name = "packet_thinner_calculator_proto",
srcs = ["packet_thinner_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)

proto_library(
name = "split_vector_calculator_proto",
srcs = ["split_vector_calculator.proto"],
@@ -102,6 +109,14 @@ mediapipe_cc_proto_library(
deps = [":packet_resampler_calculator_proto"],
)

mediapipe_cc_proto_library(
name = "packet_thinner_calculator_cc_proto",
srcs = ["packet_thinner_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_thinner_calculator_proto"],
)

mediapipe_cc_proto_library(
name = "split_vector_calculator_cc_proto",
srcs = ["split_vector_calculator.proto"],
@@ -284,7 +299,6 @@ cc_test(
srcs = ["concatenate_vector_calculator_test.cc"],
deps = [
":concatenate_vector_calculator",
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:timestamp",
@@ -451,6 +465,37 @@ cc_test(
],
)

cc_library(
name = "packet_thinner_calculator",
srcs = ["packet_thinner_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

cc_test(
name = "packet_thinner_calculator_test",
srcs = ["packet_thinner_calculator_test.cc"],
deps = [
":packet_thinner_calculator",
"//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"@com_google_absl//absl/strings",
],
)

cc_library(
name = "pass_through_calculator",
srcs = ["pass_through_calculator.cc"],
@@ -572,6 +617,7 @@ cc_test(
cc_library(
name = "packet_resampler_calculator",
srcs = ["packet_resampler_calculator.cc"],
hdrs = ["packet_resampler_calculator.h"],
visibility = [
"//visibility:public",
],
@@ -595,17 +641,17 @@ cc_library(
cc_test(
name = "packet_resampler_calculator_test",
timeout = "short",
srcs = ["packet_resampler_calculator_test.cc"],
srcs = [
"packet_resampler_calculator_test.cc",
],
deps = [
":packet_resampler_calculator",
"//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/strings",
],
)
@@ -698,7 +744,13 @@ cc_library(
"//mediapipe/util:resource_util",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//mediapipe:ios": [],
"//conditions:default": [
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
],
}),
alwayslink = 1,
)

@@ -12,23 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <cstdlib>
#include "mediapipe/calculators/core/packet_resampler_calculator.h"

#include <memory>
#include <string>

#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/deps/mathutil.h"
#include "mediapipe/framework/deps/random_base.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/tool/options_util.h"

namespace {

@@ -45,120 +31,7 @@ std::unique_ptr<RandomBase> CreateSecureRandom(const std::string& seed) {

namespace mediapipe {

// This calculator is used to normalize the frequency of the packets
// out of a stream. Given a desired frame rate, packets are going to be
// removed or added to achieve it.
//
// The jitter feature is disabled by default. To enable it, you need to
// implement CreateSecureRandom(const std::string&).
//
// The data stream may be either specified as the only stream (by index)
// or as the stream with tag "DATA".
//
// The input and output streams may be accompanied by a VIDEO_HEADER
// stream. This stream includes a VideoHeader at Timestamp::PreStream().
// The input VideoHeader on the VIDEO_HEADER stream will always be updated
// with the resampler frame rate no matter what the options value for
// output_header is before being output on the output VIDEO_HEADER stream.
// If the input VideoHeader is not available, then only the frame rate
// value will be set in the output.
//
// Related:
// packet_downsampler_calculator.cc: skips packets regardless of timestamps.
class PacketResamplerCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);

::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;

private:
// Calculates the first sampled timestamp that incorporates a jittering
// offset.
void InitializeNextOutputTimestampWithJitter();
// Calculates the next sampled timestamp that incorporates a jittering offset.
void UpdateNextOutputTimestampWithJitter();

// Logic for Process() when jitter_ != 0.0.
::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);

// Logic for Process() when jitter_ == 0.0.
::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);

// Given the current count of periods that have passed, this returns
// the next valid timestamp of the middle point of the next period:
// if count is 0, it returns the first_timestamp_.
// if count is 1, it returns the first_timestamp_ + period (corresponding
// to the first tick using exact fps)
// e.g. for frame_rate=30 and first_timestamp_=0:
// 0: 0
// 1: 33333
// 2: 66667
// 3: 100000
//
// Can only be used if jitter_ equals zero.
Timestamp PeriodIndexToTimestamp(int64 index) const;

// Given a Timestamp, finds the closest sync Timestamp based on
// first_timestamp_ and the desired fps.
//
// Can only be used if jitter_ equals zero.
int64 TimestampToPeriodIndex(Timestamp timestamp) const;

// Outputs a packet if it is in range (start_time_, end_time_).
void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;

// The timestamp of the first packet received.
Timestamp first_timestamp_;

// Number of frames per second (desired output frequency).
double frame_rate_;

// Inverse of frame_rate_.
int64 frame_time_usec_;

// Number of periods that have passed (= #packets sent to the output).
//
// Can only be used if jitter_ equals zero.
int64 period_count_;

// The last packet that was received.
Packet last_packet_;

VideoHeader video_header_;
// The "DATA" input stream.
CollectionItemId input_data_id_;
// The "DATA" output stream.
CollectionItemId output_data_id_;

// Indicator whether to flush last packet even if its timestamp is greater
// than the final stream timestamp. Set to false when jitter_ is non-zero.
bool flush_last_packet_;

// Jitter-related variables.
std::unique_ptr<RandomBase> random_;
double jitter_ = 0.0;
Timestamp next_output_timestamp_;

// If specified, output timestamps are aligned with base_timestamp.
// Otherwise, they are aligned with the first input timestamp.
Timestamp base_timestamp_;

// If specified, only outputs at/after start_time are included.
Timestamp start_time_;

// If specified, only outputs before end_time are included.
Timestamp end_time_;

// If set, the output timestamps nearest to start_time and end_time
// are included in the output, even if the nearest timestamp is not
// between start_time and end_time.
bool round_limits_;
};

REGISTER_CALCULATOR(PacketResamplerCalculator);

namespace {
// Returns a TimestampDiff (assuming microseconds) corresponding to the
// given time in seconds.
@@ -279,7 +152,10 @@ ::mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
"SecureRandom is not available. With \"jitter\" specified, "
"PacketResamplerCalculator processing cannot proceed.");
}
packet_reservoir_random_ = CreateSecureRandom(seed);
}
packet_reservoir_ =
std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
return ::mediapipe::OkStatus();
}

@@ -294,6 +170,14 @@ ::mediapipe::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
}
}
if (jitter_ != 0.0 && random_ != nullptr) {
// Packet reservior is used to make sure there's an output for every period,
// e.g. partial period at the end of the stream.
if (packet_reservoir_->IsEnabled() &&
(first_timestamp_ == Timestamp::Unset() ||
(cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
packet_reservoir_->AddSample(curr_packet);
}
MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
} else {
MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
@@ -303,11 +187,14 @@ ::mediapipe::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
}

void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
next_output_timestamp_min_ = first_timestamp_;
next_output_timestamp_ =
first_timestamp_ + frame_time_usec_ * random_->RandFloat();
}

void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
packet_reservoir_->Clear();
packet_reservoir_->Disable();
next_output_timestamp_ +=
frame_time_usec_ *
((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
@@ -339,10 +226,10 @@ ::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
while (true) {
const int64 last_diff =
(next_output_timestamp_ - last_packet_.Timestamp()).Value();
RET_CHECK_GT(last_diff, 0.0);
RET_CHECK_GT(last_diff, 0);
const int64 curr_diff =
(next_output_timestamp_ - cc->InputTimestamp()).Value();
if (curr_diff > 0.0) {
if (curr_diff > 0) {
break;
}
OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
@@ -431,6 +318,9 @@ ::mediapipe::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
OutputWithinLimits(cc,
last_packet_.At(PeriodIndexToTimestamp(period_count_)));
}
if (!packet_reservoir_->IsEmpty()) {
OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return ::mediapipe::OkStatus();
}

0 comments on commit ae6be10

Please sign in to comment.
You can’t perform that action at this time.