Skip to content
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
425 lines (391 sloc) 20.7 KB
// Copyright 2019 The MediaPipe Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
// Forked from mediapipe/framework/calculator.proto.
// The forked proto must remain identical to the original proto and should be
// ONLY used by mediapipe open source project.
syntax = "proto3";
package mediapipe;
import public "mediapipe/framework/calculator_options.proto";
import "google/protobuf/any.proto";
import "mediapipe/framework/mediapipe_options.proto";
import "mediapipe/framework/packet_factory.proto";
import "mediapipe/framework/packet_generator.proto";
import "mediapipe/framework/status_handler.proto";
import "mediapipe/framework/stream_handler.proto";
option java_package = "";
option java_outer_classname = "CalculatorProto";
// Describes a MediaPipe Executor.
message ExecutorConfig {
// The name of the executor (used by a CalculatorGraphConfig::Node or
// PacketGeneratorConfig to specify which executor it will execute on).
// This field must be unique within a CalculatorGraphConfig. If this field
// is omitted or is an empty string, the ExecutorConfig describes the
// default executor.
// NOTE: The names "default" and "gpu" are reserved and must not be used.
string name = 1;
// The registered type of the executor. For example: "ThreadPoolExecutor".
// The framework will create an executor of this type (with the options in
// the options field) for the CalculatorGraph.
// The ExecutorConfig for the default executor may omit this field and let
// the framework choose an appropriate executor type. Note: If the options
// field is used in this case, it should contain the
// ThreadPoolExecutorOptions.
// If the ExecutorConfig for an additional (non-default) executor omits this
// field, the executor must be created outside the CalculatorGraph and
// passed to the CalculatorGraph for use.
string type = 2;
// The options passed to the Executor. The extension in the options field
// must match the type field. For example, if the type field is
// "ThreadPoolExecutor", then the options field should contain the
// ThreadPoolExecutorOptions.
MediaPipeOptions options = 3;
// A collection of input data to a CalculatorGraph.
message InputCollection {
// The name of the input collection. Name must match [a-z_][a-z0-9_]*
string name = 1;
// The names of each side packet. The number of side_packet_name
// must match the number of packets generated by the input file.
repeated string side_packet_name = 2;
// DEPRECATED: old way of referring to side_packet_name.
repeated string external_input_name = 1002;
// The input can be specified in several ways.
enum InputType {
// An invalid default value. This value is guaranteed to be the
// lowest enum value (i.e. don't add negative enum values).
// A recordio where each record is a serialized PacketManagerConfig.
// Each PacketManagerConfig must have the same number of packet
// factories in it as the number of side packet names. Furthermore,
// the output side packet name field in each PacketFactoryConfig
// must not be set. This is the most general input, and allows
// multiple side packet values to be set in arbitrarily complicated
// ways before each run.
// A recordio where each record is a serialized packet payload.
// For example a recordio of serialized OmniaFeature protos dumped
// from Omnia.
// A text file where each line is a comma separated list. The number
// of elements for each csv string must be the same as the number
// of side_packet_name (and the order must match). Each line must
// be less than 1MiB in size. Lines comprising of only whitespace
// or only whitespace and a pound comment will be skipped.
// This and all higher values are invalid. Update this value to
// always be larger than any other enum values you add.
// Sets the source of the input collection data.
// The default value is UNKNOWN.
InputType input_type = 3;
// A file name pointing to the data. The format of the data is
// specified by the "input_type" field. Multiple shards may be
// specified using @N or glob expressions.
string file_name = 4;
// A convenient way to specify a number of InputCollections.
message InputCollectionSet {
repeated InputCollection input_collection = 1;
// Additional information about an input stream.
message InputStreamInfo {
// A description of the input stream.
// This description uses the Calculator visible specification of
// a stream. The format is a tag, then an index with both being
// optional. If the tag is missing it is assumed to be "" and if
// the index is missing then it is assumed to be 0. If the index
// is provided then a colon (':') must be used.
// Examples:
// "TAG" -> tag "TAG", index 0
// "" -> tag "", index 0
// ":0" -> tag "", index 0
// ":3" -> tag "", index 3
// "VIDEO:0" -> tag "VIDEO", index 0
// "VIDEO:2" -> tag "VIDEO", index 2
string tag_index = 1;
// Whether the input stream is a back edge.
// By default, MediaPipe requires graphs to be acyclic and treats cycles in a
// graph as errors. To allow MediaPipe to accept a cyclic graph, set the
// back_edge fields of the input streams that are back edges to true. A
// cyclic graph usually has an obvious forward direction, and a back edge
// goes in the opposite direction. For a formal definition of a back edge,
// please see
bool back_edge = 2;
// Configs for the profiler for a calculator. Not applicable to subgraphs.
message ProfilerConfig {
// Size of the runtimes histogram intervals (in microseconds) to generate the
// histogram of the Process() time. The last interval extends to +inf.
// If not specified, the interval is 1000000 usec = 1 sec.
int64 histogram_interval_size_usec = 1;
// Number of intervals to generate the histogram of the Process() runtime.
// If not specified, one interval is used.
int64 num_histogram_intervals = 2;
// TODO: clean up after migration to MediaPipeProfiler.
// DEPRECATED: If true, the profiler also profiles the input output latency.
// Should be true only if the packet timestamps corresponds to the
// microseconds wall time from epoch.
bool enable_input_output_latency = 3 [deprecated = true];
// If true, the profiler starts profiling when graph is initialized.
bool enable_profiler = 4;
// If true, the profiler also profiles the stream latency and input-output
// latency.
// No-op if enable_profiler is false.
bool enable_stream_latency = 5;
// If true, the profiler uses packet timestamp (as production time and source
// production time) for packets added by calling
// CalculatorGraph::AddPacketToInputStream().
// If false, uses profiler's clock.
bool use_packet_timestamp_for_added_packet = 6;
// The maximum number of trace events buffered in memory.
// The default value buffers up to 20000 events.
int64 trace_log_capacity = 7;
// Trace event types that are not logged.
repeated int32 trace_event_types_disabled = 8;
// The output directory and base-name prefix for trace log files.
// Log files are written to: StrCat(trace_log_path, index, ".binarypb")
string trace_log_path = 9;
// The number of trace log files retained.
// The trace log files are named "trace_0.log" through "trace_k.log".
// The default value specifies 2 output files retained.
int32 trace_log_count = 10;
// The interval in microseconds between trace log output.
// The default value specifies trace log output once every 0.5 sec.
int64 trace_log_interval_usec = 11;
// The interval in microseconds between TimeNow and the highest times
// included in trace log output. This margin allows time for events
// to be appended to the TraceBuffer.
int64 trace_log_margin_usec = 12;
// False specifies an event for each calculator invocation.
// True specifies a separate event for each start and finish time.
bool trace_log_duration_events = 13;
// The number of trace log intervals per file. The total log duration is:
// trace_log_interval_usec * trace_log_file_count * trace_log_interval_count.
// The default value specifies 10 intervals per file.
int32 trace_log_interval_count = 14;
// An option to turn ON/OFF writing trace files to disk. Saving trace files to
// disk is enabled by default.
bool trace_log_disabled = 15;
// If true, tracer timing events are recorded and reported.
bool trace_enabled = 16;
// Describes the topology and function of a MediaPipe Graph. The graph of
// Nodes must be a Directed Acyclic Graph (DAG) except as annotated by
// "back_edge" in InputStreamInfo. Use a mediapipe::CalculatorGraph object to
// run the graph.
message CalculatorGraphConfig {
// A single node in the DAG.
message Node {
// The name of the node. This field is optional and doesn't generally
// need to be specified, but does improve error messaging.
string name = 1;
// The registered type of a calculator (provided via REGISTER_CALCULATOR),
// or of a subgraph (via REGISTER_MEDIAPIPE_GRAPH).
string calculator = 2;
// A Calculator can choose to access its input streams, output
// streams, and input side packets either by tag or by index. If the
// calculator chooses indexes then it will receive the streams or side
// packets in the same order as they are specified in this proto.
// If the calculator chooses to use tags then it must specify a
// tag along with each name. The field is given as "TAG:name".
// Meaning a tag name followed by a colon followed by the name.
// Tags use only upper case letters, numbers, and underscores, whereas
// names use only lower case letters, numbers, and underscores.
// Example:
// Node {
// calculator: "SomeAudioVideoCalculator"
// # This calculator accesses its inputs by index (no tag needed).
// input_stream: "combined_input"
// # This calculator accesses its outputs by tags, so all
// # output_streams must specify a tag.
// output_stream: "AUDIO:audio_stream"
// output_stream: "VIDEO:video_stream"
// # This calculator accesses its input side packets by tag.
// input_side_packet: "MODEL:model_01"
// }
// String(s) representing "TAG:name" of the stream(s) from which the current
// node will get its inputs. "TAG:" part is optional, see above.
// A calculator with no input stream is a source.
repeated string input_stream = 3;
// String(s) representing "TAG:name" of the stream(s) produced by this node.
// "TAG:" part is optional, see above. These must be different from any
// other output_streams specified for other nodes in the graph.
repeated string output_stream = 4;
// String(s) representing "TAG:name" of the input side packet(s).
// "TAG:" part is optional, see above.
repeated string input_side_packet = 5;
// String(s) representing "TAG:name" of the output side packet(s). Only
// used by subgraphs.
// "TAG:" part is optional, see above.
repeated string output_side_packet = 6;
// The options passed to the Calculator, in proto2 syntax.
CalculatorOptions options = 7;
// The options passed to the Calculator, in proto3 syntax.
// Each node_options message must have a different message type.
// If the same message type is specified in |options| and |node_options|,
// only the message in |options| is used.
repeated google.protobuf.Any node_options = 8;
// Note: the following fields are only applicable to calculators, not
// subgraphs.
// For a Source Calculator (i.e. a calculator with no inputs),
// this is the "layer" on which the calculator is executed. For a
// non-source calculator (i.e. a calculator with one or more input
// streams) this field has no effect. The sources on each layer
// are completely exhausted before Process() is called on any source
// calculator on a higher numbered layer.
// Example:
// Decoder -> Median Frame (requires all frames) -> Image Subtraction
// --------------------------------------->
// The entire video will be buffered on the edge from the decoder
// to the Image subtraction. To fix this problem, layers can be used.
// Decoder (layer 0) -> Median Frame -> Image Subtraction
// Decoder (layer 1) ----------------->
// The frames from layer 0 will no longer be buffered, but the video
// will be decoded again instead. Note, that different options can
// be used in the second decoder.
int32 source_layer = 9;
// Optional parameter that allows the user to indicate to the scheduler that
// this node has a buffering behavior (i.e. waits for a bunch of packets
// before emitting any) and specify the size of the buffer that is built up.
// The scheduler will then try to keep the maximum size of any input queues
// in the graph to remain below the maximum of all buffer_size_hints and
// max_queue_size (if specified). The ideal value is typically something
// larger than the actual number of buffered packets to maintain pipelining.
// The default value 0 indicates that the node has no buffering behavior.
int32 buffer_size_hint = 10;
// Config for this node's InputStreamHandler.
// If unspecified, the graph-level input stream handler will be used.
InputStreamHandlerConfig input_stream_handler = 11;
// Config for this node's OutputStreamHandler.
// If unspecified, the graph-level output stream handler will be used.
OutputStreamHandlerConfig output_stream_handler = 12;
// Additional information about an input stream. The |name| field of the
// InputStreamInfo must match an input_stream.
repeated InputStreamInfo input_stream_info = 13;
// Set the executor which the calculator will execute on.
string executor = 14;
// TODO: Remove from Node when switched to Profiler.
// DEPRECATED: Configs for the profiler.
ProfilerConfig profiler_config = 15 [deprecated = true];
// The maximum number of invocations that can be executed in parallel.
// If not specified, the limit is one invocation.
int32 max_in_flight = 16;
// DEPRECATED: For backwards compatibility we allow users to
// specify the old name for "input_side_packet" in proto configs.
// These are automatically converted to input_side_packets during
// config canonicalization.
repeated string external_input = 1005;
// The nodes.
repeated Node node = 1;
// Create a side packet using a PacketFactory. This side packet is
// created as close to the worker that does the work as possible. A
// PacketFactory is basically a PacketGenerator that takes no input side
// packets and produces a single output side packet.
repeated PacketFactoryConfig packet_factory = 6;
// Configs for PacketGenerators. Generators take zero or more
// input side packets and produce any number of output side
// packets. For example, MediaDecoderCalculator takes an input
// side packet with type DeletingFile. However, most users want
// to specify videos by ContentIdHex (i.e. video id). By using
// the VideoIdToLocalFileGenerator, a user can specify a video id
// (as a string) and obtain a DeletingFile to use with the decoder.
// PacketGenerators can take as a input side packet the output side
// packet of another PacketGenerator. The graph of PacketGenerators
// must be a directed acyclic graph.
repeated PacketGeneratorConfig packet_generator = 7;
// Number of threads for running calculators in multithreaded mode.
// If not specified, the scheduler will pick an appropriate number
// of threads depending on the number of available processors.
// To run on the calling thread, specify "ApplicationThreadExecutor"
// see: http://g3doc/mediapipe/g3doc/
int32 num_threads = 8;
// Configs for StatusHandlers that will be called after each call to
// Run() on the graph. StatusHandlers take zero or more input side
// packets and the ::util::Status returned by a graph run. For example,
// a StatusHandler could store information about graph failures and
// their causes for later monitoring. Note that graph failures during
// initialization may cause required input side packets (created by a
// PacketFactory or PacketGenerator) to be missing. In these cases,
// the handler with missing input side packets will be skipped.
repeated StatusHandlerConfig status_handler = 9;
// Specify input streams to the entire graph. Streams specified here may have
// packets added to them using CalculatorGraph::AddPacketToInputStream. This
// works much like a source calculator, except that the source is outside of
// the mediapipe graph.
repeated string input_stream = 10;
// Output streams for the graph when used as a subgraph.
repeated string output_stream = 15;
// Input side packets for the graph when used as a subgraph.
repeated string input_side_packet = 16;
// Output side packets for the graph when used as a subgraph.
repeated string output_side_packet = 17;
// Maximum queue size of any input stream in the graph. This can be used to
// control the memory usage of a MediaPipe graph by preventing fast sources
// from flooding the graph with packets. Any source that is connected to an
// input stream that has hit its maximum capacity will not be scheduled until
// the queue size falls under the specified limits, or if the scheduler queue
// is empty and no other nodes are running (to prevent possible deadlocks due
// to a incorrectly specified value). This global parameter is set to 100
// packets by default to enable pipelining. If any node indicates that it
// buffers packets before emitting them, then the max(node_buffer_size,
// max_queue_size) is used. Set this parameter to -1 to disable throttling
// (i.e. the graph will use as much memory as it requires). If not specified,
// the limit is 100 packets.
int32 max_queue_size = 11;
// If true, the graph run fails with an error when throttling prevents all
// calculators from running. If false, max_queue_size for an input stream
// is adjusted when throttling prevents all calculators from running.
bool report_deadlock = 21;
// Config for this graph's InputStreamHandler.
// If unspecified, the framework will automatically install the default
// handler, which works as follows.
// The calculator's Process() method is called for timestamp t when:
// - at least one stream has a packet available at t; and,
// - all other streams either have packets at t, or it is known that they will
// not have packets at t (i.e. their next timestamp bound is greater than t).
// The handler then provides all available packets with timestamp t, with no
// preprocessing.
InputStreamHandlerConfig input_stream_handler = 12;
// Config for this graph's OutputStreamHandler.
// If unspecified, the default output stream handler will be automatically
// installed by the framework which does not modify any outgoing packets.
OutputStreamHandlerConfig output_stream_handler = 13;
// Configs for Executors.
// The names of the executors must be distinct. The default executor, whose
// name is the empty string, is predefined. The num_threads field of the
// CalculatorGraphConfig specifies the number of threads in the default
// executor. If the config for the default executor is specified, the
// CalculatorGraphConfig must not have the num_threads field.
repeated ExecutorConfig executor = 14;
// The default profiler-config for all calculators. If set, this defines the
// profiling settings such as num_histogram_intervals for every calculator in
// the graph. Each of these settings can be overridden by the
// |profiler_config| specified for a node.
ProfilerConfig profiler_config = 18;
// The namespace used for class name lookup within this graph.
// An unqualified or partially qualified class name is looked up in
// this namespace first and then in enclosing namespaces.
string package = 19;
// The type name for the graph config, used for registering and referencing
// the graph config.
string type = 20;
// Can be used for annotating a graph.
MediaPipeOptions options = 1001;
You can’t perform that action at this time.