native.proto

// Copyright 2016, Postmates Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

//
// Welcome!
//
// This file defines the protocol that cernan speaks natively. We hope that it's
// a relatively straightforward protocol to implement. Cernan's native transport
// is TCP. We require that all on-wire payloads have the following form:
//
//     [--------------------------------|~~~~~~~~~~ . . . ~~~~~~~~~~~~]
//     ^                                ^
//     u32, payload length in bytes     protobuf payload, of prefix len
//
// The protobuf payload conforms to the following definition.
syntax = "proto2";

package com.postmates.cernan;
option java_package = "com.postmates.cernan";

// 'Payload' - the top-level structure in each on-wire payload
//
// Payload is a container for repeated Telemetry and LogLines. There's not much
// more to it than that.
message Payload {
  repeated Telemetry points = 2;
  repeated LogLine lines = 3;
}

// 'LogLine' - a bit of unstructure text
//
// One of cernan's gigs is picking up logs from disk and transforming them
// in-flight, shipping them off. This structure allows you to ship lines
// directly via the native protocol without having to round-trip through disk
// first.
message LogLine {
  optional string path = 1; // unique 'location' of the log line
  optional string value = 2; // the line itself
  map<string, string> metadata = 3; // associated key/value metadata
  optional int64 timestamp_ms = 4; // milliseconds since the Unix epoch
}

// 'Telemetry' - a numeric measure of a thing
//
// Cernan's slightly more complicated gig is its 'telemetry'
// subsystem. Telemetry is defined as a name and time associated collection of
// measurements. In the structure we refer to these measurements as
// 'samples'. The Telemetry structure makes is possible to associate multiple
// samples in a single millisecond time window. Cernan will build a quantile
// structure over these samples but you may further choose aggregation
// interpretations by setting AggregationMethod.
message Telemetry {
  optional string name = 1; // the unique name of the telemetry
  repeated double samples = 2 [ packed = true ]; // telemetry samples present in timestamp_ms
  optional bool persisted = 3 [ default = false ]; // persist metric across time windows
  optional AggregationMethod method = 4 [ default = SUMMARIZE ]; // see below
  map<string, string> metadata = 5; // associated key/value metadata
  optional int64 timestamp_ms = 6; // milliseconds since the Unix epoch
  repeated double bin_bounds = 7; // BIN inclusive upper bounds
}

// 'AggregationMethod' - an interpretation signal
//
// Cernan maintains quantile summaries for all Telemetry samples. Not all sinks
// are capable of interpreting summaries natively. Cernan allows the client to
// set preferred aggregations over the summaries for reporting to 'flat'
// sinks. Sinks are allows to ignore AggregationMethod at their
// convenience. Additionally, aggregation time windows may be configured
// per-sink and are not controllable through the protocol.
enum AggregationMethod {
  // SUM keeps a sum of samples. This is often interpreted as a
  // per-window counter.
  SUM = 1;
  // SET preserves the last sample set into the Telemetry per time
  // window.
  SET = 2;
  // SUMMARIZE produces a quantile summary of the input samples per time
  // window. This is the default behaviour.
  SUMMARIZE = 3;
  // BIN produces a histogram summary of the input samples per time window. The
  // user will specify the bins' upper bounds.
  BIN = 4;
}