diff --git a/pkg/api/conn_track.go b/pkg/api/conn_track.go index 9f5ae4fd0..a25fb95a0 100644 --- a/pkg/api/conn_track.go +++ b/pkg/api/conn_track.go @@ -21,10 +21,10 @@ import "time" type ConnTrack struct { // TODO: should by a pointer instead? - KeyDefinition KeyDefinition `yaml:"keyDefinition" doc:"fields that are used to identify the connection"` - OutputRecordTypes []string `yaml:"outputRecordTypes" enum:"ConnTrackOutputRecordTypeEnum" doc:"output record types to emit"` - OutputFields []OutputField `yaml:"outputFields" doc:"list of output fields"` - EndConnectionTimeout time.Duration `yaml:"endConnectionTimeout" doc:"duration of time to wait from the last flow log to end a connection"` + KeyDefinition KeyDefinition `yaml:"keyDefinition,omitempty" doc:"fields that are used to identify the connection"` + OutputRecordTypes []string `yaml:"outputRecordTypes,omitempty" enum:"ConnTrackOutputRecordTypeEnum" doc:"output record types to emit"` + OutputFields []OutputField `yaml:"outputFields,omitempty" doc:"list of output fields"` + EndConnectionTimeout time.Duration `yaml:"endConnectionTimeout,omitempty" doc:"duration of time to wait from the last flow log to end a connection"` } type ConnTrackOutputRecordTypeEnum struct { @@ -38,12 +38,12 @@ func ConnTrackOutputRecordTypeName(operation string) string { } type KeyDefinition struct { - FieldGroups []FieldGroup `yaml:"fieldGroups" doc:"list of field group definitions"` - Hash ConnTrackHash `yaml:"hash" doc:"how to build the connection hash"` + FieldGroups []FieldGroup `yaml:"fieldGroups,omitempty" doc:"list of field group definitions"` + Hash ConnTrackHash `yaml:"hash,omitempty" doc:"how to build the connection hash"` } type FieldGroup struct { - Name string `yaml:"name" doc:"field group name"` + Name string `yaml:"name,omitempty" doc:"field group name"` Fields []string `yaml:"fields" doc:"list of fields in the group"` } @@ -54,16 +54,16 @@ type FieldGroup struct { // When they are not set, a different hash will be computed for A->B and B->A, // and they are tracked as different connections. type ConnTrackHash struct { - FieldGroupRefs []string `yaml:"fieldGroupRefs" doc:"list of field group names to build the hash"` - FieldGroupARef string `yaml:"fieldGroupARef" doc:"field group name of endpoint A"` - FieldGroupBRef string `yaml:"fieldGroupBRef" doc:"field group name of endpoint B"` + FieldGroupRefs []string `yaml:"fieldGroupRefs,omitempty" doc:"list of field group names to build the hash"` + FieldGroupARef string `yaml:"fieldGroupARef,omitempty" doc:"field group name of endpoint A"` + FieldGroupBRef string `yaml:"fieldGroupBRef,omitempty" doc:"field group name of endpoint B"` } type OutputField struct { - Name string `yaml:"name" doc:"output field name"` - Operation string `yaml:"operation" enum:"ConnTrackOperationEnum" doc:"aggregate operation on the field value"` - SplitAB bool `yaml:"splitAB" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."` - Input string `yaml:"input" doc:"The input field to base the operation on. When omitted, 'name' is used"` + Name string `yaml:"name,omitempty" doc:"output field name"` + Operation string `yaml:"operation,omitempty" enum:"ConnTrackOperationEnum" doc:"aggregate operation on the field value"` + SplitAB bool `yaml:"splitAB,omitempty" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."` + Input string `yaml:"input,omitempty" doc:"The input field to base the operation on. When omitted, 'name' is used"` } type ConnTrackOperationEnum struct { diff --git a/pkg/api/decode_aws.go b/pkg/api/decode_aws.go index 52af88c00..c13e78c37 100644 --- a/pkg/api/decode_aws.go +++ b/pkg/api/decode_aws.go @@ -18,5 +18,5 @@ package api type DecodeAws struct { - Fields []string `yaml:"fields" json:"fields" doc:"list of aws flow log fields"` + Fields []string `yaml:"fields,omitempty" json:"fields" doc:"list of aws flow log fields"` } diff --git a/pkg/api/encode_kafka.go b/pkg/api/encode_kafka.go index b7f136fe6..08094be2b 100644 --- a/pkg/api/encode_kafka.go +++ b/pkg/api/encode_kafka.go @@ -20,11 +20,11 @@ package api type EncodeKafka struct { Address string `yaml:"address" json:"address" doc:"address of kafka server"` Topic string `yaml:"topic" json:"topic" doc:"kafka topic to write to"` - Balancer string `yaml:"balancer" json:"balancer" enum:"KafkaEncodeBalancerEnum" doc:"one of the following:"` - WriteTimeout int64 `yaml:"writeTimeout" json:"writeTimeout" doc:"timeout (in seconds) for write operation performed by the Writer"` - ReadTimeout int64 `yaml:"readTimeout" json:"readTimeout" doc:"timeout (in seconds) for read operation performed by the Writer"` - BatchBytes int64 `yaml:"batchBytes" json:"batchBytes" doc:"limit the maximum size of a request in bytes before being sent to a partition"` - BatchSize int `yaml:"batchSize" json:"batchSize" doc:"limit on how many messages will be buffered before being sent to a partition"` + Balancer string `yaml:"balancer,omitempty" json:"balancer,omitempty" enum:"KafkaEncodeBalancerEnum" doc:"one of the following:"` + WriteTimeout int64 `yaml:"writeTimeout,omitempty" json:"writeTimeout,omitempty" doc:"timeout (in seconds) for write operation performed by the Writer"` + ReadTimeout int64 `yaml:"readTimeout,omitempty" json:"readTimeout,omitempty" doc:"timeout (in seconds) for read operation performed by the Writer"` + BatchBytes int64 `yaml:"batchBytes,omitempty" json:"batchBytes,omitempty" doc:"limit the maximum size of a request in bytes before being sent to a partition"` + BatchSize int `yaml:"batchSize,omitempty" json:"batchSize,omitempty" doc:"limit on how many messages will be buffered before being sent to a partition"` } type KafkaEncodeBalancerEnum struct { diff --git a/pkg/api/encode_prom.go b/pkg/api/encode_prom.go index 5b6584d3b..e069a90d1 100644 --- a/pkg/api/encode_prom.go +++ b/pkg/api/encode_prom.go @@ -18,10 +18,10 @@ package api type PromEncode struct { - Metrics PromMetricsItems `yaml:"metrics" json:"metrics" doc:"list of prometheus metric definitions, each includes:"` - Port int `yaml:"port" json:"port" doc:"port number to expose \"/metrics\" endpoint"` - Prefix string `yaml:"prefix" json:"prefix" doc:"prefix added to each metric name"` - ExpiryTime int `yaml:"expiryTime" json:"expiryTime" doc:"seconds of no-flow to wait before deleting prometheus data item"` + Metrics PromMetricsItems `yaml:"metrics" json:"metrics,omitempty" doc:"list of prometheus metric definitions, each includes:"` + Port int `yaml:"port" json:"port,omitempty" doc:"port number to expose \"/metrics\" endpoint"` + Prefix string `yaml:"prefix" json:"prefix,omitempty" doc:"prefix added to each metric name"` + ExpiryTime int `yaml:"expiryTime" json:"expiryTime,omitempty" doc:"seconds of no-flow to wait before deleting prometheus data item"` } type PromEncodeOperationEnum struct { diff --git a/pkg/api/extract_aggregate.go b/pkg/api/extract_aggregate.go index 87e1702e2..6f0bfb426 100644 --- a/pkg/api/extract_aggregate.go +++ b/pkg/api/extract_aggregate.go @@ -4,9 +4,9 @@ type AggregateBy []string type AggregateOperation string type AggregateDefinition struct { - Name string `yaml:"name" json:"name" doc:"description of aggregation result"` - By AggregateBy `yaml:"by" json:"by" doc:"list of fields on which to aggregate"` - Operation AggregateOperation `yaml:"operation" json:"operation" doc:"sum, min, max, avg or raw_values"` - RecordKey string `yaml:"recordKey" json:"recordKey" doc:"internal field on which to perform the operation"` - TopK int `yaml:"topK" json:"topK" doc:"number of highest incidence to report (default - report all)"` + Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"description of aggregation result"` + By AggregateBy `yaml:"by,omitempty" json:"by,omitempty" doc:"list of fields on which to aggregate"` + Operation AggregateOperation `yaml:"operation,omitempty" json:"operation,omitempty" doc:"sum, min, max, avg or raw_values"` + RecordKey string `yaml:"recordKey,omitempty" json:"recordKey,omitempty" doc:"internal field on which to perform the operation"` + TopK int `yaml:"topK,omitempty" json:"topK,omitempty" doc:"number of highest incidence to report (default - report all)"` } diff --git a/pkg/api/ingest_collector.go b/pkg/api/ingest_collector.go index 3b2ddd6de..72fa97a1e 100644 --- a/pkg/api/ingest_collector.go +++ b/pkg/api/ingest_collector.go @@ -18,8 +18,8 @@ package api type IngestCollector struct { - HostName string `yaml:"hostName" json:"hostName" doc:"the hostname to listen on"` - Port int `yaml:"port" json:"port" doc:"the port number to listen on, for IPFIX/NetFlow v9. Omit or set to 0 to disable IPFIX/NetFlow v9 ingestion"` - PortLegacy int `yaml:"portLegacy" json:"portLegacy" doc:"the port number to listen on, for legacy NetFlow v5. Omit or set to 0 to disable NetFlow v5 ingestion"` - BatchMaxLen int `yaml:"batchMaxLen" json:"batchMaxLen" doc:"the number of accumulated flows before being forwarded for processing"` + HostName string `yaml:"hostName,omitempty" json:"hostName,omitempty" doc:"the hostname to listen on"` + Port int `yaml:"port,omitempty" json:"port,omitempty" doc:"the port number to listen on, for IPFIX/NetFlow v9. Omit or set to 0 to disable IPFIX/NetFlow v9 ingestion"` + PortLegacy int `yaml:"portLegacy,omitempty" json:"portLegacy,omitempty" doc:"the port number to listen on, for legacy NetFlow v5. Omit or set to 0 to disable NetFlow v5 ingestion"` + BatchMaxLen int `yaml:"batchMaxLen,omitempty" json:"batchMaxLen,omitempty" doc:"the number of accumulated flows before being forwarded for processing"` } diff --git a/pkg/api/ingest_grpc.go b/pkg/api/ingest_grpc.go index ee06a3cef..5da4128cc 100644 --- a/pkg/api/ingest_grpc.go +++ b/pkg/api/ingest_grpc.go @@ -1,6 +1,6 @@ package api type IngestGRPCProto struct { - Port int `yaml:"port" json:"port" doc:"the port number to listen on"` - BufferLen int `yaml:"bufferLength" json:"bufferLength" doc:"the length of the ingest channel buffer, in groups of flows, containing each group hundreds of flows (default: 100)"` + Port int `yaml:"port,omitempty" json:"port,omitempty" doc:"the port number to listen on"` + BufferLen int `yaml:"bufferLength,omitempty" json:"bufferLength,omitempty" doc:"the length of the ingest channel buffer, in groups of flows, containing each group hundreds of flows (default: 100)"` } diff --git a/pkg/api/ingest_kafka.go b/pkg/api/ingest_kafka.go index 3faed0a0f..98d9466db 100644 --- a/pkg/api/ingest_kafka.go +++ b/pkg/api/ingest_kafka.go @@ -18,10 +18,10 @@ package api type IngestKafka struct { - Brokers []string `yaml:"brokers" json:"brokers" doc:"list of kafka broker addresses"` - Topic string `yaml:"topic" json:"topic" doc:"kafka topic to listen on"` - GroupId string `yaml:"groupid" json:"groupid" doc:"separate groupid for each consumer on specified topic"` - GroupBalancers []string `yaml:"groupBalancers" json:"groupBalancers" doc:"list of balancing strategies (range, roundRobin, rackAffinity)"` - StartOffset string `yaml:"startOffset" json:"startOffset" doc:"FirstOffset (least recent - default) or LastOffset (most recent) offset available for a partition"` - BatchReadTimeout int64 `yaml:"batchReadTimeout" json:"batchReadTimeout" doc:"how often (in milliseconds) to process input"` + Brokers []string `yaml:"brokers,omitempty" json:"brokers,omitempty" doc:"list of kafka broker addresses"` + Topic string `yaml:"topic,omitempty" json:"topic,omitempty" doc:"kafka topic to listen on"` + GroupId string `yaml:"groupid,omitempty" json:"groupid,omitempty" doc:"separate groupid for each consumer on specified topic"` + GroupBalancers []string `yaml:"groupBalancers,omitempty" json:"groupBalancers,omitempty" doc:"list of balancing strategies (range, roundRobin, rackAffinity)"` + StartOffset string `yaml:"startOffset,omitempty" json:"startOffset,omitempty" doc:"FirstOffset (least recent - default) or LastOffset (most recent) offset available for a partition"` + BatchReadTimeout int64 `yaml:"batchReadTimeout,omitempty" json:"batchReadTimeout,omitempty" doc:"how often (in milliseconds) to process input"` } diff --git a/pkg/api/transform_filter.go b/pkg/api/transform_filter.go index 866319d0e..d467474ab 100644 --- a/pkg/api/transform_filter.go +++ b/pkg/api/transform_filter.go @@ -18,7 +18,7 @@ package api type TransformFilter struct { - Rules []TransformFilterRule `yaml:"rules" json:"rules" doc:"list of filter rules, each includes:"` + Rules []TransformFilterRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of filter rules, each includes:"` } type TransformFilterOperationEnum struct { @@ -32,6 +32,6 @@ func TransformFilterOperationName(operation string) string { } type TransformFilterRule struct { - Input string `yaml:"input" json:"input" doc:"entry input field"` - Type string `yaml:"type" json:"type" enum:"TransformFilterOperationEnum" doc:"one of the following:"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformFilterOperationEnum" doc:"one of the following:"` } diff --git a/pkg/api/transform_generic.go b/pkg/api/transform_generic.go index f0d1826dd..3fbeb63e1 100644 --- a/pkg/api/transform_generic.go +++ b/pkg/api/transform_generic.go @@ -18,8 +18,8 @@ package api type TransformGeneric struct { - Policy string `yaml:"policy" json:"policy" enum:"TransformGenericOperationEnum" doc:"key replacement policy; may be one of the following:"` - Rules []GenericTransformRule `yaml:"rules" json:"rules" doc:"list of transform rules, each includes:"` + Policy string `yaml:"policy,omitempty" json:"policy,omitempty" enum:"TransformGenericOperationEnum" doc:"key replacement policy; may be one of the following:"` + Rules []GenericTransformRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of transform rules, each includes:"` } type TransformGenericOperationEnum struct { @@ -32,8 +32,8 @@ func TransformGenericOperationName(operation string) string { } type GenericTransformRule struct { - Input string `yaml:"input" json:"input" doc:"entry input field"` - Output string `yaml:"output" json:"output" doc:"entry output field"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` } type GenericTransform []GenericTransformRule diff --git a/pkg/api/transform_network.go b/pkg/api/transform_network.go index 4147b117c..d1b369d00 100644 --- a/pkg/api/transform_network.go +++ b/pkg/api/transform_network.go @@ -39,9 +39,9 @@ func TransformNetworkOperationName(operation string) string { } type NetworkTransformRule struct { - Input string `yaml:"input" json:"input" doc:"entry input field"` - Output string `yaml:"output" json:"output" doc:"entry output field"` - Type string `yaml:"type" json:"type" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` } diff --git a/pkg/api/write_stdout.go b/pkg/api/write_stdout.go index 17191ee9d..f22ae4054 100644 --- a/pkg/api/write_stdout.go +++ b/pkg/api/write_stdout.go @@ -1,5 +1,5 @@ package api type WriteStdout struct { - Format string `yaml:"format" json:"format" doc:"the format of each line: printf (default) or json"` + Format string `yaml:"format,omitempty" json:"format,omitempty" doc:"the format of each line: printf (default) or json"` } diff --git a/pkg/config/pipeline_builder_test.go b/pkg/config/pipeline_builder_test.go index adc4f771d..357c26558 100644 --- a/pkg/config/pipeline_builder_test.go +++ b/pkg/config/pipeline_builder_test.go @@ -49,7 +49,7 @@ func TestLokiPipeline(t *testing.T) { b, err = json.Marshal(params[0]) require.NoError(t, err) - require.Equal(t, `{"name":"ingest","ingest":{"type":"collector","collector":{"hostName":"127.0.0.1","port":9999,"portLegacy":0,"batchMaxLen":0}}}`, string(b)) + require.Equal(t, `{"name":"ingest","ingest":{"type":"collector","collector":{"hostName":"127.0.0.1","port":9999}}}`, string(b)) b, err = json.Marshal(params[1]) require.NoError(t, err) @@ -136,7 +136,7 @@ func TestKafkaPromPipeline(t *testing.T) { b, err = json.Marshal(params[0]) require.NoError(t, err) - require.Equal(t, `{"name":"ingest","ingest":{"type":"kafka","kafka":{"brokers":["http://kafka"],"topic":"netflows","groupid":"my-group","groupBalancers":null,"startOffset":"","batchReadTimeout":0}}}`, string(b)) + require.Equal(t, `{"name":"ingest","ingest":{"type":"kafka","kafka":{"brokers":["http://kafka"],"topic":"netflows","groupid":"my-group"}}}`, string(b)) b, err = json.Marshal(params[1]) require.NoError(t, err) @@ -144,11 +144,11 @@ func TestKafkaPromPipeline(t *testing.T) { b, err = json.Marshal(params[2]) require.NoError(t, err) - require.Equal(t, `{"name":"aggregate","extract":{"type":"aggregates","aggregates":[{"name":"src_as_connection_count","by":["srcAS"],"operation":"count","recordKey":"","topK":0}]}}`, string(b)) + require.Equal(t, `{"name":"aggregate","extract":{"type":"aggregates","aggregates":[{"name":"src_as_connection_count","by":["srcAS"],"operation":"count"}]}}`, string(b)) b, err = json.Marshal(params[3]) require.NoError(t, err) - require.Equal(t, `{"name":"prom","encode":{"type":"prom","prom":{"metrics":[{"name":"connections_per_source_as","type":"counter","filter":{"key":"name","value":"src_as_connection_count"},"valueKey":"recent_count","labels":["by","aggregate"],"buckets":[]}],"port":9090,"prefix":"flp_","expiryTime":0}}}`, string(b)) + require.Equal(t, `{"name":"prom","encode":{"type":"prom","prom":{"metrics":[{"name":"connections_per_source_as","type":"counter","filter":{"key":"name","value":"src_as_connection_count"},"valueKey":"recent_count","labels":["by","aggregate"],"buckets":[]}],"port":9090,"prefix":"flp_"}}}`, string(b)) } func TestForkPipeline(t *testing.T) { @@ -167,7 +167,7 @@ func TestForkPipeline(t *testing.T) { b, err = json.Marshal(params[0]) require.NoError(t, err) - require.Equal(t, `{"name":"ingest","ingest":{"type":"collector","collector":{"hostName":"127.0.0.1","port":9999,"portLegacy":0,"batchMaxLen":0}}}`, string(b)) + require.Equal(t, `{"name":"ingest","ingest":{"type":"collector","collector":{"hostName":"127.0.0.1","port":9999}}}`, string(b)) b, err = json.Marshal(params[1]) require.NoError(t, err) @@ -175,5 +175,5 @@ func TestForkPipeline(t *testing.T) { b, err = json.Marshal(params[2]) require.NoError(t, err) - require.Equal(t, `{"name":"stdout","write":{"type":"stdout","stdout":{"format":""}}}`, string(b)) + require.Equal(t, `{"name":"stdout","write":{"type":"stdout","stdout":{}}}`, string(b)) }