[pull] master from timberio:master

.meta/.schema.json

@@ -522,10 +522,6 @@
           "description": "A simple title for this transform, typically one word.",
           "minLength": 1,
           "max_lenfth": 50
-        },
-        "types_coercion": {
-          "type": "boolean",
-          "description": "If this transform offers a `type` table for field coercion."
         }
       }
     }

.meta/_partials/_types.toml

@@ -0,0 +1,28 @@
+[<%= namespace %>.options.types]
+type = "table"
+common = <%= common %>
+description = "Key/Value pairs representing mapped log field types."
+
+[<%= namespace %>.options.types.options."`[field-name]`"]
+type = "string"
+common = <%= common %>
+examples = [
+{status = "int"},
+{duration = "float"},
+{success = "bool"},
+{timestamp = "timestamp|%F"},
+{timestamp = "timestamp|%a %b %e %T %Y"}
+]
+null = false
+description = """\
+A definition of log field type conversions. They key is the log field name and \
+the value is the type. [`strptime` specifiers][urls.strptime_specifiers] are \
+supported for the `timestamp` type.\
+"""
+
+[<%= namespace %>.options.types.options."`[field-name]`".enum]
+bool = "Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean."
+float = "Coerce to a 64 bit float."
+int = "Coerce to a 64 bit integer."
+string = "Coerce to a string."
+timestamp = "Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."

.meta/transforms/coercer.toml

@@ -5,7 +5,6 @@ common = false
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.coercer.options.drop_unspecified]
 type = "bool"
@@ -18,6 +17,8 @@ as their absense will cause the original message data to be dropped along \
 with other extraneous fields.\
 """
 
+<%= render("_partials/_types.toml", namespace: "transforms.coercer", common: true) %>
+
 [[transforms.coercer.output.log.examples]]
 label = "Generic"
 body = """\

.meta/transforms/grok_parser.toml

@@ -5,7 +5,6 @@ common = false
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.grok_parser.options.drop_field]
 type = "bool"
@@ -31,3 +30,5 @@ common = true
 examples = ["%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}"]
 required = true
 description = "The [Grok pattern][urls.grok_patterns]"
+
+<%= render("_partials/_types.toml", namespace: "transforms.grok_parser", common: true) %>

.meta/transforms/logfmt_parser.toml

@@ -7,7 +7,6 @@ common = true
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.logfmt_parser.options.field]
 type = "string"
@@ -22,3 +21,5 @@ common = true
 default = true
 required = true
 description = "If the specified `field` should be dropped (removed) after parsing."
+
+<%= render("_partials/_types.toml", namespace: "transforms.logfmt_parser", common: true) %>

.meta/transforms/regex_parser.toml

@@ -7,7 +7,6 @@ common = true
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.regex_parser.options.drop_field]
 type = "bool"
@@ -35,3 +34,5 @@ required = true
 description = """\
 The Regular Expression to apply. Do not include the leading or trailing `/`.\
 """
+
+<%= render("_partials/_types.toml", namespace: "transforms.regex_parser", common: true) %>

.meta/transforms/split.toml

@@ -8,7 +8,6 @@ common = false
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.split.options.field]
 type = "string"
@@ -41,3 +40,5 @@ common = true
 default = true
 required = true
 description = "If `true` the `field` will be dropped after parsing."
+
+<%= render("_partials/_types.toml", namespace: "transforms.split", common: true) %>

.meta/transforms/tokenizer.toml

@@ -8,7 +8,6 @@ common = true
 function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
-types_coercion = true
 
 [transforms.tokenizer.options.field]
 type = "string"
@@ -30,3 +29,5 @@ common = true
 default = true
 required = true
 description = "If `true` the `field` will be dropped after parsing."
+
+<%= render("_partials/_types.toml", namespace: "transforms.tokenizer", common: true) %>

config/vector.spec.toml

@@ -853,8 +853,6 @@ dns_servers = ["0.0.0.0:53"]
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 
@@ -1023,8 +1021,6 @@ dns_servers = ["0.0.0.0:53"]
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 
@@ -1203,8 +1199,6 @@ dns_servers = ["0.0.0.0:53"]
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 
@@ -1351,8 +1345,6 @@ end
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 
@@ -1498,8 +1490,6 @@ end
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 
@@ -1560,8 +1550,6 @@ end
     status = "int"
     duration = "float"
     success = "bool"
-    timestamp = "timestamp|%s"
-    timestamp = "timestamp|%+"
     timestamp = "timestamp|%F"
     timestamp = "timestamp|%a %b %e %T %Y"
 

scripts/util/metadata.rb

@@ -1,3 +1,4 @@
+require "erb"
 require "json_schemer"
 require "ostruct"
 require "toml-rb"
@@ -17,6 +18,23 @@
 # This represents the /.meta directory in object form. Sub-classes represent
 # each sub-component.
 class Metadata
+  module Template
+    extend self
+
+    def render(path, args = {})
+      context = binding
+
+      args.each do |key, value|
+        context.local_variable_set("#{key}", value)
+      end
+
+      full_path = path.start_with?("/") ? path : "#{META_ROOT}/#{path}"
+      body = File.read(full_path)
+      renderer = ERB.new(body, nil, '-')
+      renderer.result(context)
+    end
+  end
+
   class << self
     def load!(meta_dir, docs_root, pages_root)
       metadata = load_metadata!(meta_dir)
@@ -40,17 +58,18 @@ def load_json_schema!(meta_dir)
       def load_metadata!(meta_dir)
         metadata = {}
 
-        Dir.glob("#{meta_dir}/**/*.toml").each do |file|
-          hash = TomlRB.load_file(file)
+        Dir.glob("#{meta_dir}/**/[^_]*.toml").each do |file|
+          content = Template.render(file)
+          hash = TomlRB.parse(content)
           metadata.deep_merge!(hash)
         end
 
         metadata
       end
 
       def read_json(path)
-        json_data = File.read(path)
-        JSON.parse(json_data)
+        body = File.read(path)
+        JSON.parse(body)
       end
 
       def validate_schema!(schema, metadata)

scripts/util/metadata/transform.rb

@@ -18,7 +18,6 @@ def initialize(hash)
     @input_types = hash.fetch("input_types")
     @output = OpenStruct.new
     @output_types = hash.fetch("output_types")
-    types_coercion = hash["types_coercion"] == true
 
     # checks
 
@@ -45,46 +44,6 @@ def initialize(hash)
     if output["metric"]
       @output.metric = Output.new(output["metric"])
     end
-
-    # types
-
-    if types_coercion
-      wildcard_option =
-        {
-          "name" => "`[field-name]`",
-          "category" => "requests",
-          "enum" => {
-            "bool" => "Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean.",
-            "float" => "Coerce to a 64 bit float.",
-            "int" => "Coerce to a 64 bit integer.",
-            "string" => "Coerce to a string.",
-            "timestamp" => "Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."
-          },
-          "examples" => [
-            {"status" => "int"},
-            {"duration" => "float"},
-            {"success" => "bool"},
-            {"timestamp" => "timestamp|%s"},
-            {"timestamp" => "timestamp|%+"},
-            {"timestamp" => "timestamp|%F"},
-            {"timestamp" => "timestamp|%a %b %e %T %Y"}
-          ],
-          "description" => "A definition of log field type conversions. They key is the log field name and the value is the type. [`strptime` specifiers][urls.strptime_specifiers] are supported for the `timestamp` type.",
-          "null" => false,
-          "simple" => true,
-          "type" => "string"
-        }
-
-      @options.types =
-        Option.new({
-          "name" => "types",
-          "common" => true,
-          "description" => "Key/Value pairs representing mapped log field types.",
-          "null" => true,
-          "options" => {"`[field-name]`" => wildcard_option},
-          "type" => "table"
-        })
-    end
   end
 
   def description

website/docs/reference/transforms/coercer.md

@@ -47,6 +47,11 @@ import CodeHeader from '@site/src/components/CodeHeader';
 
   # OPTIONAL - Types
   [transforms.my_transform_id.types]
+    status = "int" # example
+    duration = "float" # example
+    success = "bool" # example
+    timestamp = "timestamp|%F" # example
+    timestamp = "timestamp|%a %b %e %T %Y" # example
 ```
 
 </TabItem>
@@ -68,8 +73,6 @@ import CodeHeader from '@site/src/components/CodeHeader';
     status = "int" # example
     duration = "float" # example
     success = "bool" # example
-    timestamp = "timestamp|%s" # example
-    timestamp = "timestamp|%+" # example
     timestamp = "timestamp|%F" # example
     timestamp = "timestamp|%a %b %e %T %Y" # example
 ```
@@ -131,10 +134,10 @@ Key/Value pairs representing mapped log field types.
 
 
 <Field
-  common={false}
+  common={true}
   defaultValue={null}
   enumValues={{"bool":"Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean.","float":"Coerce to a 64 bit float.","int":"Coerce to a 64 bit integer.","string":"Coerce to a string.","timestamp":"Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."}}
-  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%s"},{"timestamp":"timestamp|%+"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
+  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
   name={"`[field-name]`"}
   path={"types"}
   relevantWhen={null}

website/docs/reference/transforms/grok_parser.md

@@ -37,6 +37,11 @@ import CodeHeader from '@site/src/components/CodeHeader';
 
   # OPTIONAL - Types
   [transforms.my_transform_id.types]
+    status = "int" # example
+    duration = "float" # example
+    success = "bool" # example
+    timestamp = "timestamp|%F" # example
+    timestamp = "timestamp|%a %b %e %T %Y" # example
 ```
 
 ## Options
@@ -136,10 +141,10 @@ Key/Value pairs representing mapped log field types.
 
 
 <Field
-  common={false}
+  common={true}
   defaultValue={null}
   enumValues={{"bool":"Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean.","float":"Coerce to a 64 bit float.","int":"Coerce to a 64 bit integer.","string":"Coerce to a string.","timestamp":"Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."}}
-  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%s"},{"timestamp":"timestamp|%+"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
+  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
   name={"`[field-name]`"}
   path={"types"}
   relevantWhen={null}

website/docs/reference/transforms/logfmt_parser.md

@@ -36,6 +36,11 @@ import CodeHeader from '@site/src/components/CodeHeader';
 
   # OPTIONAL - Types
   [transforms.my_transform_id.types]
+    status = "int" # example
+    duration = "float" # example
+    success = "bool" # example
+    timestamp = "timestamp|%F" # example
+    timestamp = "timestamp|%a %b %e %T %Y" # example
 ```
 
 ## Options
@@ -113,10 +118,10 @@ Key/Value pairs representing mapped log field types.
 
 
 <Field
-  common={false}
+  common={true}
   defaultValue={null}
   enumValues={{"bool":"Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean.","float":"Coerce to a 64 bit float.","int":"Coerce to a 64 bit integer.","string":"Coerce to a string.","timestamp":"Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."}}
-  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%s"},{"timestamp":"timestamp|%+"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
+  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
   name={"`[field-name]`"}
   path={"types"}
   relevantWhen={null}

website/docs/reference/transforms/regex_parser.md

@@ -37,6 +37,11 @@ import CodeHeader from '@site/src/components/CodeHeader';
 
   # OPTIONAL - Types
   [transforms.my_transform_id.types]
+    status = "int" # example
+    duration = "float" # example
+    success = "bool" # example
+    timestamp = "timestamp|%F" # example
+    timestamp = "timestamp|%a %b %e %T %Y" # example
 ```
 
 ## Options
@@ -136,10 +141,10 @@ Key/Value pairs representing mapped log field types. See [Regex Syntax](#regex-s
 
 
 <Field
-  common={false}
+  common={true}
   defaultValue={null}
   enumValues={{"bool":"Coerces `\"true\"`/`/\"false\"`, `\"1\"`/`\"0\"`, and `\"t\"`/`\"f\"` values into boolean.","float":"Coerce to a 64 bit float.","int":"Coerce to a 64 bit integer.","string":"Coerce to a string.","timestamp":"Coerces to a Vector timestamp. [`strptime` specificiers][urls.strptime_specifiers] must be used to parse the string."}}
-  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%s"},{"timestamp":"timestamp|%+"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
+  examples={[{"status":"int"},{"duration":"float"},{"success":"bool"},{"timestamp":"timestamp|%F"},{"timestamp":"timestamp|%a %b %e %T %Y"}]}
   name={"`[field-name]`"}
   path={"types"}
   relevantWhen={null}