From aae1f170df1504451cb9e488ab0f38ca3b8b00df Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Wed, 18 Nov 2020 13:47:26 -0600 Subject: [PATCH] Beta label support (#1051) Co-authored-by: Mathieu Martin --- CHANGELOG.next.md | 1 + schemas/README.md | 15 ++++++++++++++ scripts/generators/asciidoc_fields.py | 3 ++- scripts/generators/ecs_helpers.py | 4 ++-- scripts/schema/cleaner.py | 14 +++++++++++++ scripts/schema/finalizer.py | 3 +++ scripts/templates/field_details.j2 | 22 +++++++++++++++++++++ scripts/tests/unit/test_schema_finalizer.py | 4 ++-- 8 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 0dcfdfaa7c..f87d61f45f 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -36,6 +36,7 @@ Thanks, you're awesome :-) --> * Added support for `scaled_float`'s mandatory parameter `scaling_factor`. #1042 * Added ability for --oss flag to fall back `constant_keyword` to `keyword`. #1046 * Added support in the generated Go source go for `wildcard`, `version`, and `constant_keyword` data types. #1050 +* Added support for marking fields, field sets, or field reuse as beta in the documentation. #1051 * Added support for `constant_keyword`'s optional parameter `value`. #1112 #### Improvements diff --git a/schemas/README.md b/schemas/README.md index 39b18f4bd7..2c14737c4a 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -32,6 +32,8 @@ Optional field set attributes: - type (ignored): at this level, should always be `group` - reusable (optional): Used to identify which field sets are expected to be reused in multiple places. See "Field set reuse" for details. +- beta: Adds a beta marker for the entire fieldset. The text provided in this attribute is used as content of the beta marker in the documentation. + Beta notices should not have newlines. ### Field set reuse @@ -104,6 +106,18 @@ The above defines all process fields in both places: } ``` +The `beta` marker can optionally be used along with `at` and `as` to include a beta marker in the field reuses section, marking specific reuse locations as beta. +Beta notices should not have newlines. + +``` + reusable: + top_level: true + expected: + - at: user + as: target + beta: Reusing these fields in this location is currently considered beta. +``` + ### List of fields Array of YAML objects: @@ -134,6 +148,7 @@ Supported keys to describe fields - format: Field format that can be used in a Kibana index template. - normalize: Normalization steps that should be applied at ingestion time. Supported values: - array: the content of the field should be an array (even when there's only one value). +- beta (optional): Adds a beta marker for the field to the description. The text provided in this attribute is used as content of the beta marker in the documentation. Note that when a whole field set is marked as beta, it is not necessary nor recommended to mark all fields in the field set as beta. Beta notices should not have newlines. Supported keys to describe expected values for a field diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index e5e2262bd0..04fd1fcacf 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -39,7 +39,8 @@ def render_nestings_reuse_section(fieldset): rows.append({ 'flat_nesting': "{}.*".format(reused_here_entry['full']), 'name': reused_here_entry['schema_name'], - 'short': reused_here_entry['short'] + 'short': reused_here_entry['short'], + 'beta': reused_here_entry.get('beta', '') }) return sorted(rows, key=lambda x: x['flat_nesting']) diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index 801319854c..086f4d592d 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -189,5 +189,5 @@ def strict_warning(msg): :param msg: custom text which will be displayed with wrapped boilerplate for strict warning messages. """ - warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode." - warnings.warn(warn_message) + warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode.\nWarning check:" + warnings.warn(warn_message, stacklevel=3) diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py index 185d0abedc..83f2d15933 100644 --- a/scripts/schema/cleaner.py +++ b/scripts/schema/cleaner.py @@ -76,6 +76,8 @@ def schema_mandatory_attributes(schema): def schema_assertions_and_warnings(schema): '''Additional checks on a fleshed out schema''' single_line_short_description(schema, strict=strict_mode) + if 'beta' in schema['field_details']: + single_line_beta_description(schema, strict=strict_mode) def normalize_reuse_notation(schema): @@ -181,6 +183,8 @@ def field_assertions_and_warnings(field): # check short description length if in strict mode single_line_short_description(field, strict=strict_mode) check_example_value(field, strict=strict_mode) + if 'beta' in field['field_details']: + single_line_beta_description(field, strict=strict_mode) if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS: msg = "Invalid level for field '{}'.\nValue: {}\nAcceptable values: {}".format( field['field_details']['name'], field['field_details']['level'], @@ -220,3 +224,13 @@ def check_example_value(field, strict=True): raise ValueError(msg) else: ecs_helpers.strict_warning(msg) + + +def single_line_beta_description(schema_or_field, strict=True): + if "\n" in schema_or_field['field_details']['beta']: + msg = "Beta descriptions must be single line.\n" + msg += f"Offending field or field set: {schema_or_field['field_details']['name']}" + if strict: + raise ValueError(msg) + else: + ecs_helpers.strict_warning(msg) diff --git a/scripts/schema/finalizer.py b/scripts/schema/finalizer.py index 3d1c7202b2..d1b4928507 100644 --- a/scripts/schema/finalizer.py +++ b/scripts/schema/finalizer.py @@ -128,6 +128,9 @@ def append_reused_here(reused_schema, reuse_entry, destination_schema): 'full': reuse_entry['full'], 'short': reused_schema['field_details']['short'], } + # Check for beta attribute + if 'beta' in reuse_entry: + reused_here_entry['beta'] = reuse_entry['beta'] destination_schema['schema_details']['reused_here'].extend([reused_here_entry]) diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 index 3eef363fa8..643c2ccf5d 100644 --- a/scripts/templates/field_details.j2 +++ b/scripts/templates/field_details.j2 @@ -10,6 +10,13 @@ Find additional usage and examples in the {{ fieldset['name'] }} fields <> +{#- Beta marker on nested fields -#} +{%- if entry['beta'] -%} +| beta:[ {{ entry['beta'] }}] + +{{ entry['short'] }} +{%- else %} | {{ entry['short'] }} +{%- endif %} // =============================================================== diff --git a/scripts/tests/unit/test_schema_finalizer.py b/scripts/tests/unit/test_schema_finalizer.py index 8a193a0454..cea5c01e6d 100644 --- a/scripts/tests/unit/test_schema_finalizer.py +++ b/scripts/tests/unit/test_schema_finalizer.py @@ -92,7 +92,7 @@ def schema_user(self): 'order': 2, 'expected': [ {'full': 'server.user', 'at': 'server', 'as': 'user'}, - {'full': 'user.target', 'at': 'user', 'as': 'target'}, + {'full': 'user.target', 'at': 'user', 'as': 'target', 'beta': 'Some beta notice'}, {'full': 'user.effective', 'at': 'user', 'as': 'effective'}, ] } @@ -211,7 +211,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self): fields['process']['schema_details']['reused_here']) self.assertIn({'full': 'user.effective', 'schema_name': 'user', 'short': 'short desc'}, fields['user']['schema_details']['reused_here']) - self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc'}, + self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc', 'beta': 'Some beta notice'}, fields['user']['schema_details']['reused_here']) self.assertIn({'full': 'server.user', 'schema_name': 'user', 'short': 'short desc'}, fields['server']['schema_details']['reused_here'])