Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[1.x] Beta label support (#1051) #1133

Merged
merged 1 commit into from
Nov 18, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Thanks, you're awesome :-) -->
* Added support for `scaled_float`'s mandatory parameter `scaling_factor`. #1042
* Added ability for --oss flag to fall back `constant_keyword` to `keyword`. #1046
* Added support in the generated Go source go for `wildcard`, `version`, and `constant_keyword` data types. #1050
* Added support for marking fields, field sets, or field reuse as beta in the documentation. #1051
* Added support for `constant_keyword`'s optional parameter `value`. #1112

#### Improvements
Expand Down
15 changes: 15 additions & 0 deletions schemas/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ Optional field set attributes:
- type (ignored): at this level, should always be `group`
- reusable (optional): Used to identify which field sets are expected to be reused in multiple places.
See "Field set reuse" for details.
- beta: Adds a beta marker for the entire fieldset. The text provided in this attribute is used as content of the beta marker in the documentation.
Beta notices should not have newlines.

### Field set reuse

Expand Down Expand Up @@ -104,6 +106,18 @@ The above defines all process fields in both places:
}
```

The `beta` marker can optionally be used along with `at` and `as` to include a beta marker in the field reuses section, marking specific reuse locations as beta.
Beta notices should not have newlines.

```
reusable:
top_level: true
expected:
- at: user
as: target
beta: Reusing these fields in this location is currently considered beta.
```

### List of fields

Array of YAML objects:
Expand Down Expand Up @@ -134,6 +148,7 @@ Supported keys to describe fields
- format: Field format that can be used in a Kibana index template.
- normalize: Normalization steps that should be applied at ingestion time. Supported values:
- array: the content of the field should be an array (even when there's only one value).
- beta (optional): Adds a beta marker for the field to the description. The text provided in this attribute is used as content of the beta marker in the documentation. Note that when a whole field set is marked as beta, it is not necessary nor recommended to mark all fields in the field set as beta. Beta notices should not have newlines.

Supported keys to describe expected values for a field

Expand Down
3 changes: 2 additions & 1 deletion scripts/generators/asciidoc_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ def render_nestings_reuse_section(fieldset):
rows.append({
'flat_nesting': "{}.*".format(reused_here_entry['full']),
'name': reused_here_entry['schema_name'],
'short': reused_here_entry['short']
'short': reused_here_entry['short'],
'beta': reused_here_entry.get('beta', '')
})

return sorted(rows, key=lambda x: x['flat_nesting'])
Expand Down
4 changes: 2 additions & 2 deletions scripts/generators/ecs_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,5 +189,5 @@ def strict_warning(msg):
:param msg: custom text which will be displayed with wrapped boilerplate
for strict warning messages.
"""
warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode."
warnings.warn(warn_message)
warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode.\nWarning check:"
warnings.warn(warn_message, stacklevel=3)
14 changes: 14 additions & 0 deletions scripts/schema/cleaner.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ def schema_mandatory_attributes(schema):
def schema_assertions_and_warnings(schema):
'''Additional checks on a fleshed out schema'''
single_line_short_description(schema, strict=strict_mode)
if 'beta' in schema['field_details']:
single_line_beta_description(schema, strict=strict_mode)


def normalize_reuse_notation(schema):
Expand Down Expand Up @@ -181,6 +183,8 @@ def field_assertions_and_warnings(field):
# check short description length if in strict mode
single_line_short_description(field, strict=strict_mode)
check_example_value(field, strict=strict_mode)
if 'beta' in field['field_details']:
single_line_beta_description(field, strict=strict_mode)
if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:
msg = "Invalid level for field '{}'.\nValue: {}\nAcceptable values: {}".format(
field['field_details']['name'], field['field_details']['level'],
Expand Down Expand Up @@ -220,3 +224,13 @@ def check_example_value(field, strict=True):
raise ValueError(msg)
else:
ecs_helpers.strict_warning(msg)


def single_line_beta_description(schema_or_field, strict=True):
if "\n" in schema_or_field['field_details']['beta']:
msg = "Beta descriptions must be single line.\n"
msg += f"Offending field or field set: {schema_or_field['field_details']['name']}"
if strict:
raise ValueError(msg)
else:
ecs_helpers.strict_warning(msg)
3 changes: 3 additions & 0 deletions scripts/schema/finalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,9 @@ def append_reused_here(reused_schema, reuse_entry, destination_schema):
'full': reuse_entry['full'],
'short': reused_schema['field_details']['short'],
}
# Check for beta attribute
if 'beta' in reuse_entry:
reused_here_entry['beta'] = reuse_entry['beta']
destination_schema['schema_details']['reused_here'].extend([reused_here_entry])


Expand Down
22 changes: 22 additions & 0 deletions scripts/templates/field_details.j2
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ Find additional usage and examples in the {{ fieldset['name'] }} fields <<ecs-{{

{% endif %}

{# Fieldset label beta header -#}
{% if fieldset['beta'] -%}

beta::[ {{ fieldset['beta'] }}]

{% endif -%}

{# Field Details Table Header -#}
[discrete]
==== {{ fieldset['title'] }} Field Details
Expand All @@ -27,7 +34,14 @@ Find additional usage and examples in the {{ fieldset['name'] }} fields <<ecs-{{
{# `Field` column -#}
| {{ field['flat_name'] }}
{# `Description` column -#}
{#- Beta fields will add the `beta` label -#}
{% if field['beta'] -%}
| beta:[ {{ field['beta'] }} ]

{{ field['description']|replace("\n", "\n\n") }}
{%- else -%}
| {{ field['description']|replace("\n", "\n\n") }}
{%- endif %}

type: {{ field['type'] }}

Expand Down Expand Up @@ -108,8 +122,16 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d

{% for entry in render_nestings_reuse_section -%}

{#- Beta marker on nested fields -#}
| <<ecs-{{ entry['name'] }},{{ entry['flat_nesting'] }}>>
{#- Beta marker on nested fields -#}
{%- if entry['beta'] -%}
| beta:[ {{ entry['beta'] }}]

{{ entry['short'] }}
{%- else %}
| {{ entry['short'] }}
{%- endif %}

// ===============================================================

Expand Down
4 changes: 2 additions & 2 deletions scripts/tests/unit/test_schema_finalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def schema_user(self):
'order': 2,
'expected': [
{'full': 'server.user', 'at': 'server', 'as': 'user'},
{'full': 'user.target', 'at': 'user', 'as': 'target'},
{'full': 'user.target', 'at': 'user', 'as': 'target', 'beta': 'Some beta notice'},
{'full': 'user.effective', 'at': 'user', 'as': 'effective'},
]
}
Expand Down Expand Up @@ -211,7 +211,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
fields['process']['schema_details']['reused_here'])
self.assertIn({'full': 'user.effective', 'schema_name': 'user', 'short': 'short desc'},
fields['user']['schema_details']['reused_here'])
self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc'},
self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc', 'beta': 'Some beta notice'},
fields['user']['schema_details']['reused_here'])
self.assertIn({'full': 'server.user', 'schema_name': 'user', 'short': 'short desc'},
fields['server']['schema_details']['reused_here'])
Expand Down