-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
New pipeline functions
remove_single_field
and `remove_multiple_fie…
…lds` (#19268) * new pipeline functions remove_single_field and remove_multiple_fields * CL * improve rulebuilder descriptions * compile regex and other feedback changes * pre-compile pattern * UI string improvements
- Loading branch information
1 parent
f5e0d0b
commit 4a1cbb4
Showing
9 changed files
with
262 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
type="a" | ||
message="Introduce new pipeline functions `remove_single_field` and `remove_multiple_fields` to (eventually) replace `remove_field`." | ||
|
||
details.user=""" | ||
GL 5.1 added regex-matching to the pipeline function `remove_field`. This breaks existing pipeline rules that call | ||
`remove_field` with a field name containing a regex reserved character, notably `.`. Performance of existing rules | ||
may also be degraded. | ||
Both issues are addressed by introducing alternate, more specific functions: | ||
`remove_single_field` removes just a single field specified by name. It is simple and fast. | ||
`remove_multiple_fields` removes fields matching a regex pattern and/or list of names. Depending on the | ||
complexity of the matching it is slower. | ||
'remove_field' will be deprecated and removed in the next major version. Do not use it. | ||
""" | ||
|
||
issues=["19098"] | ||
pulls=["19268"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
...n/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveMultipleFields.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
/* | ||
* Copyright (C) 2020 Graylog, Inc. | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the Server Side Public License, version 1, | ||
* as published by MongoDB, Inc. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* Server Side Public License for more details. | ||
* | ||
* You should have received a copy of the Server Side Public License | ||
* along with this program. If not, see | ||
* <http://www.mongodb.com/licensing/server-side-public-license>. | ||
*/ | ||
package org.graylog.plugins.pipelineprocessor.functions.messages; | ||
|
||
import com.google.common.collect.ImmutableList; | ||
import org.graylog.plugins.pipelineprocessor.EvaluationContext; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; | ||
import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup; | ||
import org.graylog2.plugin.Message; | ||
|
||
import java.util.List; | ||
import java.util.regex.Pattern; | ||
|
||
import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; | ||
|
||
public class RemoveMultipleFields extends AbstractFunction<Void> { | ||
public static final String NAME = "remove_multiple_fields"; | ||
private static final String REGEX_PATTERN = "pattern"; | ||
private static final String LIST_OF_NAMES = "names"; | ||
private final ParameterDescriptor<String, Pattern> regexParam; | ||
private final ParameterDescriptor<List, List> namesParam; | ||
private final ParameterDescriptor<Message, Message> messageParam; | ||
|
||
public RemoveMultipleFields() { | ||
regexParam = ParameterDescriptor.string(REGEX_PATTERN, Pattern.class) | ||
.optional() | ||
.transform(Pattern::compile) | ||
.description("A regex specifying field names to be removed").build(); | ||
namesParam = type(LIST_OF_NAMES, List.class).optional().description("A list of field names to be removed").build(); | ||
messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); | ||
} | ||
|
||
@Override | ||
public Void evaluate(FunctionArgs args, EvaluationContext context) { | ||
final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); | ||
if (regexParam.optional(args, context).isPresent()) { | ||
removeRegex(message, regexParam.optional(args, context).get()); | ||
} | ||
if (namesParam.optional(args, context).isPresent()) { | ||
removeNames(message, namesParam.optional(args, context).get()); | ||
} | ||
return null; | ||
} | ||
|
||
private void removeRegex(Message message, Pattern pattern) { | ||
message.getFieldNames().stream() | ||
.filter(name -> pattern.matcher(name).matches()) | ||
.toList() // required to avoid ConcurrentModificationException | ||
.forEach(message::removeField); | ||
} | ||
|
||
private void removeNames(Message message, List names) { | ||
for (Object name : names) { | ||
message.removeField(String.valueOf(name)); | ||
} | ||
} | ||
|
||
@Override | ||
public FunctionDescriptor<Void> descriptor() { | ||
return FunctionDescriptor.<Void>builder() | ||
.name(NAME) | ||
.returnType(Void.class) | ||
.params(ImmutableList.of(regexParam, namesParam, messageParam)) | ||
.description("Removes the specified field(s) from message, unless the field name is reserved. If no specific message is provided, it uses the currently processed message.") | ||
.ruleBuilderEnabled() | ||
.ruleBuilderName("Remove field - multiple") | ||
.ruleBuilderTitle("Remove multiple fields by<#if pattern??> regex '${pattern}'</#if><#if pattern?? && names??> or</#if><#if names??> name list '${names}'</#if>") | ||
.ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.MESSAGE) | ||
.build(); | ||
} | ||
} |
63 changes: 63 additions & 0 deletions
63
...main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveSingleField.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
/* | ||
* Copyright (C) 2020 Graylog, Inc. | ||
* | ||
* This program is free software: you can redistribute it and/or modify | ||
* it under the terms of the Server Side Public License, version 1, | ||
* as published by MongoDB, Inc. | ||
* | ||
* This program is distributed in the hope that it will be useful, | ||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
* Server Side Public License for more details. | ||
* | ||
* You should have received a copy of the Server Side Public License | ||
* along with this program. If not, see | ||
* <http://www.mongodb.com/licensing/server-side-public-license>. | ||
*/ | ||
package org.graylog.plugins.pipelineprocessor.functions.messages; | ||
|
||
import com.google.common.collect.ImmutableList; | ||
import org.graylog.plugins.pipelineprocessor.EvaluationContext; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; | ||
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; | ||
import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup; | ||
import org.graylog2.plugin.Message; | ||
|
||
import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; | ||
|
||
public class RemoveSingleField extends AbstractFunction<Void> { | ||
public static final String NAME = "remove_single_field"; | ||
public static final String FIELD = "field"; | ||
private final ParameterDescriptor<String, String> fieldParam; | ||
private final ParameterDescriptor<Message, Message> messageParam; | ||
|
||
public RemoveSingleField() { | ||
fieldParam = ParameterDescriptor.string(FIELD).description("The field to remove").build(); | ||
messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); | ||
} | ||
|
||
@Override | ||
public Void evaluate(FunctionArgs args, EvaluationContext context) { | ||
final String field = fieldParam.required(args, context); | ||
final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); | ||
|
||
message.removeField(field); | ||
return null; | ||
} | ||
|
||
@Override | ||
public FunctionDescriptor<Void> descriptor() { | ||
return FunctionDescriptor.<Void>builder() | ||
.name(NAME) | ||
.returnType(Void.class) | ||
.params(ImmutableList.of(fieldParam, messageParam)) | ||
.description("Removes a field from a message") | ||
.ruleBuilderEnabled() | ||
.ruleBuilderName("Remove field - single") | ||
.ruleBuilderTitle("Remove the single field '${field}'") | ||
.ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.MESSAGE) | ||
.build(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
15 changes: 15 additions & 0 deletions
15
...src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFieldsByName.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
rule "remove_fields_by_name" | ||
when true | ||
then | ||
set_field(field: "a.1", value: "a.1"); | ||
set_field(field: "a_1", value: "a_1"); | ||
set_field(field: "f1", value: "f1"); | ||
set_field(field: "f2", value: "f2"); | ||
|
||
remove_multiple_fields(names:["a.1", "f1"]); | ||
|
||
// invalid - should be NOOP | ||
remove_multiple_fields(names:["dummy"]); | ||
remove_multiple_fields(names:[]); | ||
remove_multiple_fields(); | ||
end |
10 changes: 10 additions & 0 deletions
10
...rc/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFieldsByRegex.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
rule "remove_fields_by_regex_and_name)" | ||
when true | ||
then | ||
set_field(field: "a.1", value: "a.1"); | ||
set_field(field: "a_1", value: "a_1"); | ||
set_field(field: "f1", value: "f1"); | ||
set_field(field: "f2", value: "f2"); | ||
|
||
remove_multiple_fields(pattern:"a.1", names:["f2"]); | ||
end |
15 changes: 15 additions & 0 deletions
15
.../src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeSingleField.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
rule "remove_single_field" | ||
when true | ||
then | ||
set_field(field: "a.1", value: "a.1"); | ||
set_field(field: "a_1", value: "a_1"); | ||
set_field(field: "f1", value: "f1"); | ||
set_field(field: "f2", value: "f2"); | ||
|
||
remove_single_field(field:"a.1"); | ||
remove_single_field(field:"f1"); | ||
|
||
// invalid - should be NO-OP | ||
remove_single_field(field:"f."); | ||
remove_single_field(field:"dummy"); | ||
end |