From db0ba3497cd687b1c7efca884020a0c0a4a423c9 Mon Sep 17 00:00:00 2001 From: Laura Schanno Date: Tue, 9 Apr 2024 13:59:32 -0400 Subject: [PATCH] Allow multiple fields via f:noExpansion The #NOEXPANSION function should support specifying multiple fields. Update the NoExpansion JEXL function implementation to allow multiple fields. Fixes #2341 --- .../language/functions/jexl/NoExpansion.java | 29 ++++++--- .../functions/jexl/NoExpansionTest.java | 63 +++++++++++++++++++ .../parser/jexl/TestLuceneToJexlParser.java | 5 +- .../query/transformer/NoExpansionTests.java | 29 ++++++++- .../VisibilityWiseGuysIngestWithModel.java | 24 +++++++ 5 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 warehouse/query-core/src/test/java/datawave/query/language/functions/jexl/NoExpansionTest.java diff --git a/warehouse/query-core/src/main/java/datawave/query/language/functions/jexl/NoExpansion.java b/warehouse/query-core/src/main/java/datawave/query/language/functions/jexl/NoExpansion.java index d601d49b8c9..3a06f4ac411 100644 --- a/warehouse/query-core/src/main/java/datawave/query/language/functions/jexl/NoExpansion.java +++ b/warehouse/query-core/src/main/java/datawave/query/language/functions/jexl/NoExpansion.java @@ -4,26 +4,27 @@ import java.util.ArrayList; import java.util.List; +import datawave.query.jexl.functions.QueryFunctions; import datawave.query.language.functions.QueryFunction; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; /** * This function accepts a comma separated list of fields to be excluded from QueryModel expansion. The purpose is to provide users with an easy way to avoid - * undesirable query model expansions. - * - * Note: The exclude is only applied to the fields in the original query. An original field can be expanded into an excluded field. + * undesirable query model expansions.
+ * Note: The exclusion is only applied to the fields in the original query. An original field can be expanded into an excluded field. */ public class NoExpansion extends JexlQueryFunction { public NoExpansion() { - super("noExpansion", new ArrayList<>()); + super(QueryFunctions.NO_EXPANSION, new ArrayList<>()); } @Override public void validate() throws IllegalArgumentException { - if (this.parameterList.size() != 1) { - BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INVALID_FUNCTION_ARGUMENTS, MessageFormat.format("{0}", this.name)); + if (this.parameterList.isEmpty()) { + BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INVALID_FUNCTION_ARGUMENTS, + MessageFormat.format("{0} requires at least one argument", this.name)); throw new IllegalArgumentException(qe); } } @@ -35,7 +36,19 @@ public QueryFunction duplicate() { @Override public String toString() { - List params = getParameterList(); - return "f:noExpansion(" + String.join("", params) + ")"; + StringBuilder sb = new StringBuilder(); + + sb.append(QueryFunctions.QUERY_FUNCTION_NAMESPACE).append(':').append(QueryFunctions.NO_EXPANSION); + if (parameterList.isEmpty()) { + sb.append("()"); + } else { + char separator = '('; + for (String param : parameterList) { + sb.append(separator).append(escapeString(param)); + separator = ','; + } + sb.append(')'); + } + return sb.toString(); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/language/functions/jexl/NoExpansionTest.java b/warehouse/query-core/src/test/java/datawave/query/language/functions/jexl/NoExpansionTest.java new file mode 100644 index 00000000000..06dd0ff76ee --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/language/functions/jexl/NoExpansionTest.java @@ -0,0 +1,63 @@ +package datawave.query.language.functions.jexl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.util.List; + +import org.junit.Test; + +public class NoExpansionTest { + + /** + * Verify that {@link NoExpansion#validate()} throws an exception given an empty parameter list. + */ + @Test + public void testValidateWithEmptyParameters() { + NoExpansion noExpansion = new NoExpansion(); + noExpansion.setParameterList(List.of()); + Exception exception = assertThrows(IllegalArgumentException.class, noExpansion::validate); + assertEquals("datawave.webservice.query.exception.BadRequestQueryException: Invalid arguments to function. noExpansion requires at least one argument", + exception.getMessage()); + } + + /** + * Verify that {@link NoExpansion#validate()} does not throw an error for a single parameter. + */ + @Test + public void testValidateWithOneField() { + NoExpansion noExpansion = new NoExpansion(); + noExpansion.setParameterList(List.of("field1")); + noExpansion.validate(); + } + + /** + * Verify that {@link NoExpansion#validate()} does not throw an error for multiple parameters. + */ + @Test + public void testValidateWithMultipleFields() { + NoExpansion noExpansion = new NoExpansion(); + noExpansion.setParameterList(List.of("field1", "field2", "field3")); + noExpansion.validate(); + } + + @Test + public void testToStringWithNoParameters() { + NoExpansion noExpansion = new NoExpansion(); + assertEquals("f:noExpansion()", noExpansion.toString()); + } + + @Test + public void testToStringWithOneParameter() { + NoExpansion noExpansion = new NoExpansion(); + noExpansion.setParameterList(List.of("field1")); + assertEquals("f:noExpansion('field1')", noExpansion.toString()); + } + + @Test + public void testToStringWithMultipleParameter() { + NoExpansion noExpansion = new NoExpansion(); + noExpansion.setParameterList(List.of("field1", "field2", "field3")); + assertEquals("f:noExpansion('field1','field2','field3')", noExpansion.toString()); + } +} diff --git a/warehouse/query-core/src/test/java/datawave/query/language/parser/jexl/TestLuceneToJexlParser.java b/warehouse/query-core/src/test/java/datawave/query/language/parser/jexl/TestLuceneToJexlParser.java index 4cce7c7bcdd..65d8c5111a9 100644 --- a/warehouse/query-core/src/test/java/datawave/query/language/parser/jexl/TestLuceneToJexlParser.java +++ b/warehouse/query-core/src/test/java/datawave/query/language/parser/jexl/TestLuceneToJexlParser.java @@ -57,7 +57,10 @@ public void test1() throws Exception { public void testParseFunction_NoExpansion() throws ParseException { LuceneToJexlQueryParser parser = getQueryParser(); QueryNode node = parser.parse("FIELD:SOMETHING AND #NOEXPANSION(FIELD)"); - Assert.assertEquals("FIELD == 'SOMETHING' && f:noExpansion(FIELD)", node.getOriginalQuery()); + Assert.assertEquals("FIELD == 'SOMETHING' && f:noExpansion('FIELD')", node.getOriginalQuery()); + + node = parser.parse("FIELD:SOMETHING AND #NOEXPANSION(FIELD1,FIELD2)"); + Assert.assertEquals("FIELD == 'SOMETHING' && f:noExpansion('FIELD1','FIELD2')", node.getOriginalQuery()); } @Test diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java b/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java index da5fb7abfd1..7c8f5c69ba6 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java @@ -149,7 +149,7 @@ private void runTestQuery() throws Exception { // order of terms in planned script is arbitrary, fall back to comparing the jexl trees ASTJexlScript plannedScript = JexlASTHelper.parseJexlQuery(plan); ASTJexlScript expectedScript = JexlASTHelper.parseJexlQuery(this.expectedPlan); - JexlNodeAssert.assertThat(expectedScript).isEqualTo(plannedScript); + JexlNodeAssert.assertThat(plannedScript).isEqualTo(expectedScript); } private AccumuloClient createClient() throws Exception { @@ -179,8 +179,8 @@ private void givenExpectedPlan(String expectedPlan) { */ @Test public void testDefaultQueryModelExpansion() throws Exception { - givenQuery("COLOR == 'blue'"); - givenExpectedPlan("(COLOR == 'blue' || HUE == 'blue')"); + givenQuery("COLOR == 'blue' && FASTENER == 'bolt'"); + givenExpectedPlan("(COLOR == 'blue' || HUE == 'blue') && (FASTENER == 'bolt' || FIXTURE == 'bolt')"); runTestQuery(); } @@ -196,6 +196,17 @@ public void testNoExpansionViaFunction() throws Exception { runTestQuery(); } + /** + * Verify that when #NO_EXPANSION is specified in the query string itself with multiple fields, expansion does not occur. + */ + @Test + public void testNoExpansionViaFunctionWithMultipleFields() throws Exception { + givenQuery("COLOR == 'blue' && FASTENER == 'bolt' && f:noExpansion(COLOR,FASTENER)"); + givenExpectedPlan("COLOR == 'blue' && FASTENER == 'bolt'"); + + runTestQuery(); + } + /** * Verify that when #NO_EXPANSION is specified via the query parameters, expansion does not occur. */ @@ -208,6 +219,18 @@ public void testNoExpansionViaQueryParameters() throws Exception { runTestQuery(); } + /** + * Verify that when #NO_EXPANSION is specified via the query parameters, expansion does not occur. + */ + @Test + public void testNoExpansionViaQueryParametersWithMultipleFields() throws Exception { + givenQuery("COLOR == 'blue' && FASTENER == 'bolt'"); + givenQueryParameter(QueryParameters.NO_EXPANSION_FIELDS, "COLOR,FASTENER"); + givenExpectedPlan("COLOR == 'blue' && FASTENER == 'bolt'"); + + runTestQuery(); + } + /** * Verify that when #NO_EXPANSION is specified in the query string itself and in query parameters, expansion does not occur. */ diff --git a/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java b/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java index d38f646e984..0e0170ac4e7 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/VisibilityWiseGuysIngestWithModel.java @@ -776,6 +776,24 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); bw.addMutation(mutation); + // for testing #NOEXPANSION function + mutation = new Mutation("FASTENER"); + mutation.put(ColumnFamilyConstants.COLF_E, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(10L))); + mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); + bw.addMutation(mutation); + + // for testing #NOEXPANSION function + mutation = new Mutation("FIXTURE"); + mutation.put(ColumnFamilyConstants.COLF_E, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(10L))); + mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); + mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); + bw.addMutation(mutation); + } finally { if (null != bw) { bw.close(); @@ -822,6 +840,12 @@ public static void writeItAll(AccumuloClient client, WhatKindaRange range) throw mutation.put("DATAWAVE", "HUE" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); bw.addMutation(mutation); + // specifically for testing the #NOEXPANSION function + mutation = new Mutation("FASTENER"); + mutation.put("DATAWAVE", "FASTENER" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); + mutation.put("DATAWAVE", "FIXTURE" + "\u0000" + "forward", columnVisibility, timeStamp, emptyValue); + bw.addMutation(mutation); + } finally { if (null != bw) { bw.close();