From 99d2f87b03a3ea8f00cf62a1fc5f98016baec56e Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Fri, 12 Jun 2020 17:02:20 +0200 Subject: [PATCH] Fix building and testing ElasticSearch plugin --- .gitignore | 1 + pom.xml | 13 ++++ src/main/docs/README-Usage.md | 6 +- udfs/elasticsearch/pom.xml | 36 +++++++--- .../plugin/ingest/yauaa/YauaaProcessor.java | 5 -- ...ssorTests.java => YauaaProcessorTest.java} | 71 ++++++++++++++++++- udfs/pom.xml | 13 ++++ 7 files changed, 127 insertions(+), 18 deletions(-) rename udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/{YauaaProcessorTests.java => YauaaProcessorTest.java} (66%) diff --git a/.gitignore b/.gitignore index 1d7b04db8b..5840e53e38 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ bin #IntelliJ stuff .idea *.iml +.attach_pid* #test files x diff --git a/pom.xml b/pom.xml index dff2c1f1eb..385501be69 100644 --- a/pom.xml +++ b/pom.xml @@ -155,9 +155,16 @@ + org.junit.jupiter junit-jupiter-engine + + + org.hamcrest + hamcrest-core + + test @@ -498,6 +505,11 @@ maven-surefire-plugin + + org.apache.maven.plugins + maven-failsafe-plugin + + org.apache.maven.plugins maven-deploy-plugin @@ -803,6 +815,7 @@ maven-site-plugin3.8.2 maven-source-plugin3.2.1 maven-surefire-plugin3.0.0-M4 + maven-failsafe-plugin3.0.0-M4 maven-war-plugin3.2.3 diff --git a/src/main/docs/README-Usage.md b/src/main/docs/README-Usage.md index d21a4ad0b3..18baa4aabd 100644 --- a/src/main/docs/README-Usage.md +++ b/src/main/docs/README-Usage.md @@ -102,13 +102,13 @@ Essentially you can redirect all JCL logging into SLF4J and then into the actual org.slf4j jcl-over-slf4j - 1.7.29 + 1.7.30 org.slf4j slf4j-simple - 1.7.29 + 1.7.30 or (if you really have to) can redirect all SLF4J into JCL by adding something like this @@ -116,7 +116,7 @@ or (if you really have to) can redirect all SLF4J into JCL by adding something l org.slf4j slf4j-jcl - 1.7.29 + 1.7.30 commons-logging diff --git a/udfs/elasticsearch/pom.xml b/udfs/elasticsearch/pom.xml index 8b0c0d17d0..9c54c65663 100644 --- a/udfs/elasticsearch/pom.xml +++ b/udfs/elasticsearch/pom.xml @@ -45,12 +45,6 @@ yauaa - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - - org.elasticsearch elasticsearch @@ -65,6 +59,10 @@ commons-logging commons-logging + + org.apache.logging.log4j + log4j-api + @@ -90,6 +88,14 @@ commons-logging commons-logging + + commons-codec + commons-codec + + + org.hamcrest + hamcrest + @@ -98,6 +104,12 @@ log4j-slf4j-impl ${log4j2.version} test + + + org.slf4j + slf4j-api + + @@ -114,10 +126,16 @@ test + - junit - junit - 4.13 + org.junit.vintage + junit-vintage-engine + + + org.hamcrest + hamcrest-core + + test diff --git a/udfs/elasticsearch/src/main/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessor.java b/udfs/elasticsearch/src/main/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessor.java index ccf1b42d60..163748cc74 100644 --- a/udfs/elasticsearch/src/main/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessor.java +++ b/udfs/elasticsearch/src/main/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessor.java @@ -23,8 +23,6 @@ import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.List; @@ -78,9 +76,6 @@ public YauaaProcessor(String tag, } this.uaa = builder.build(); - - Logger log = LoggerFactory.getLogger("PROCESSOR"); - log.error("{}", uaa); } @Override diff --git a/udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTests.java b/udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTest.java similarity index 66% rename from udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTests.java rename to udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTest.java index a6546e07db..0b86e43d35 100644 --- a/udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTests.java +++ b/udfs/elasticsearch/src/test/java/org/elasticsearch/plugin/ingest/yauaa/YauaaProcessorTest.java @@ -18,9 +18,12 @@ package org.elasticsearch.plugin.ingest.yauaa; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.RandomDocumentPicks; import org.hamcrest.MatcherAssert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.HashMap; @@ -33,7 +36,8 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; -public class YauaaProcessorTests { +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +public class YauaaProcessorTest { private static final String SOURCE_FIELD = "source_field"; private static final String TARGET_FIELD = "target_field"; @@ -139,4 +143,69 @@ private void assertHasNotKey(Map results, String key) { MatcherAssert.assertThat(results, not(hasKey(key))); } + @SuppressWarnings("unchecked") + @Test + public void testIngestPlugin() throws Exception { + IngestYauaaPlugin plugin = new IngestYauaaPlugin(); + + Map processors = plugin.getProcessors(null); + + Processor.Factory yauaaFactory = processors.get("yauaa"); + + Map configuration = new HashMap<>(); + + configuration.put("field", SOURCE_FIELD); + configuration.put("target_field", TARGET_FIELD); + configuration.put("fieldNames", Arrays.asList("DeviceClass", "DeviceBrand", "DeviceName", "AgentNameVersionMajor", "FirstProductName")); + configuration.put("cacheSize", 10); + configuration.put("preheat", 10); + configuration.put("extraRules", "config:\n- matcher:\n extract:\n - 'FirstProductName : 1 :agent.(1)product.(1)name'\n"); + + Processor processor = yauaaFactory.create(processors, randomAlphaOfLength(10), configuration); + + Map document = new HashMap<>(); + document.put(SOURCE_FIELD, + "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " + + "AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/53.0.2785.124 Mobile Safari/537.36"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + Map data = processor.execute(ingestDocument).getSourceAndMetadata(); + + MatcherAssert.assertThat(data, hasKey(TARGET_FIELD)); + + Map results = (Map) data.get(TARGET_FIELD); + + // The EXPLICITLY requested fields + assertHasKValue(results, "FirstProductName", "Mozilla"); + assertHasKValue(results, "DeviceClass", "Phone"); + assertHasKValue(results, "DeviceBrand", "Google"); + assertHasKValue(results, "DeviceName", "Google Nexus 6"); + assertHasKValue(results, "AgentNameVersionMajor", "Chrome 53"); + + // The IMPLICITLY requested fields (i.e. partials of the actually requested ones) + assertHasKValue(results, "AgentName", "Chrome"); + assertHasKValue(results, "AgentVersion", "53.0.2785.124"); + assertHasKValue(results, "AgentVersionMajor", "53"); + + // The NOT requested fields + assertHasNotKey(results, "OperatingSystemClass"); + assertHasNotKey(results, "OperatingSystemName"); + assertHasNotKey(results, "OperatingSystemNameVersion"); + assertHasNotKey(results, "OperatingSystemNameVersionMajor"); + assertHasNotKey(results, "OperatingSystemVersion"); + assertHasNotKey(results, "OperatingSystemVersionBuild"); + assertHasNotKey(results, "OperatingSystemVersionMajor"); + assertHasNotKey(results, "LayoutEngineClass"); + assertHasNotKey(results, "LayoutEngineName"); + assertHasNotKey(results, "LayoutEngineNameVersion"); + assertHasNotKey(results, "LayoutEngineNameVersionMajor"); + assertHasNotKey(results, "LayoutEngineVersion"); + assertHasNotKey(results, "LayoutEngineVersionMajor"); + assertHasNotKey(results, "AgentClass"); + assertHasNotKey(results, "AgentNameVersion"); + + LoggerFactory.getLogger("TestYauaaProcessor").info("Complete set of returned results:{}", results); + } + } diff --git a/udfs/pom.xml b/udfs/pom.xml index c3494950cf..5aa0402ef0 100644 --- a/udfs/pom.xml +++ b/udfs/pom.xml @@ -70,7 +70,20 @@ spring-core ${spring.version} provided + + + org.springframework + spring-jcl + + + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + org.yaml snakeyaml