WIP

nielsbasjes · Apr 15, 2021 · 1555fb0 · 1555fb0
1 parent 02d6512
commit 1555fb0
Show file tree

Hide file tree

Showing 25 changed files with 154 additions and 46 deletions.
diff --git a/analyzer-it/pom.xml b/analyzer-it/pom.xml
@@ -88,12 +88,26 @@
         </executions>
       </plugin>
 
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-install-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>default-install</id>
+            <phase>none</phase>
+          </execution>
+        </executions>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-deploy-plugin</artifactId>
-        <configuration>
-          <skip>true</skip>
-        </configuration>
+        <executions>
+          <execution>
+            <id>default-deploy</id>
+            <phase>none</phase>
+          </execution>
+        </executions>
       </plugin>
 
       <plugin>

diff --git a/analyzer-it/serialization/base-tests/pom.xml b/analyzer-it/serialization/base-tests/pom.xml
@@ -39,6 +39,12 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+
 <!--    <dependency>-->
 <!--      <groupId>org.slf4j</groupId>-->
 <!--      <artifactId>jcl-over-slf4j</artifactId>-->

diff --git a/analyzer-it/serialization/base-tests/src/test/resources/log4j2.properties b/analyzer-it/serialization/base-tests/src/test/resources/log4j2.properties
@@ -0,0 +1,35 @@
+#
+# Yet Another UserAgent Analyzer
+# Copyright (C) 2013-2021 Niels Basjes
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+status = error
+name = PropertiesConfig
+
+filters = threshold
+
+filter.threshold.type = ThresholdFilter
+filter.threshold.level = info
+
+appenders = console
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ABSOLUTE} [%-5p] %-40c{1}:%5L: %m%n
+
+rootLogger.level = debug
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
diff --git a/analyzer-it/serialization/pom.xml b/analyzer-it/serialization/pom.xml
@@ -30,6 +30,22 @@
   <artifactId>yauaa-serialization-tests-parent</artifactId>
   <name>Yauaa : IT : Serialization : </name>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <!-- Since these serialization tests do not create any normal jar we disable 'default-jar' -->
+            <id>default-jar</id>
+            <phase>none</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
   <modules>
     <module>base-tests</module>
     <module>java-default</module>

diff --git a/analyzer/pom.xml b/analyzer/pom.xml
@@ -94,6 +94,12 @@
       <version>${log4j2.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-jcl</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
@@ -140,6 +146,12 @@
           <minimizeJar>true</minimizeJar>
           <createDependencyReducedPom>true</createDependencyReducedPom>
           <filters>
+            <filter>
+              <artifact>*:*</artifact>
+              <excludes>
+                <exclude>META-INF/MANIFEST.MF</exclude>
+              </excludes>
+            </filter>
             <filter>
               <artifact>org.springframework:spring-core</artifact>
               <excludes>

diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java b/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java
@@ -26,10 +26,10 @@
 import nl.basjes.parse.useragent.UserAgentAnalyzerDirect;
 import nl.basjes.parse.useragent.analyze.Matcher;
 import nl.basjes.parse.useragent.analyze.MatchesList.Match;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.message.MessageFormatMessage;
-//import org.slf4j.helpers.MessageFormatter;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.MessageFactory;
+import org.apache.logging.log4j.message.ReusableMessageFactory;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -74,11 +74,13 @@ static class TestResult {
         long confidence;
     }
 
+    private static MessageFactory messageFactory = new ReusableMessageFactory();
+
     private static String determineLogMessage(String format, Object... args){
         if (args == null || args.length == 0) {
             return format;
         }
-        return new MessageFormatMessage(format, args).getFormattedMessage();
+        return messageFactory.newMessage(format, args).getFormattedMessage();
     }
 
     private static void logInfo(StringBuilder errorMessageReceiver, String format, Object... args) {

diff --git a/analyzer/src/test/resources/log4j2.properties b/analyzer/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
@@ -164,6 +164,7 @@
                   -->
                   <artifact>*:*</artifact>
                   <excludes>
+                    <exclude>META-INF/MANIFEST.MF</exclude>
                     <exclude>META-INF/*.SF</exclude>
                     <exclude>META-INF/*.DSA</exclude>
                     <exclude>META-INF/*.RSA</exclude>

diff --git a/commandline/src/main/resources/log4j2.properties b/commandline/src/main/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/pom.xml b/pom.xml
@@ -81,6 +81,8 @@
     <elastic.version>7.12.0</elastic.version>
     <logstash.version>${elastic.version}</logstash.version>
     <elasticsearch.version>${elastic.version}</elasticsearch.version>
+    <!-- This is the log4j2 version used in ES -->
+    <elasticsearch.log4j2.version>2.11.1</elasticsearch.log4j2.version>
 
     <!-- ============================================== -->
 

diff --git a/udfs/beam-sql/src/test/resources/log4j2.properties b/udfs/beam-sql/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/udfs/beam/src/test/resources/log4j2.properties b/udfs/beam/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/udfs/drill/tests/pom.xml b/udfs/drill/tests/pom.xml
@@ -38,6 +38,12 @@
       <groupId>nl.basjes.parse.useragent</groupId>
       <artifactId>yauaa-drill</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!--TEST DEPENDENCIES-->
@@ -60,6 +66,10 @@
           <groupId>org.slf4j</groupId>
           <artifactId>log4j-over-slf4j</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -77,6 +87,11 @@
           <groupId>org.slf4j</groupId>
           <artifactId>log4j-over-slf4j</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+
       </exclusions>
     </dependency>
 
@@ -95,6 +110,11 @@
           <groupId>org.slf4j</groupId>
           <artifactId>log4j-over-slf4j</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+
       </exclusions>
     </dependency>
 
@@ -113,6 +133,10 @@
           <groupId>org.slf4j</groupId>
           <artifactId>log4j-over-slf4j</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
 

diff --git a/udfs/drill/tests/src/test/resources/log4j2.properties b/udfs/drill/tests/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/udfs/elasticsearch/pom.xml b/udfs/elasticsearch/pom.xml
@@ -36,21 +36,18 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
+    <!-- We disable the depencency-convergence check because it is too messy. -->
+<!--    <depencency-convergence.phase>none</depencency-convergence.phase>-->
   </properties>
 
   <dependencies>
 
     <dependency>
       <groupId>nl.basjes.parse.useragent</groupId>
       <artifactId>yauaa</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.elasticsearch</groupId>
-      <artifactId>elasticsearch</artifactId>
-      <version>${elasticsearch.version}</version>
-      <scope>provided</scope>
       <exclusions>
+        <!-- To avoid the "jar hell!" failure when loading in ES. -->
         <exclusion>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-api</artifactId>
@@ -60,22 +57,22 @@
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
+      <artifactId>log4j-api</artifactId>
       <version>${log4j2.version}</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-api</artifactId>
-        </exclusion>
-      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.elasticsearch</groupId>
+      <artifactId>elasticsearch</artifactId>
+      <version>${elasticsearch.version}</version>
+      <scope>provided</scope>
     </dependency>
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-api</artifactId>
-      <version>${log4j2.version}</version>
-      <scope>test</scope>
+      <artifactId>log4j-jcl</artifactId>
+      <version>${elasticsearch.log4j2.version}</version>
     </dependency>
 
     <dependency>
@@ -85,7 +82,6 @@
       <scope>test</scope>
     </dependency>
 
-
     <dependency>
       <groupId>org.junit.vintage</groupId>
       <artifactId>junit-vintage-engine</artifactId>

diff --git a/udfs/flink-table/src/test/resources/log4j2.properties b/udfs/flink-table/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/udfs/flink/src/test/resources/log4j2.properties b/udfs/flink/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console
 

diff --git a/udfs/hive/pom.xml b/udfs/hive/pom.xml
@@ -84,16 +84,16 @@
       <scope>provided</scope>
       <exclusions>
         <exclusion>
-          <artifactId>log4j-slf4j-impl</artifactId>
           <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j-impl</artifactId>
         </exclusion>
         <exclusion>
-          <artifactId>log4j</artifactId>
           <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
         </exclusion>
         <exclusion>
-          <artifactId>log4j-1.2-api</artifactId>
           <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-1.2-api</artifactId>
         </exclusion>
       </exclusions>
     </dependency>

diff --git a/udfs/hive/src/test/resources/log4j2.properties b/udfs/hive/src/test/resources/log4j2.properties
@@ -21,7 +21,7 @@ name = PropertiesConfig
 filters = threshold
 
 filter.threshold.type = ThresholdFilter
-filter.threshold.level = debug
+filter.threshold.level = info
 
 appenders = console