Replace logging infrastructure with Log4j

- Remove the existing logging infrastructure and replace it with the
  Log4j Scala API. Log4j Scala API and Log4j API are added as
  dependencies. Log4j Core is added as a test dependency and a CLI
  dependency. Users of the Daffodil API are expected to manually add
  log4j-core or some other Log4j implementation to enable logging
- Distribute a Log4j configuration with the CLI to maintain a similar
  log outut as with the previous implementation
- The Scala/Java API logging functions are deprecated and are now
  no-ops. The functions were not thread safe and could lead to random
  test failures. Logging tests are removed--we simply use the API and
  rely on Log4j testing for correctness
- We no longer have the different log levels as before, we know just
  have error, warn, info, debug, and trace
- Modify .sh and .bat files to be consistent and add a more clear way
  to set default java options that can be overridden by the user if
  needed
- Minor refactor of UDF to improve logging and allow warn+ignore
  provider loading errors

DAFFODIL-2510

daffodil-cli/bin.NOTICE

@@ -18,6 +18,35 @@ Apache Commons IO (lib/commons-io.commons-io-<VERSION>.jar)
   This product includes software developed at
   The Apache Software Foundation (https://www.apache.org/).
 
+Apache Log4j (lib/org.apache.logging.log4j.log4j-api-<VERSION>.jar, org.apache.logging.log4j.log4j-core-<VERSION>.jar)
+  Apache Log4j
+  Copyright 1999-2019 Apache Software Foundation
+
+  This product includes software developed at
+  The Apache Software Foundation (http://www.apache.org/).
+
+  ResolverUtil.java
+  Copyright 2005-2006 Tim Fennell
+
+  Dumbster SMTP test server
+  Copyright 2004 Jason Paul Kitchen
+
+  TypeUtil.java
+  Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams
+
+  picocli (http://picocli.info)
+  Copyright 2017 Remko Popma
+
+Apache Log4j Scala API (lib/org.apache.logging.log4j.log4j-api-scala_<VERSION>.jar)
+  Copyright 2016-2018 Apache Software Foundation
+
+  This product includes software developed at
+  The Apache Software Foundation (http://www.apache.org/).
+
+  AsciidocPlugin.scala
+  Copyright (c) 2008, 2009, 2010, 2011 Josh Suereth, Steven Blundy, Josh Cough,
+  Mark Harrah, Stuart Roebuck, Tony Sloane, Vesa Vilhonen, Jason Zaugg
+
 Apache Xerces Java (lib/xerces.xercesImpl-<VERSION>.jar)
   Apache Xerces Java
   Copyright 1999-2020 The Apache Software Foundation

daffodil-cli/build.sbt

@@ -39,6 +39,7 @@ Universal / mappings ++= Seq(
   baseDirectory.value / "bin.LICENSE" -> "LICENSE",
   baseDirectory.value / "bin.NOTICE" -> "NOTICE",
   baseDirectory.value / "README.md" -> "README.md",
+  sourceDirectory.value / "conf" / "log4j2.xml" -> "conf/log4j2.xml",
 )
 
 maintainer := "Apache Daffodil <dev@daffodil.apache.org>"

daffodil-cli/src/conf/log4j2.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<Configuration>
+  <Appenders>
+    <Console name="STDERR" target="SYSTEM_ERR">
+      <PatternLayout pattern="[%p{lowerCase=true}] %m%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="WARN">
+      <AppenderRef ref="STDERR"/>
+    </Root>
+  </Loggers>
+</Configuration>

daffodil-cli/src/it/scala/org/apache/daffodil/parsing/TestCLIParsing.scala

@@ -450,14 +450,10 @@ class TestCLIparsing {
     val shell = Util.start("")
 
     try {
-
       shell.sendLine(String.format("echo 0,1| %s -v parse -s %s -r matrix -", Util.binPath, testSchemaFile))
       shell.expectIn(1, contains("[info]"))
 
       shell.sendLine(String.format("echo 0,1| %s -vv parse -s %s -r matrix -", Util.binPath, testSchemaFile))
-      shell.expectIn(1, contains("[compile]"))
-
-      shell.sendLine(String.format("echo 0,1| %s -vvv parse -s %s -r matrix -", Util.binPath, testSchemaFile))
       shell.expectIn(1, contains("[debug]"))
 
       Util.expectExitCode(ExitCode.LeftOverData, shell)

daffodil-cli/src/it/scala/org/apache/daffodil/saving/TestCLISaveParser.scala

@@ -195,9 +195,6 @@ class TestCLISaveParser {
       shell.expectIn(1, contains("[info]"))
 
       shell.sendLine(String.format("%s -vv save-parser -s %s -r matrix %s", Util.binPath, testSchemaFile, savedParserFile.getName()))
-      shell.expectIn(1, contains("[compile]"))
-
-      shell.sendLine(String.format("%s -vvv save-parser -s %s -r matrix %s", Util.binPath, testSchemaFile, savedParserFile.getName()))
       shell.expectIn(1, contains("[debug]"))
 
       Util.expectExitCode(ExitCode.Success, shell)
@@ -314,23 +311,23 @@ class TestCLISaveParser {
 
     var cmd2 = "echo -ne 'test'| " + Util.binPath + " parse --parser savedParser.xml \n"
     shell.send(cmd2)
-    shell.expect(contains("[warning] Validation Error: validation_check: cvc-pattern-valid"))
-    shell.expect(contains("[warning] Validation Error: validation_check failed"))
+    shell.expect(contains("[warn] Validation Error: validation_check: cvc-pattern-valid"))
+    shell.expect(contains("[warn] Validation Error: validation_check failed"))
 
     cmd = Util.binPath + " save-parser --validate -s daffodil-cli/src/it/resources/org/apache/daffodil/CLI/cli_schema.dfdl.xsd -r validation_check savedParser.xml\n"
     shell.send(cmd)
 
     cmd2 = "echo -ne 'test'| " + Util.binPath + " parse --parser savedParser.xml \n"
     shell.send(cmd2)
-    shell.expect(contains("[warning] Validation Error: validation_check: cvc-pattern-valid"))
-    shell.expect(contains("[warning] Validation Error: validation_check failed"))
+    shell.expect(contains("[warn] Validation Error: validation_check: cvc-pattern-valid"))
+    shell.expect(contains("[warn] Validation Error: validation_check failed"))
 
     cmd = Util.binPath + " save-parser --validate limited -s daffodil-cli/src/it/resources/org/apache/daffodil/CLI/cli_schema.dfdl.xsd -r validation_check savedParser.xml\n"
     shell.send(cmd)
 
     cmd2 = "echo -ne 'test'| " + Util.binPath + " parse --parser savedParser.xml \n"
     shell.send(cmd2)
-    shell.expect(contains("[warning] Validation Error: validation_check failed"))
+    shell.expect(contains("[warn] Validation Error: validation_check failed"))
 
     cmd = Util.binPath + " save-parser --validate off -s daffodil-cli/src/it/resources/org/apache/daffodil/CLI/cli_schema.dfdl.xsd -r validation_check savedParser.xml\n"
     shell.send(cmd)

daffodil-cli/src/it/scala/org/apache/daffodil/udf/TestCLIUdfs.scala

@@ -95,7 +95,6 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -132,10 +131,8 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] Error while loading User Defined Function Providers:" +
-            " org.apache.daffodil.udf.UserDefinedFunctionProvider:" +
-            " Provider org.nonexistentclass.example.StringFunctions.StringFunctionsProvider not found"),
-          contains("[info] No User Defined Functions loaded."),
+          contains("[warn] User Defined Function Provider failed to load: org.apache.daffodil.udf.UserDefinedFunctionProvider:"),
+          contains("Provider org.nonexistentclass.example.StringFunctions.StringFunctionsProvider not found"),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -166,7 +163,6 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -199,10 +195,9 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function Provider ignored:" +
+          contains("[warn] User Defined Function Provider ignored:" +
             " org.badudfs.functionclasses1.StringFunctions.StringFunctionsProvider." +
             " No User Defined Functions found."),
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -235,10 +230,9 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function Provider ignored:" +
+          contains("[warn] User Defined Function Provider ignored:" +
             " org.badudfs.functionclasses2.StringFunctions.StringFunctionsProvider." +
             " No User Defined Functions found."),
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -272,13 +266,12 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.nonUDF.StringFunctions.FuncA." +
             " Doesn't implement org.apache.daffodil.udf.UserDefinedFunction"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.nonUDF.StringFunctions.Replace." +
             " Doesn't implement org.apache.daffodil.udf.UserDefinedFunction"),
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -312,20 +305,19 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.annotations.StringFunctions.FuncB." +
             " Missing org.apache.daffodil.udf.UserDefinedFunctionIdentification annotation"),
           anyOf(
-            contains("[warning] User Defined Function ignored:" +
+            contains("[warn] User Defined Function ignored:" +
               " org.badudfs.annotations.StringFunctions.Compare." +
               " Annotation namespace field is empty or invalid."),
-            contains("[warning] User Defined Function ignored:" +
+            contains("[warn] User Defined Function ignored:" +
               " org.badudfs.annotations.StringFunctions.Compare." +
               " Annotation name field is empty or invalid.")),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.annotations.StringFunctions.Replace." +
             " Annotation name field is empty or invalid."),
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -362,25 +354,24 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.FuncA." +
             " Overloaded evaluate method: urn:example:com:ext:badudfs:stringfunctions:funcA"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.Replace." +
             " Missing evaluate method: urn:example:com:ext:badudfs:stringfunctions:replace"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.FuncB." +
             " Unsupported return type: void"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.FuncC." +
             " Unsupported parameter type(s): String[],int[]"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.FuncD." +
             " Unsupported parameter type(s): String[]"),
-          contains("[warning] User Defined Function ignored:" +
+          contains("[warn] User Defined Function ignored:" +
             " org.badudfs.evaluate.StringFunctions.FuncE." +
             " Unsupported return type: String[]"),
-          contains("[info] No User Defined Functions loaded."),
           contains("[error] Schema Definition Error: Unsupported function: jsudf:replace")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -414,9 +405,8 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] User Defined Function 'ssudf:reverse' Error: UDF Error!"),
-          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException: "),
-          contains("at org.sbadudfs.udfexceptions.evaluating.StringFunctions.Reverse.evaluate")))
+          contains("[error] User Defined Function 'ssudf:reverse' Error. Cause: org.sbadudfs.udfexceptions.evaluating.StringFunctions.Reverse$CustomException: UDF Error!"),
+          contains("at org.sbadudfs.udfexceptions.evaluating.StringFunctions.Reverse.evaluate(StringFunctionsProvider.scala:56)")))
 
       Util.expectExitCode(ExitCode.UserDefinedFunctionError, shell)
       shell.send("exit\n")
@@ -479,12 +469,9 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] Error initializing User Defined Function:"),
-          contains("http://example.com/scala/udf:rev-words."),
-          contains("Error thrown: org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords$CustomException: UDF Error!"),
-          contains("[error] User Defined Function 'http://example.com/scala/udf:rev-words' Error: UDF Error!"),
-          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException:"),
-          contains("at org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords")))
+          contains("[error] User Defined Function could not be initialized: {http://example.com/scala/udf}rev-words."),
+          contains("Cause: org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords$CustomException: UDF Error!"),
+          contains("at org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords.<init>(StringFunctionsProvider.scala:65)")))
 
       Util.expectExitCode(ExitCode.UserDefinedFunctionError, shell)
       shell.send("exit\n")
@@ -517,7 +504,7 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[warning] User Defined Function Provider ignored:"),
+          contains("[warn] User Defined Function Provider ignored:"),
           contains("org.sbadudfs.udfpexceptions.StringFunctions.StringFunctionsProvider"),
           contains("Error loading User Defined Functions:"),
           contains("org.sbadudfs.udfpexceptions.StringFunctions.StringFunctionsProvider$CustomException"),
@@ -554,10 +541,8 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] Error while loading User Defined Function Providers:" +
-            " org.apache.daffodil.udf.UserDefinedFunctionProvider:" +
-            " Provider org.sbadudfs.udfpexceptions2.StringFunctions.StringFunctionsProvider could not be instantiated"),
-          contains("[info] No User Defined Functions loaded."),
+          contains("[warn] User Defined Function Provider failed to load: org.apache.daffodil.udf.UserDefinedFunctionProvider"),
+          contains("Provider org.sbadudfs.udfpexceptions2.StringFunctions.StringFunctionsProvider could not be instantiated"),
           contains("[error] Schema Definition Error: Unsupported function: ssudf:rev-words")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -592,9 +577,9 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] User Defined Function Class Mismatch: http://example.com/scala/udf:rev-words." +
-            " Expected: class org.sbadudfs.functionclasses.StringFunctions.ReverseWords" +
-            " Actual: class org.sbadudfs.functionclasses.StringFunctions.Reverse"),
+          contains("[warn] User Defined Function class mismatch: {http://example.com/scala/udf}rev-words."),
+          contains("Expected: class org.sbadudfs.functionclasses.StringFunctions.ReverseWords"),
+          contains("Actual: class org.sbadudfs.functionclasses.StringFunctions.Reverse"),
           contains("[error] Schema Definition Error: Unsupported function: ssudf:rev-words")))
 
       Util.expectExitCode(ExitCode.UnableToCreateProcessor, shell)
@@ -628,11 +613,9 @@ class TestCLIUdfs {
       shell.expectIn(
         1,
         allOf(
-          contains("[error] Error initializing User Defined Function:"),
-          contains("http://example.com/scala/udf:reverse."),
-          contains("Error thrown: scala.MatchError:"),
-          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException:"),
-          contains("at org.sbadudfs.functionclasses.StringFunctions.StringFunctionsProvider.createUserDefinedFunction")))
+          contains("[error] User Defined Function could not be initialized: {http://example.com/scala/udf}reverse."),
+          contains("Cause: scala.MatchError: http://example.com/scala/udf:reverse (of class java.lang.String)"),
+          contains("at org.sbadudfs.functionclasses.StringFunctions.StringFunctionsProvider.createUserDefinedFunction(StringFunctionsProvider.scala:34)")))
 
       Util.expectExitCode(ExitCode.UserDefinedFunctionError, shell)
       shell.send("exit\n")
@@ -660,12 +643,17 @@ class TestCLIUdfs {
     val shell = Util.start("", envp = Map("DAFFODIL_CLASSPATH" -> dafClassPath))
 
     try {
-      val cmd = String.format("%s -v save-parser -s %s -r user_func4", Util.binPath, testSchemaFile)
+      val cmd = String.format("%s -vv save-parser -s %s -r user_func4", Util.binPath, testSchemaFile)
       shell.sendLine(cmd)
       shell.expectIn(
         1,
         allOf(
-          contains("[error] Error serializing initialized User Defined Function: org.sbadudfs.functionclasses2.StringFunctions.GetNonSerializableState"),
+          contains("[debug] User Defined Function loaded: org.sbadudfs.functionclasses2.StringFunctions.GetNonSerializableState => {http://example.com/scala/udf}get-nonserializable-state"),
+          contains("[debug] User Defined Function loaded: org.sbadudfs.functionclasses2.StringFunctions.GetSerializableState => {http://example.com/scala/udf}get-serializable-state")))
+      shell.expectIn(
+        1,
+        allOf(
+          contains("[warn] User Defined Function is not serializable: org.sbadudfs.functionclasses2.StringFunctions.GetNonSerializableState."),
           contains("Could not serialize member of class: org.sbadudfs.functionclasses2.StringFunctions.SomeNonSerializableClass"),
           contains("[error] Schema Definition Error: Unsupported function: ssudf:get-nonserializable-state")))
 
@@ -698,10 +686,15 @@ class TestCLIUdfs {
 
     try {
       val cmds = Array(
-        String.format("%s -v save-parser -s %s -r user_func5 %s", Util.binPath, testSchemaFile, savedParserFile.getAbsolutePath),
+        String.format("%s -vv save-parser -s %s -r user_func5 %s", Util.binPath, testSchemaFile, savedParserFile.getAbsolutePath),
         String.format(Util.echoN("strng") + "| %s -v parse -P %s", Util.binPath, savedParserFile.getAbsolutePath))
       val cmd = Util.makeMultipleCmds(cmds)
       shell.sendLine(cmd)
+      shell.expectIn(
+        1,
+        allOf(
+          contains("[debug] User Defined Function loaded: org.sbadudfs.functionclasses2.StringFunctions.GetNonSerializableState => {http://example.com/scala/udf}get-nonserializable-state"),
+          contains("[debug] User Defined Function loaded: org.sbadudfs.functionclasses2.StringFunctions.GetSerializableState => {http://example.com/scala/udf}get-serializable-state")))
       shell.expectIn(
         0,
         allOf(