From 1f7082cb88a4f796bf305d680ab49509049eec11 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Thu, 23 Nov 2017 17:56:23 +0000 Subject: [PATCH 1/4] Add --stop-warnings to riot (JENA-1434) Add the option to allow riot to abort on warnings --- .../java/org/apache/jena/riot/RDFParserBuilder.java | 2 +- jena-cmds/src/main/java/arq/cmdline/ModLangParse.java | 10 ++++++++++ jena-cmds/src/main/java/riotcmd/CmdLangParse.java | 11 ++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/jena-arq/src/main/java/org/apache/jena/riot/RDFParserBuilder.java b/jena-arq/src/main/java/org/apache/jena/riot/RDFParserBuilder.java index f2d4063a189..8efdfd6630c 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/RDFParserBuilder.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/RDFParserBuilder.java @@ -407,7 +407,7 @@ private RDFParserBuilder langTagForm(LangTagForm form) { * *

* See also {@link #errorHandler(ErrorHandler)} to control the output. The default is to log. - * Thsi can also be used to turn warnings into exceptions. + * This can also be used to turn warnings into exceptions. */ public RDFParserBuilder checking(boolean flag) { this.checking = Optional.of(flag) ; return this; } diff --git a/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java b/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java index 212807dd71c..fd24adf0835 100644 --- a/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java +++ b/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java @@ -44,6 +44,7 @@ public class ModLangParse extends ModBase private ArgDecl argSkip = new ArgDecl(ArgDecl.NoValue, "skip") ; private ArgDecl argNoSkip = new ArgDecl(ArgDecl.NoValue, "noSkip") ; private ArgDecl argStop = new ArgDecl(ArgDecl.NoValue, "stopOnError", "stoponerror", "stop") ; + private ArgDecl argStopWarn = new ArgDecl(ArgDecl.NoValue, "stopOnWarning", "stoponwarning", "stop-warnings") private ArgDecl argBase = new ArgDecl(ArgDecl.HasValue, "base") ; @@ -58,6 +59,7 @@ public class ModLangParse extends ModBase private boolean explicitNoCheck = false ; private boolean skipOnBadTerm = false ; private boolean stopOnBadTerm = false ; + private boolean stopOnWarnings = false ; private boolean bitbucket = false ; private boolean strict = false ; private boolean validate = false ; @@ -78,6 +80,7 @@ public void registerWith(CmdGeneral cmdLine) { // cmdLine.add(argSkip, "--noSkip", "Skip (do not output) triples failing the RDF term tests") ; // cmdLine.add(argNoSkip, "--skip", "Include triples failing the RDF term tests (not recommended)") ; cmdLine.add(argStop, "--stop", "Stop parsing on encountering a bad RDF term") ; + cmdLine.add(argStopWarn,"--stop-warnings", "Stop parsing on encountering a warning") ; } @Override @@ -122,6 +125,9 @@ public void processArgs(CmdArgModule cmdLine) { if ( cmdLine.contains(argStop) ) stopOnBadTerm = true ; + + if ( cmdLine.contains(argStopWarn) ) + stopOnWarnings = true; if ( cmdLine.contains(argSink) ) bitbucket = true ; @@ -161,6 +167,10 @@ public boolean skipOnBadTerm() { public boolean stopOnBadTerm() { return stopOnBadTerm ; } + + public boolean stopOnWarnings() { + return stopOnWarnings ; + } public boolean toBitBucket() { return bitbucket ; diff --git a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java index 34a7dec60bf..da6a72367cd 100644 --- a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java +++ b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java @@ -239,11 +239,16 @@ protected ParseRecord parseRIOT(RDFParserBuilder builder, /*Info for the Process ErrorHandler errHandler = ErrorHandlerFactory.errorHandlerWarn ; if ( checking ) { - if ( modLangParse.stopOnBadTerm() ) - errHandler = ErrorHandlerFactory.errorHandlerStd ; - else + if ( modLangParse.stopOnBadTerm() ) { + if ( modLangParse.stopOnWarnings() ) { + errHandler = ErrorHandlerFactory.errorHandlerStrict ; + } else { + errHandler = ErrorHandlerFactory.errorHandlerStd ; + } + } else { // Try to go on if possible. This is the default behaviour. errHandler = ErrorHandlerFactory.errorHandlerWarn ; + } } if ( modLangParse.skipOnBadTerm() ) { From 2b09b81a6a462bbf2006a7232def3173e2ddcb0e Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Thu, 23 Nov 2017 17:57:30 +0000 Subject: [PATCH 2/4] Fix syntax error (JENA-1434) --- jena-cmds/src/main/java/arq/cmdline/ModLangParse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java b/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java index fd24adf0835..527fae6f2d6 100644 --- a/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java +++ b/jena-cmds/src/main/java/arq/cmdline/ModLangParse.java @@ -44,7 +44,7 @@ public class ModLangParse extends ModBase private ArgDecl argSkip = new ArgDecl(ArgDecl.NoValue, "skip") ; private ArgDecl argNoSkip = new ArgDecl(ArgDecl.NoValue, "noSkip") ; private ArgDecl argStop = new ArgDecl(ArgDecl.NoValue, "stopOnError", "stoponerror", "stop") ; - private ArgDecl argStopWarn = new ArgDecl(ArgDecl.NoValue, "stopOnWarning", "stoponwarning", "stop-warnings") + private ArgDecl argStopWarn = new ArgDecl(ArgDecl.NoValue, "stopOnWarning", "stoponwarning", "stop-warnings") ; private ArgDecl argBase = new ArgDecl(ArgDecl.HasValue, "base") ; From af4ae4f29a77cd3e324baf3b3dd5d46e73e11a6a Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 24 Nov 2017 10:25:09 +0000 Subject: [PATCH 3/4] Rework solution for --stop-warnings (JENA-1434) - Track total warnings and errors - Optionally stop on errors and/or warnings - Report error and warning count as part of statistics - Always exit non-zero if any errors or warnings are encountered --- .../jena/riot/system/ErrorHandlerFactory.java | 61 +++++++++++++++++++ .../src/main/java/riotcmd/CmdLangParse.java | 38 ++++++------ 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java b/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java index cc7d4bcc97b..f35c447e00e 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java @@ -65,6 +65,9 @@ public class ErrorHandlerFactory /** Ignores warnings, throws exceptions for errors */ public static ErrorHandler errorHandlerSimple() { return new ErrorHandlerSimple() ; } + /** Logs warnings and errors while tracking the counts of each and optionally throwing exceptions when errors and/or warnings are encounted */ + public static ErrorHandlerTracking errorHandlerTracking(Logger log, boolean failOnError, boolean failOnWarning) { return new ErrorHandlerTracking(log, failOnError, failOnWarning); } + /** * An error handler that throws a {@link RiotParseException}, hence it * exposes the details of errors. @@ -221,6 +224,64 @@ public void fatal(String message, long line, long col) { } } + /** An error handler that logs message for errors and warnings and throw exceptions on either */ + public static class ErrorHandlerTracking extends ErrorLogger implements ErrorHandler { + private final boolean failOnError, failOnWarning; + private long errorCount, warningCount; + + public ErrorHandlerTracking(Logger log, boolean failOnError, boolean failOnWarning) { + super(log) ; + + this.failOnError = failOnError; + this.failOnWarning = failOnWarning; + } + + /** report a warning */ + @Override + public void warning(String message, long line, long col) { + logWarning(message, line, col) ; + this.warningCount++; + if (this.failOnWarning) + throw new RiotException(fmtMessage(message, line, col)) ; + } + + /** report an error */ + @Override + public void error(String message, long line, long col) { + logError(message, line, col) ; + this.errorCount++; + if (this.failOnError) + throw new RiotException(fmtMessage(message, line, col)) ; + } + + @Override + public void fatal(String message, long line, long col) { + logFatal(message, line, col) ; + this.errorCount++; + throw new RiotException(fmtMessage(message, line, col)) ; + } + + public long getErrorCount() { + return this.errorCount; + } + + public long getWarningCount() { + return this.warningCount; + } + + public boolean hadErrors() { + return this.errorCount > 1; + } + + public boolean hadWarnings() { + return this.warningCount > 1; + } + + public boolean hadIssues() { + return hadErrors() || hadWarnings(); + } + } + /** An error handler that logs messages for errors and warnings and attempt to carry on */ private static class ErrorHandlerWarning extends ErrorLogger implements ErrorHandler { public ErrorHandlerWarning(Logger log) diff --git a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java index da6a72367cd..de14fd36628 100644 --- a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java +++ b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java @@ -44,6 +44,7 @@ import org.apache.jena.riot.process.inf.InfFactory ; import org.apache.jena.riot.process.inf.InferenceSetupRDFS ; import org.apache.jena.riot.system.* ; +import org.apache.jena.riot.system.ErrorHandlerFactory.ErrorHandlerTracking; import org.apache.jena.riot.tokens.Tokenizer ; import org.apache.jena.riot.tokens.TokenizerFactory ; import org.apache.jena.sparql.core.DatasetGraph ; @@ -104,14 +105,16 @@ static class ParseRecord { final long triples; final long quads; final long tuples = 0; + final ErrorHandlerTracking errHandler; public ParseRecord(String filename, boolean successful, long timeMillis, - long countTriples, long countQuads) { + long countTriples, long countQuads, ErrorHandlerTracking errHandler) { this.filename = filename; this.success = successful; this.timeMillis = timeMillis; this.triples = countTriples; this.quads = countQuads; + this.errHandler = errHandler; } } @@ -173,6 +176,8 @@ protected void exec() { long totalTriples = 0; long totalQuads = 0; long totalTuples = 0; + long totalErrors = 0; + long totalWarnings = 0; boolean allSuccessful = true; for ( ParseRecord pRec : outcomes ) { @@ -181,9 +186,11 @@ protected void exec() { totalTriples += pRec.triples; totalQuads += pRec.quads; totalTuples += pRec.tuples; + totalErrors += pRec.errHandler.getErrorCount(); + totalWarnings += pRec.errHandler.getWarningCount(); allSuccessful = allSuccessful & pRec.success; } - output("Total", true, totalTriples, totalQuads, totalTuples, totalMillis); + output("Total", true, totalTriples, totalQuads, totalTuples, totalMillis, totalErrors, totalWarnings); } } finally { if ( outputWrite != System.out ) @@ -195,7 +202,7 @@ protected void exec() { // exit(1) if there were any errors. for ( ParseRecord pr : outcomes ) { - if ( ! pr.success ) + if ( ! pr.success || pr.errHandler.hadIssues() ) throw new CmdException(); } } @@ -237,19 +244,7 @@ protected ParseRecord parseRIOT(RDFParserBuilder builder, /*Info for the Process checking = false; builder.checking(checking); - ErrorHandler errHandler = ErrorHandlerFactory.errorHandlerWarn ; - if ( checking ) { - if ( modLangParse.stopOnBadTerm() ) { - if ( modLangParse.stopOnWarnings() ) { - errHandler = ErrorHandlerFactory.errorHandlerStrict ; - } else { - errHandler = ErrorHandlerFactory.errorHandlerStd ; - } - } else { - // Try to go on if possible. This is the default behaviour. - errHandler = ErrorHandlerFactory.errorHandlerWarn ; - } - } + ErrorHandlerTracking errHandler = ErrorHandlerFactory.errorHandlerTracking(ErrorHandlerFactory.stdLogger, modLangParse.stopOnBadTerm(), modLangParse.stopOnWarnings()); if ( modLangParse.skipOnBadTerm() ) { // skipOnBadterm - this needs collaboration from the parser. @@ -292,7 +287,7 @@ protected ParseRecord parseRIOT(RDFParserBuilder builder, /*Info for the Process sink.finish() ; long x = modTime.endTimer() ; // TEMP - ParseRecord outcome = new ParseRecord(filename, successful, x, sink.countTriples(), sink.countQuads()); + ParseRecord outcome = new ParseRecord(filename, successful, x, sink.countTriples(), sink.countQuads(), errHandler); return outcome; } @@ -341,10 +336,11 @@ protected Tokenizer makeTokenizer(InputStream in) { protected void output(ParseRecord rtn) { output(rtn.filename, rtn.success, rtn.triples, rtn.quads, rtn.tuples, - rtn.timeMillis) ; + rtn.timeMillis, rtn.errHandler.getErrorCount(), + rtn.errHandler.getWarningCount()) ; } - protected void output(String label, boolean success, long numberTriples, long numberQuads, long numberTuples, long timeMillis) { + protected void output(String label, boolean success, long numberTriples, long numberQuads, long numberTuples, long timeMillis, long errorCount, long warningCount) { double timeSec = timeMillis/1000.0 ; long total = numberTriples + numberQuads + numberTuples; StringBuilder sb = new StringBuilder(); @@ -360,6 +356,10 @@ protected void output(String label, boolean success, long numberTriples, long nu } else { appendFmt(sb, "%s : (No Output)", label) ; } + if ( errorCount > 0 || warningCount > 0 ) { + appendFmt(sb," : %,d %s", errorCount, "errors"); + appendFmt(sb," : %,d %s", warningCount, "warnings"); + } System.err.println(sb.toString()); } From 8df3702132a41221f122699c469fc2687ad7cffc Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 24 Nov 2017 10:54:52 +0000 Subject: [PATCH 4/4] Minor fixes to new code (JENA-1434) - Logic error in hadErrors() and hadWarnings() - Minor formatting --- .../java/org/apache/jena/riot/system/ErrorHandlerFactory.java | 4 ++-- jena-cmds/src/main/java/riotcmd/CmdLangParse.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java b/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java index f35c447e00e..3eeef44d971 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/system/ErrorHandlerFactory.java @@ -270,11 +270,11 @@ public long getWarningCount() { } public boolean hadErrors() { - return this.errorCount > 1; + return this.errorCount > 0; } public boolean hadWarnings() { - return this.warningCount > 1; + return this.warningCount > 0; } public boolean hadIssues() { diff --git a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java index de14fd36628..8669e88e233 100644 --- a/jena-cmds/src/main/java/riotcmd/CmdLangParse.java +++ b/jena-cmds/src/main/java/riotcmd/CmdLangParse.java @@ -244,7 +244,9 @@ protected ParseRecord parseRIOT(RDFParserBuilder builder, /*Info for the Process checking = false; builder.checking(checking); - ErrorHandlerTracking errHandler = ErrorHandlerFactory.errorHandlerTracking(ErrorHandlerFactory.stdLogger, modLangParse.stopOnBadTerm(), modLangParse.stopOnWarnings()); + ErrorHandlerTracking errHandler = ErrorHandlerFactory.errorHandlerTracking(ErrorHandlerFactory.stdLogger, + modLangParse.stopOnBadTerm(), + modLangParse.stopOnWarnings()); if ( modLangParse.skipOnBadTerm() ) { // skipOnBadterm - this needs collaboration from the parser.