Skip to content

Commit

Permalink
Warn when treating too-many-digit doubles as string.
Browse files Browse the repository at this point in the history
  • Loading branch information
@kno10
kno10 committed Dec 7, 2017
1 parent b8df95f commit 748252b
Show file tree
Hide file tree
Showing 5 changed files with 31 additions and 6 deletions.
12 changes: 6 additions & 6 deletions elki-core-util/src/main/java/de/lmu/ifi/dbs/elki/utilities/io/ParseUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ private ParseUtil() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException EMPTY_STRING = new NumberFormatException("Parser called on an empty string.") {
public static final NumberFormatException EMPTY_STRING = new NumberFormatException("Parser called on an empty string.") {
private static final long serialVersionUID = 1L;

@Override
Expand All @@ -50,7 +50,7 @@ public synchronized Throwable fillInStackTrace() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException EXPONENT_OVERFLOW = new NumberFormatException("Precision overflow for double exponent.") {
public static final NumberFormatException EXPONENT_OVERFLOW = new NumberFormatException("Precision overflow for double exponent.") {
private static final long serialVersionUID = 1L;

@Override
Expand All @@ -62,7 +62,7 @@ public synchronized Throwable fillInStackTrace() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException INVALID_EXPONENT = new NumberFormatException("Invalid exponent") {
public static final NumberFormatException INVALID_EXPONENT = new NumberFormatException("Invalid exponent") {
private static final long serialVersionUID = 1L;

@Override
Expand All @@ -74,7 +74,7 @@ public synchronized Throwable fillInStackTrace() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException TRAILING_CHARACTERS = new NumberFormatException("String sequence was not completely consumed.") {
public static final NumberFormatException TRAILING_CHARACTERS = new NumberFormatException("String sequence was not completely consumed.") {
private static final long serialVersionUID = 1L;

@Override
Expand All @@ -86,7 +86,7 @@ public synchronized Throwable fillInStackTrace() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException PRECISION_OVERFLOW = new NumberFormatException("Precision overflow for long values.") {
public static final NumberFormatException PRECISION_OVERFLOW = new NumberFormatException("Precision overflow for long values.") {
private static final long serialVersionUID = 1L;

@Override
Expand All @@ -98,7 +98,7 @@ public synchronized Throwable fillInStackTrace() {
/**
* Preallocated exceptions.
*/
private static final NumberFormatException NOT_A_NUMBER = new NumberFormatException("Number must start with a digit or dot.") {
public static final NumberFormatException NOT_A_NUMBER = new NumberFormatException("Number must start with a digit or dot.") {
private static final long serialVersionUID = 1L;

@Override
Expand Down
5 changes: 5 additions & 0 deletions ...c/main/java/de/lmu/ifi/dbs/elki/datasource/parser/CategorialDataAsNumberVectorParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
import de.lmu.ifi.dbs.elki.utilities.io.ParseUtil;

import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;

Expand Down Expand Up @@ -112,6 +113,10 @@ protected boolean parseLineInternal() {
attributes.add(Double.NaN);
continue;
}
if(!warnedPrecision && (e == ParseUtil.PRECISION_OVERFLOW || e == ParseUtil.EXPONENT_OVERFLOW)) {
getLogger().warning("Too many digits in what looked like a double number - treating as string: " + tokenizer.getSubstring());
warnedPrecision = true;
}
int id = unique.getInt(s);
if(id == unique.defaultReturnValue()) {
id = ustart + unique.size();
Expand Down
10 changes: 10 additions & 0 deletions elki-input/src/main/java/de/lmu/ifi/dbs/elki/datasource/parser/NumberVectorLabelParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import de.lmu.ifi.dbs.elki.utilities.datastructures.BitsUtil;
import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.DoubleArray;
import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
import de.lmu.ifi.dbs.elki.utilities.io.ParseUtil;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntListParameter;
Expand Down Expand Up @@ -134,6 +135,11 @@ public class NumberVectorLabelParser<V extends NumberVector> extends AbstractStr
*/
Event nextevent = null;

/**
* Emit a double-precision limit warning once.
*/
boolean warnedPrecision = false;

/**
* Constructor.
*
Expand Down Expand Up @@ -277,6 +283,10 @@ protected boolean parseLineInternal() {
continue;
}
catch(NumberFormatException e) {
if(!warnedPrecision && (e == ParseUtil.PRECISION_OVERFLOW || e == ParseUtil.EXPONENT_OVERFLOW)) {
getLogger().warning("Too many digits in what looked like a double number - treating as string: " + tokenizer.getSubstring());
warnedPrecision = true;
}
// Ignore attempt, add to labels below.
}
}
Expand Down
5 changes: 5 additions & 0 deletions ...ut/src/main/java/de/lmu/ifi/dbs/elki/datasource/parser/SparseNumberVectorLabelParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
import de.lmu.ifi.dbs.elki.utilities.io.ParseUtil;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter;

Expand Down Expand Up @@ -167,6 +168,10 @@ protected boolean parseLineInternal() {
}
}
catch(NumberFormatException e) {
if(!warnedPrecision && (e == ParseUtil.PRECISION_OVERFLOW || e == ParseUtil.EXPONENT_OVERFLOW)) {
getLogger().warning("Too many digits in what looked like a double number - treating as string: " + tokenizer.getSubstring());
warnedPrecision = true;
}
// continue with fallback below.
}
}
Expand Down
5 changes: 5 additions & 0 deletions elki-input/src/main/java/de/lmu/ifi/dbs/elki/datasource/parser/TermFrequencyParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
import de.lmu.ifi.dbs.elki.utilities.io.ParseUtil;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag;
Expand Down Expand Up @@ -146,6 +147,10 @@ protected boolean parseLineInternal() {
curterm = null;
}
catch(NumberFormatException e) {
if(!warnedPrecision && (e == ParseUtil.PRECISION_OVERFLOW || e == ParseUtil.EXPONENT_OVERFLOW)) {
getLogger().warning("Too many digits in what looked like a double number - treating as string: " + tokenizer.getSubstring());
warnedPrecision = true;
}
if(curterm != null) {
labels.add(curterm);
}
Expand Down

0 comments on commit 748252b

Please sign in to comment.