From b9ae73ab4c732fcd3f4b8b0e4d3b9b27978bbacb Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 13 Jul 2022 16:15:11 +0100 Subject: [PATCH 01/20] SOLR-16292 : NVector for alternative great-circle distance calculations --- .../java/org/apache/solr/schema/NVector.java | 252 +++++ .../search/function/distance/NVector.java | 105 ++ .../distance/NVectorValueSourceParser.java | 46 + .../org/apache/solr/util/FastInvTrig.java | 97 ++ .../org/apache/solr/util/NVectorUtil.java | 50 + .../solr/collection1/conf/schema-nvector.xml | 930 ++++++++++++++++++ .../collection1/conf/solrconfig-nvector.xml | 528 ++++++++++ .../function/distance/NVectorDistTest.java | 91 ++ .../org/apache/solr/util/FastInvTrigTest.java | 65 ++ .../org/apache/solr/util/NVectorUtilTest.java | 54 + 10 files changed, 2218 insertions(+) create mode 100644 solr/core/src/java/org/apache/solr/schema/NVector.java create mode 100644 solr/core/src/java/org/apache/solr/search/function/distance/NVector.java create mode 100644 solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java create mode 100644 solr/core/src/java/org/apache/solr/util/FastInvTrig.java create mode 100644 solr/core/src/java/org/apache/solr/util/NVectorUtil.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-nvector.xml create mode 100644 solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java create mode 100644 solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java create mode 100644 solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java diff --git a/solr/core/src/java/org/apache/solr/schema/NVector.java b/solr/core/src/java/org/apache/solr/schema/NVector.java new file mode 100644 index 00000000000..9fe44abacd2 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/NVector.java @@ -0,0 +1,252 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. +package org.apache.solr.schema; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValueSource; +import org.apache.lucene.search.SortField; +import org.apache.solr.common.SolrException; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader; +import org.apache.solr.util.NVectorUtil; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NVector extends CoordinateFieldType { + + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + dimension = 3; + createSuffixCache(3); + } + + @Override + public List createFields(SchemaField field, Object value) { + String externalVal = value.toString(); + String[] point = parseCommaSeparatedList(externalVal, dimension); + String[] nvector = NVectorUtil.latLongToNVector(point); + + List f = new ArrayList<>((dimension * 2) + 1); + + if (field.indexed()) { + for (int i = 0; i < dimension; i++) { + SchemaField sf = subField(field, i, schema); + f.addAll(sf.createFields(nvector[i])); + } + } + + if (field.stored()) { + f.add(createField(field.getName(), externalVal, StoredField.TYPE)); + } + return f; + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser parser) { + ArrayList vs = new ArrayList<>(dimension); + for (int i = 0; i < dimension; i++) { + SchemaField sub = subField(field, i, schema); + vs.add(sub.getType() + .getValueSource(sub, parser)); + } + return new NVectorValueSource(vs); + } + + + /** + * Given a string containing dimension values encoded in it, separated by commas, + * return a String array of length dimension containing the values. + * + * @param externalVal The value to parse + * @param dimension The expected number of values for the point + * @return An array of the values that make up the point (aka vector) + * @throws SolrException if the dimension specified does not match the number found + */ + public static String[] parseCommaSeparatedList(String externalVal, int dimension) throws SolrException { + //TODO: Should we support sparse vectors? + String[] out = new String[dimension]; + int idx = externalVal.indexOf(','); + int end = idx; + int start = 0; + int i = 0; + if (idx == -1 && dimension == 1 && externalVal.length() > 0) {//we have a single point, dimension better be 1 + out[0] = externalVal.trim(); + i = 1; + } else if (idx > 0) {//if it is zero, that is an error + //Parse out a comma separated list of values, as in: 73.5,89.2,7773.4 + for (; i < dimension; i++) { + while (start < end && externalVal.charAt(start) == ' ') start++; + while (end > start && externalVal.charAt(end - 1) == ' ') end--; + if (start == end) { + break; + } + out[i] = externalVal.substring(start, end); + start = idx + 1; + end = externalVal.indexOf(',', start); + idx = end; + if (end == -1) { + end = externalVal.length(); + } + } + } + if (i != dimension) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "incompatible dimension (" + dimension + + ") and values (" + externalVal + "). Only " + i + " values specified"); + } + return out; + } + + @Override + protected void checkSupportsDocValues() { + // DocValues supported only when enabled at the fieldType + if (!hasProperty(DOC_VALUES)) { + throw new UnsupportedOperationException("PointType can't have docValues=true in the field definition, use docValues=true in the fieldType definition, or in subFieldType/subFieldSuffix"); + } + } + + @Override + public UninvertingReader.Type getUninversionType(SchemaField sf) { + return null; + } + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + writer.writeStr(name, f.stringValue(), true); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Sorting not supported on NVector " + field.getName()); + } + + @Override + public boolean isPolyField() { + return true; + } +} + +final class NVectorValueSource extends MultiValueSource { + private final List sources; + + public NVectorValueSource(List sources) { + this.sources = sources; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { + final FunctionValues x = sources.get(0) + .getValues(context, readerContext); + final FunctionValues y = sources.get(1) + .getValues(context, readerContext); + final FunctionValues z = sources.get(2) + .getValues(context, readerContext); + return new FunctionValues() { + + @Override + public void byteVal(int doc, byte[] vals) throws IOException { + vals[0] = x.byteVal(doc); + vals[1] = y.byteVal(doc); + vals[2] = z.byteVal(doc); + } + + @Override + public void shortVal(int doc, short[] vals) throws IOException { + vals[0] = x.shortVal(doc); + vals[1] = y.shortVal(doc); + vals[2] = z.shortVal(doc); + } + + @Override + public void intVal(int doc, int[] vals) throws IOException { + vals[0] = x.intVal(doc); + vals[1] = y.intVal(doc); + vals[2] = z.intVal(doc); + } + + @Override + public void longVal(int doc, long[] vals) throws IOException { + vals[0] = x.longVal(doc); + vals[1] = y.longVal(doc); + vals[2] = z.longVal(doc); + } + + @Override + public void floatVal(int doc, float[] vals) throws IOException { + vals[0] = x.floatVal(doc); + vals[1] = y.floatVal(doc); + vals[2] = z.floatVal(doc); + } + + @Override + public void doubleVal(int doc, double[] vals) throws IOException { + vals[0] = x.doubleVal(doc); + vals[1] = y.doubleVal(doc); + vals[2] = z.doubleVal(doc); + } + + @Override + public void strVal(int doc, String[] vals) throws IOException { + vals[0] = x.strVal(doc); + vals[1] = y.strVal(doc); + vals[2] = z.strVal(doc); + } + + @Override + public String toString(int doc) throws IOException { + return "nvector(" + x.toString(doc) + "," + y.toString(doc) + "," + z.toString(doc) + ")"; + } + }; + } + + @Override + public String description() { + StringBuilder sb = new StringBuilder(); + sb.append("nvector("); + boolean firstTime = true; + for (ValueSource source : sources) { + if (firstTime) { + firstTime = false; + } else { + sb.append(','); + } + sb.append(source); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NVectorValueSource)) return false; + + NVectorValueSource that = (NVectorValueSource) o; + + return sources.equals(that.sources); + + } + + @Override + public int hashCode() { + return sources.hashCode(); + } + + @Override + public int dimension() { + return sources.size(); + } +} diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java new file mode 100644 index 00000000000..18a4d0b94dc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java @@ -0,0 +1,105 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.search.function.distance; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.DoubleDocValues; +import org.apache.lucene.queries.function.valuesource.MultiValueSource; +import org.apache.lucene.search.IndexSearcher; +import org.apache.solr.common.SolrException; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.solr.util.NVectorUtil.NVectorDist; + +public class NVector extends ValueSource { + + private final MultiValueSource p1; + private final MultiValueSource p2; + private final double radius; + + public NVector(MultiValueSource p1, MultiValueSource p2, double radius) { + this.p1 = p1; + this.p2 = p2; + if (p1.dimension() != 3 || p2.dimension() != 3) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Illegal dimension for value sources"); + } + this.radius = radius; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { + + final FunctionValues vals1 = p1.getValues(context, readerContext); + final FunctionValues vals2 = p2.getValues(context, readerContext); + + return new DoubleDocValues(this) { + + @Override + public double doubleVal(int doc) throws IOException { + double[] dv1 = new double[p1.dimension()]; + double[] dv2 = new double[p2.dimension()]; + vals1.doubleVal(doc, dv1); + vals2.doubleVal(doc, dv2); + return NVectorDist(dv1, dv2, radius); + } + + @Override + public String toString(int doc) throws IOException { + return name() + + ',' + + vals1.toString(doc) + + ',' + + vals2.toString(doc) + + ')'; + } + }; + } + + protected String name() { + return "nvector"; + } + + @Override + public boolean equals(Object o) { + if (this.getClass() != o.getClass()) return false; + NVector other = (NVector) o; + return this.name() + .equals(other.name()) + && p1.equals(other.p1) && + p2.equals(other.p2) && radius == other.radius; + } + + @Override + public int hashCode() { + int result; + long temp; + result = p1.hashCode(); + result = 31 * result + p2.hashCode(); + result = 31 * result + name().hashCode(); + temp = Double.doubleToRawLongBits(radius); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public void createWeight(Map context, IndexSearcher searcher) throws IOException { + p1.createWeight(context, searcher); + p2.createWeight(context, searcher); + } + + @Override + public String description() { + return name() + '(' + + p1 + ',' + p2 + + ')'; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java new file mode 100644 index 00000000000..9cf0332d52a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java @@ -0,0 +1,46 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.search.function.distance; + +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValueSource; +import org.apache.lucene.queries.function.valuesource.VectorValueSource; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.FunctionQParser; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.search.ValueSourceParser; +import org.apache.solr.util.NVectorUtil; + +import java.util.Arrays; + +public class NVectorValueSourceParser extends ValueSourceParser { + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + double lat = fp.parseDouble(); + double lon = fp.parseDouble(); + + ValueSource vs1 = fp.parseValueSource(); + if (!(vs1 instanceof MultiValueSource)) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Field must a MultiValueSource"); + MultiValueSource mvs1 = (MultiValueSource) vs1; + + double[] nvector = NVectorUtil.latLongToNVector(lat, lon); + MultiValueSource mvs2 = new VectorValueSource( + Arrays.asList( + new DoubleConstValueSource(nvector[0]), + new DoubleConstValueSource(nvector[1]), + new DoubleConstValueSource(nvector[2]) + )); + + double radius = fp.hasMoreArguments() ? fp.parseDouble() : NVectorUtil.EARTH_RADIUS; + + return new NVector(mvs1, mvs2, radius); + } +} diff --git a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java new file mode 100644 index 00000000000..f6cf8fc7b73 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java @@ -0,0 +1,97 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.util; + +public class FastInvTrig { + + final static double pip2 = Math.PI / 2.0; + static final double SQRT2 = 1.0 / Math.sqrt(2.0); + + final static double[] TABLE; + final static int MAX_TERMS = 100; + final static int DEF_TERMS = 10; + + static { + TABLE = new double[MAX_TERMS]; + double factor = 1.0; + double divisor = 1.0; + for (int n = 0; n < MAX_TERMS; n++) { + TABLE[n] = factor / divisor; + divisor += 2; + factor *= (2 * n + 1.0) / ((n + 1) * 2); + } + } + + private static double asin2(double x, int n_terms) { + if (n_terms > MAX_TERMS) + throw new IllegalArgumentException("Too many terms"); + double acc = x; + double tempExp = x; + double x2 = x * x; + for (int n = 1; n < n_terms; n++) { + tempExp *= x2; + acc += TABLE[n] * tempExp; + } + return acc; + } + + // split domain for faster convergence i.e. fewer maclaurin terms required + // see https://www.wolframalpha.com/input/?i=arcsin%28sqrt%281-x%5E2%29%29-acos%28x%29 for x > 0 + // arcsin(sqrt(1-x^2)) = acos(x) for x > 0 + // arcsin(sqrt(1-x^2)) = acos(x) = pi/2 - arcsin(x) for x > 0 + // arcsin(sqrt(1-x^2)) = pi/2 - arcsin(x) for x > 0 + // arcsin(x) = pi/2 - arcsin(sqrt(1-x^2)) for x > 0 .... 1 + // + // see https://www.wolframalpha.com/input/?i=arcsin%28x%29+-+arcsin%28sqrt%281-x%5E2%29%29+%2B+pi%2F2+ for x < 0 + // arcsin(sqrt(1-x^2)) = pi-acos(x) for x < 0 + // arcsin(sqrt(1-x^2)) = pi-(pi/2 - arcsin(x)) for x < 0 + // arcsin(sqrt(1-x^2)) = pi/2 + arcsin(x) for x < 0 + // arcsin(x) = arcsin(sqrt(1-x^2)) - pi/2 for x < 0 .... 2 + + // within domain [-SQRT2 <= x <= SQRT2] use arcsin(x). outside this range use formulae above. + // so that convergence is faster + //This way we can transform input x into [-1/sqrt(2),1/sqrt(2)], where convergence is relatively fast. + + public static double asin(double x, int n_terms) { + if (x > SQRT2) + return pip2 - asin2(Math.sqrt(1 - (x * x)), n_terms); + else if (Math.abs(x) <= SQRT2) + return asin2(x, n_terms); + return asin2(Math.sqrt(1 - (x * x)), n_terms) - pip2; + } + + public static double asin(double x) { + return asin(x, DEF_TERMS); + } + + public static double acos(double x, int n_terms) { + return Math.abs(x) <= SQRT2 ? pip2 - asin2(x, n_terms) : asin2(Math.sqrt(1 - (x * x)), n_terms); + } + + public static double acos(double x) { + return acos(x, DEF_TERMS); + } + + //Following for completion for Inverse trigonometric functions + public static double atan(double x) { + return asin(x / Math.sqrt(1 + x * x)); + } + + public static double acot(double x) { + return pip2 - atan(x); + } + + public static double asec(double x) { + return acos(1 / x); + } + + public static double acsc(double x) { + return pip2 - asec(x); + } + +} diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java new file mode 100644 index 00000000000..695ccf0424b --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -0,0 +1,50 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.util; + +public class NVectorUtil { + + public static final double EARTH_RADIUS = 6173.008;//km google:standard mean earth radius; + + public static double[] latLongToNVector(double lat, double lon) { + double latRad = lat * (Math.PI / 180); + double lonRad = lon * (Math.PI / 180); + double x = Math.cos(latRad) * Math.cos(lonRad); + double y = Math.cos(latRad) * Math.sin(lonRad); + double z = Math.sin(latRad); + return new double[]{x, y, z}; + } + + public static String[] latLongToNVector(String lat, String lon) { + double[] nvec = latLongToNVector(Double.parseDouble(lat), Double.parseDouble(lon)); + return new String[]{Double.toString(nvec[0]), Double.toString(nvec[1]), Double.toString(nvec[2])}; + } + + public static String[] latLongToNVector(String[] latlon) { + return latLongToNVector(latlon[0], latlon[1]); + } + + public static double[] NVectorToLatLong(double[] n) { + return new double[]{Math.asin(n[2]) * (180 / Math.PI), Math.atan(n[1] / n[0]) * (180 / Math.PI)}; + } + + public static double[] NVectorToLatLong(String[] n) { + return NVectorToLatLong(new double[]{ + Double.parseDouble(n[0]), + Double.parseDouble(n[1]), + Double.parseDouble(n[2])}); + } + + public static double NVectorDist(double[] a, double[] b) { + return NVectorDist(a, b, EARTH_RADIUS); + } + + public static double NVectorDist(double[] a, double[] b, double radius) { + return radius * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml new file mode 100644 index 00000000000..898a7217709 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml @@ -0,0 +1,930 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + I am your default sim + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nvector.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nvector.xml new file mode 100644 index 00000000000..d41555b0172 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nvector.xml @@ -0,0 +1,528 @@ + + + + + + + + + + + + + ${solr.data.dir:} + + + + 1000000 + 2000000 + 3000000 + 4000000 + ${solr.hdfs.home:} + ${solr.hdfs.blockcache.enabled:true} + ${solr.hdfs.blockcache.global:true} + ${solr.hdfs.blockcache.write.enabled:false} + ${solr.hdfs.blockcache.blocksperbank:10} + ${solr.hdfs.blockcache.slab.count:1} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + + + ${solr.autoCommit.maxTime:-1} + + + + + + ${solr.ulog.dir:} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + ${solr.max.booleanClauses:1024} + + + + + + + + + + + + true + + + + + + 10 + + + + + + + + + + + + 2000 + + + + + + + + true + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + weight^0.5 recip(rord(id),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + false + + + direct + DirectSolrSpellChecker + lowerfilt + 3 + + + wordbreak + solr.WordBreakSolrSpellChecker + lowerfilt + true + true + 10 + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + false + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + false + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + direct + false + false + 1 + + + spellcheck + + + + + default + wordbreak + 20 + + + spellcheck + + + + + direct + wordbreak + 20 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + max-age=30, public + + + + + + foo_s + + + foo_s:bar + + + + + foo_s + foo_s:bar + + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + + uniq + uniq2 + uniq3 + + + + + + + + + regex_dup_A_s + x + x_x + + + + regex_dup_B_s + x + x_x + + + + + + + + regex_dup_A_s + x + x_x + + + regex_dup_B_s + x + x_x + + + + + + + org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO + + + + + + text + + + + + + text + + + nl + + + diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java new file mode 100644 index 00000000000..b8a71ae0f57 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java @@ -0,0 +1,91 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.search.function.distance; + +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; +import org.junit.Test; + +public class NVectorDistTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + initCore("solrconfig-nvector.xml", "schema-nvector.xml"); + } + + @Test + public void testNVector() throws Exception { + assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); + assertU(commit()); + assertJQ(req("defType", "lucene", "q", "*:*", "fl", "id,nvector*","sort","id asc"), + "/response/docs/[0]== {" + + "'id':'0'," + + "'nvector_0_d1':0.6152990562577377," + + "'nvector_1_d1':-0.005263047078845837," + + "'nvector_2_d1':0.7882762026750415," + + "'nvector':'52.02471051274793, -0.49007556238612354'}"); + + assertJQ(req( + "defType", "lucene", + "q", "*:*", + "nvd","nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "fl", "dist:$nvd", + "sort" ,"$nvd asc"), + "/response/docs/[0]/dist==0.7706622667641961"); + } + + @Test + public void testNVectorRadiusFilter() throws Exception { + assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); + assertU(adoc("id", "1", "nvector", "51.927619, -0.186636")); + assertU(adoc("id", "2", "nvector", "51.480043, -0.196508")); + assertU(commit()); + assertJQ(req( + "defType", "lucene", + "q", "{!frange u=30}nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "fl", "id,dist:nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "sort","nvdist(52.01966071979866, -0.4983083573742952,nvector) asc"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==22.939789336475414" + ); + + assertJQ(req( + "defType", "lucene", + "dist","nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "q", "{!frange u=30}$dist", + "fl", "id,dist:$dist", + "sort","$dist asc"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==22.939789336475414" + ); + + assertJQ(req( + "defType", "lucene", + "lat","52.01966071979866", + "lon","-0.4983083573742952", + "dist","nvdist($lat,$lon,nvector)", + "q","*:*", + "fq", "{!frange u=30}$dist", + "fl", "id,dist:$dist", + "sort","$dist asc" + ), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==22.939789336475414" + ); + } +} diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java new file mode 100644 index 00000000000..7a3038a92dd --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -0,0 +1,65 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.util; + +import org.apache.solr.SolrTestCase; +import org.junit.Before; +import org.junit.Test; + +import java.util.Random; + +import static org.apache.solr.util.NVectorUtil.EARTH_RADIUS; + +public class FastInvTrigTest extends SolrTestCase { + + final static int num_points = 100000; + final static double EPSILON = 0.0001; + static final Random r = new Random(); + private static final double TEN_METERS = 0.01; + + static double[][] points = new double[num_points][2]; + + @Before + public void initAll() { + for (int i = 0; i < num_points; i++) { + points[i] = generateRandomPoint(); + } + } + + public static double deg2rad(double deg) { + return deg * (Math.PI / 180); + } + + public static double[] generateRandomPoint() { + double u = r.nextDouble(); + double v = r.nextDouble(); + + double latitude = deg2rad(Math.toDegrees(Math.acos(u * 2 - 1)) - 90); + double longitude = deg2rad(360 * v - 180); + return new double[]{latitude, longitude}; + } + + @Test + public void acos() { + for (double i = -1; i <= 1; i = i + 0.00001) { + assertTrue(FastInvTrig.acos(i) - Math.acos(i) <= EPSILON); + } + } + + @Test + public void dist() { + double[] a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); + + for (int i = 0; i < num_points; i++) { + double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); + double d1 = EARTH_RADIUS * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + double d2 = EARTH_RADIUS * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + assertTrue(Math.abs(d1 - d2) <= TEN_METERS); + } + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java new file mode 100644 index 00000000000..ef64d5df305 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java @@ -0,0 +1,54 @@ + +//Copyright (c) 2021, Dan Rosher +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +package org.apache.solr.util; + +import org.junit.Test; + +import java.text.DecimalFormat; + +import static org.junit.Assert.assertEquals; + +public class NVectorUtilTest { + + DecimalFormat df = new DecimalFormat("##.####"); + + @Test + public void latLongToNVector() { + double lat = 52.024535; + double lon = -0.490155; + double[] n = NVectorUtil.latLongToNVector(lat, lon); + double[] ll = NVectorUtil.NVectorToLatLong(n); + assertSimilar(lat, ll[0]); + assertSimilar(lon, ll[1]); + } + + void assertSimilar(double expected, double actual) { + assertEquals(df.format(expected), df.format(actual)); + } + + @Test + public void latLongToNVectorStr() { + String lat = "52.024535"; + String lon = "-0.490155"; + String[] n = NVectorUtil.latLongToNVector(lat, lon); + double[] ll = NVectorUtil.NVectorToLatLong(n); + assertSimilar(Double.parseDouble(lat), ll[0]); + assertSimilar(Double.parseDouble(lon), ll[1]); + } + + @Test + public void NVectorDist() { + double[] a = NVectorUtil.latLongToNVector(52.019819, -0.490155); + double[] b = NVectorUtil.latLongToNVector(52.019660, -0.498308); + double dist = NVectorUtil.NVectorDist(a, b); + assertEquals(0.5408290558849004, dist,0); + a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); + b = NVectorUtil.latLongToNVector(51.92756819110318, -0.18695373636718815); + assertEquals(22.673000657942616, NVectorUtil.NVectorDist(a, b),0); + } +} \ No newline at end of file From f2f2625c563d8cf5508991148bf76e44c55a02bb Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 13 Jul 2022 17:01:51 +0100 Subject: [PATCH 02/20] SOLR-16292 : NVector for alternative great-circle distance calculations --- .../java/org/apache/solr/schema/NVector.java | 21 +++++++++++++----- .../org/apache/solr/util/FastInvTrig.java | 21 +++++++++++++----- .../org/apache/solr/util/NVectorUtil.java | 22 ++++++++++++++----- .../function/distance/NVectorDistTest.java | 22 ++++++++++++++----- .../org/apache/solr/util/FastInvTrigTest.java | 22 ++++++++++++++----- .../org/apache/solr/util/NVectorUtilTest.java | 21 +++++++++++++----- 6 files changed, 96 insertions(+), 33 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/NVector.java b/solr/core/src/java/org/apache/solr/schema/NVector.java index 9fe44abacd2..f37651481bf 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVector.java +++ b/solr/core/src/java/org/apache/solr/schema/NVector.java @@ -1,9 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. package org.apache.solr.schema; import org.apache.lucene.document.StoredField; diff --git a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java index f6cf8fc7b73..0eb9e09dc99 100644 --- a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java +++ b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java @@ -1,9 +1,20 @@ -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.util; diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index 695ccf0424b..b4119f695c1 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -1,9 +1,19 @@ - -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.util; diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java index b8a71ae0f57..938dcb8c7ce 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java @@ -1,9 +1,19 @@ - -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.search.function.distance; diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java index 7a3038a92dd..ebe8986bccb 100644 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -1,9 +1,19 @@ - -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.util; diff --git a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java index ef64d5df305..9d82aa2214b 100644 --- a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java @@ -1,9 +1,20 @@ -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.util; From 493af2b4d5d41f66297ae4d64bc39ad949e7bf65 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 13 Jul 2022 17:03:42 +0100 Subject: [PATCH 03/20] SOLR-16292 : NVector for alternative great-circle distance calculations --- .../search/function/distance/NVector.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java index 18a4d0b94dc..b2f877a4ae3 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java @@ -1,9 +1,19 @@ - -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.search.function.distance; From fa3a33de13492d8c81ef28667ce8040a9b41a83b Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 13 Jul 2022 17:05:22 +0100 Subject: [PATCH 04/20] SOLR-16292 : NVector for alternative great-circle distance calculations --- .../distance/NVectorValueSourceParser.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java index 9cf0332d52a..4610714ffd8 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java @@ -1,9 +1,19 @@ - -//Copyright (c) 2021, Dan Rosher -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.search.function.distance; From f7ec79edcbf009338a95a48f0eb094a88aa1f92a Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Fri, 15 Jul 2022 16:44:03 +0100 Subject: [PATCH 05/20] use test random(),fix EARTH_MEAN_RADIUS_KM, rename NVector -> NvectorField and NVectorFunction to avoid confusion, use better asserts for tests --- .../{NVector.java => NVectorField.java} | 2 +- .../{NVector.java => NVectorFunction.java} | 59 ++++++++----------- .../distance/NVectorValueSourceParser.java | 11 ++-- .../org/apache/solr/util/NVectorUtil.java | 6 +- .../org/apache/solr/util/FastInvTrigTest.java | 16 +++-- .../org/apache/solr/util/NVectorUtilTest.java | 22 +++---- 6 files changed, 51 insertions(+), 65 deletions(-) rename solr/core/src/java/org/apache/solr/schema/{NVector.java => NVectorField.java} (99%) rename solr/core/src/java/org/apache/solr/search/function/distance/{NVector.java => NVectorFunction.java} (62%) diff --git a/solr/core/src/java/org/apache/solr/schema/NVector.java b/solr/core/src/java/org/apache/solr/schema/NVectorField.java similarity index 99% rename from solr/core/src/java/org/apache/solr/schema/NVector.java rename to solr/core/src/java/org/apache/solr/schema/NVectorField.java index f37651481bf..c15e9481ad8 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVector.java +++ b/solr/core/src/java/org/apache/solr/schema/NVectorField.java @@ -35,7 +35,7 @@ import java.util.List; import java.util.Map; -public class NVector extends CoordinateFieldType { +public class NVectorField extends CoordinateFieldType { @Override protected void init(IndexSchema schema, Map args) { diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java similarity index 62% rename from solr/core/src/java/org/apache/solr/search/function/distance/NVector.java rename to solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java index b2f877a4ae3..576da68b9df 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVector.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java @@ -27,19 +27,20 @@ import java.io.IOException; import java.util.Map; +import java.util.Objects; import static org.apache.solr.util.NVectorUtil.NVectorDist; -public class NVector extends ValueSource { +public class NVectorFunction extends ValueSource { - private final MultiValueSource p1; - private final MultiValueSource p2; + private final MultiValueSource nvector1; + private final MultiValueSource nvector2; private final double radius; - public NVector(MultiValueSource p1, MultiValueSource p2, double radius) { - this.p1 = p1; - this.p2 = p2; - if (p1.dimension() != 3 || p2.dimension() != 3) { + public NVectorFunction(MultiValueSource nvector1, MultiValueSource nvector2, double radius) { + this.nvector1 = nvector1; + this.nvector2 = nvector2; + if (nvector1.dimension() != 3 || nvector2.dimension() != 3) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Illegal dimension for value sources"); } this.radius = radius; @@ -48,27 +49,27 @@ public NVector(MultiValueSource p1, MultiValueSource p2, double radius) { @Override public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - final FunctionValues vals1 = p1.getValues(context, readerContext); - final FunctionValues vals2 = p2.getValues(context, readerContext); + final FunctionValues nvector_v1 = nvector1.getValues(context, readerContext); + final FunctionValues nvector_v2 = nvector2.getValues(context, readerContext); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { - double[] dv1 = new double[p1.dimension()]; - double[] dv2 = new double[p2.dimension()]; - vals1.doubleVal(doc, dv1); - vals2.doubleVal(doc, dv2); - return NVectorDist(dv1, dv2, radius); + double[] nvector_dv1 = new double[nvector1.dimension()]; + double[] nvector_dv2 = new double[nvector2.dimension()]; + nvector_v1.doubleVal(doc, nvector_dv1); + nvector_v2.doubleVal(doc, nvector_dv2); + return NVectorDist(nvector_dv1, nvector_dv2, radius); } @Override public String toString(int doc) throws IOException { return name() + ',' + - vals1.toString(doc) + + nvector_v1.toString(doc) + ',' + - vals2.toString(doc) + + nvector_v2.toString(doc) + ')'; } }; @@ -80,36 +81,28 @@ protected String name() { @Override public boolean equals(Object o) { - if (this.getClass() != o.getClass()) return false; - NVector other = (NVector) o; - return this.name() - .equals(other.name()) - && p1.equals(other.p1) && - p2.equals(other.p2) && radius == other.radius; + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NVectorFunction that = (NVectorFunction) o; + return Double.compare(that.radius, radius) == 0 && + nvector1.equals(that.nvector1) && nvector2.equals(that.nvector2); } @Override public int hashCode() { - int result; - long temp; - result = p1.hashCode(); - result = 31 * result + p2.hashCode(); - result = 31 * result + name().hashCode(); - temp = Double.doubleToRawLongBits(radius); - result = 31 * result + (int) (temp ^ (temp >>> 32)); - return result; + return Objects.hash(nvector1, nvector2, radius); } @Override public void createWeight(Map context, IndexSearcher searcher) throws IOException { - p1.createWeight(context, searcher); - p2.createWeight(context, searcher); + nvector1.createWeight(context, searcher); + nvector2.createWeight(context, searcher); } @Override public String description() { return name() + '(' + - p1 + ',' + p2 + + nvector1 + ',' + nvector2 + ')'; } } diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java index 4610714ffd8..afed202e20c 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java @@ -29,6 +29,8 @@ import java.util.Arrays; +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; + public class NVectorValueSourceParser extends ValueSourceParser { @Override public ValueSource parse(FunctionQParser fp) throws SyntaxError { @@ -39,18 +41,19 @@ public ValueSource parse(FunctionQParser fp) throws SyntaxError { if (!(vs1 instanceof MultiValueSource)) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field must a MultiValueSource"); - MultiValueSource mvs1 = (MultiValueSource) vs1; + MultiValueSource nvector_vs1 = (MultiValueSource) vs1; double[] nvector = NVectorUtil.latLongToNVector(lat, lon); - MultiValueSource mvs2 = new VectorValueSource( + + MultiValueSource nvector_vs2 = new VectorValueSource( Arrays.asList( new DoubleConstValueSource(nvector[0]), new DoubleConstValueSource(nvector[1]), new DoubleConstValueSource(nvector[2]) )); - double radius = fp.hasMoreArguments() ? fp.parseDouble() : NVectorUtil.EARTH_RADIUS; + double radius = fp.hasMoreArguments() ? fp.parseDouble() : EARTH_MEAN_RADIUS_KM; - return new NVector(mvs1, mvs2, radius); + return new NVectorFunction(nvector_vs1, nvector_vs2, radius); } } diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index b4119f695c1..18722c53d68 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -17,9 +17,9 @@ package org.apache.solr.util; -public class NVectorUtil { +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; - public static final double EARTH_RADIUS = 6173.008;//km google:standard mean earth radius; +public class NVectorUtil { public static double[] latLongToNVector(double lat, double lon) { double latRad = lat * (Math.PI / 180); @@ -51,7 +51,7 @@ public static double[] NVectorToLatLong(String[] n) { } public static double NVectorDist(double[] a, double[] b) { - return NVectorDist(a, b, EARTH_RADIUS); + return NVectorDist(a, b, EARTH_MEAN_RADIUS_KM); } public static double NVectorDist(double[] a, double[] b, double radius) { diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java index ebe8986bccb..415e5e0c94e 100644 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -21,15 +21,13 @@ import org.junit.Before; import org.junit.Test; -import java.util.Random; - -import static org.apache.solr.util.NVectorUtil.EARTH_RADIUS; +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; public class FastInvTrigTest extends SolrTestCase { final static int num_points = 100000; final static double EPSILON = 0.0001; - static final Random r = new Random(); + //static final Random r = new Random(); private static final double TEN_METERS = 0.01; static double[][] points = new double[num_points][2]; @@ -46,8 +44,8 @@ public static double deg2rad(double deg) { } public static double[] generateRandomPoint() { - double u = r.nextDouble(); - double v = r.nextDouble(); + double u = random().nextDouble(); + double v = random().nextDouble(); double latitude = deg2rad(Math.toDegrees(Math.acos(u * 2 - 1)) - 90); double longitude = deg2rad(360 * v - 180); @@ -67,9 +65,9 @@ public void dist() { for (int i = 0; i < num_points; i++) { double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); - double d1 = EARTH_RADIUS * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - double d2 = EARTH_RADIUS * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - assertTrue(Math.abs(d1 - d2) <= TEN_METERS); + double d1 = EARTH_MEAN_RADIUS_KM * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + double d2 = EARTH_MEAN_RADIUS_KM * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + assertTrue("Math.acos should be close to FastInvTrig.acos",Math.abs(d1 - d2) <= TEN_METERS); } } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java index 9d82aa2214b..54edce4aaba 100644 --- a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java @@ -20,26 +20,18 @@ import org.junit.Test; -import java.text.DecimalFormat; - import static org.junit.Assert.assertEquals; public class NVectorUtilTest { - DecimalFormat df = new DecimalFormat("##.####"); - @Test public void latLongToNVector() { double lat = 52.024535; double lon = -0.490155; double[] n = NVectorUtil.latLongToNVector(lat, lon); double[] ll = NVectorUtil.NVectorToLatLong(n); - assertSimilar(lat, ll[0]); - assertSimilar(lon, ll[1]); - } - - void assertSimilar(double expected, double actual) { - assertEquals(df.format(expected), df.format(actual)); + assertEquals(lat, ll[0],0.0001); + assertEquals(lon, ll[1],0.0001); } @Test @@ -48,8 +40,8 @@ public void latLongToNVectorStr() { String lon = "-0.490155"; String[] n = NVectorUtil.latLongToNVector(lat, lon); double[] ll = NVectorUtil.NVectorToLatLong(n); - assertSimilar(Double.parseDouble(lat), ll[0]); - assertSimilar(Double.parseDouble(lon), ll[1]); + assertEquals(Double.parseDouble(lat), ll[0],0.0001); + assertEquals(Double.parseDouble(lon), ll[1],0.0001); } @Test @@ -57,9 +49,9 @@ public void NVectorDist() { double[] a = NVectorUtil.latLongToNVector(52.019819, -0.490155); double[] b = NVectorUtil.latLongToNVector(52.019660, -0.498308); double dist = NVectorUtil.NVectorDist(a, b); - assertEquals(0.5408290558849004, dist,0); + assertEquals(0.5581762827572362, dist,0); a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); b = NVectorUtil.latLongToNVector(51.92756819110318, -0.18695373636718815); - assertEquals(22.673000657942616, NVectorUtil.NVectorDist(a, b),0); + assertEquals(23.400242809617353, NVectorUtil.NVectorDist(a, b),0); } -} \ No newline at end of file +} From cf4b4ff8b41f3335eff47e653d2d7e6796bea3c9 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Fri, 15 Jul 2022 17:21:45 +0100 Subject: [PATCH 06/20] use better asserts for tests --- solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java index 415e5e0c94e..9477d1e301f 100644 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -67,7 +67,7 @@ public void dist() { double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); double d1 = EARTH_MEAN_RADIUS_KM * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); double d2 = EARTH_MEAN_RADIUS_KM * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - assertTrue("Math.acos should be close to FastInvTrig.acos",Math.abs(d1 - d2) <= TEN_METERS); + assertEquals("Math.acos should be close to FastInvTrig.acos",d1,d2 , TEN_METERS); } } } \ No newline at end of file From 2e4069ea93ff38bf15ec03d31bd97329fd61bce2 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Fri, 15 Jul 2022 18:11:15 +0100 Subject: [PATCH 07/20] reduce schema size, fix assert values for adjusted earth radius --- .../solr/collection1/conf/schema-nvector.xml | 862 +----------------- .../function/distance/NVectorDistTest.java | 14 +- 2 files changed, 12 insertions(+), 864 deletions(-) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml index 898a7217709..d2879423926 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml @@ -37,891 +37,39 @@ - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java index 938dcb8c7ce..be0949fdc30 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java @@ -47,7 +47,7 @@ public void testNVector() throws Exception { "nvd","nvdist(52.01966071979866, -0.4983083573742952,nvector)", "fl", "dist:$nvd", "sort" ,"$nvd asc"), - "/response/docs/[0]/dist==0.7706622667641961"); + "/response/docs/[0]/dist==0.7953814512052634"); } @Test @@ -63,9 +63,9 @@ public void testNVectorRadiusFilter() throws Exception { "sort","nvdist(52.01966071979866, -0.4983083573742952,nvector) asc"), "/response/numFound==2", "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[0]/dist==0.7953814512052634", "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==22.939789336475414" + "/response/docs/[1]/dist==23.675588801562068" ); assertJQ(req( @@ -76,9 +76,9 @@ public void testNVectorRadiusFilter() throws Exception { "sort","$dist asc"), "/response/numFound==2", "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[0]/dist==0.7953814512052634", "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==22.939789336475414" + "/response/docs/[1]/dist==23.675588801562068" ); assertJQ(req( @@ -93,9 +93,9 @@ public void testNVectorRadiusFilter() throws Exception { ), "/response/numFound==2", "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7706622667641961", + "/response/docs/[0]/dist==0.7953814512052634", "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==22.939789336475414" + "/response/docs/[1]/dist==23.675588801562068" ); } } From 4a242231de219053e72644a4ab495f74ca885327 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Fri, 15 Jul 2022 18:20:19 +0100 Subject: [PATCH 08/20] gradle tidy --- .../org/apache/solr/schema/NVectorField.java | 425 +++++++++--------- .../function/distance/NVectorFunction.java | 147 +++--- .../distance/NVectorValueSourceParser.java | 38 +- .../org/apache/solr/util/FastInvTrig.java | 166 ++++--- .../org/apache/solr/util/NVectorUtil.java | 76 ++-- .../function/distance/NVectorDistTest.java | 117 ++--- .../org/apache/solr/util/FastInvTrigTest.java | 74 +-- .../org/apache/solr/util/NVectorUtilTest.java | 65 ++- 8 files changed, 554 insertions(+), 554 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/NVectorField.java b/solr/core/src/java/org/apache/solr/schema/NVectorField.java index c15e9481ad8..2866d856bc8 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/NVectorField.java @@ -17,6 +17,10 @@ package org.apache.solr.schema; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -30,234 +34,235 @@ import org.apache.solr.uninverting.UninvertingReader; import org.apache.solr.util.NVectorUtil; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - public class NVectorField extends CoordinateFieldType { - @Override - protected void init(IndexSchema schema, Map args) { - super.init(schema, args); - dimension = 3; - createSuffixCache(3); + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + dimension = 3; + createSuffixCache(3); + } + + @Override + public List createFields(SchemaField field, Object value) { + String externalVal = value.toString(); + String[] point = parseCommaSeparatedList(externalVal, dimension); + String[] nvector = NVectorUtil.latLongToNVector(point); + + List f = new ArrayList<>((dimension * 2) + 1); + + if (field.indexed()) { + for (int i = 0; i < dimension; i++) { + SchemaField sf = subField(field, i, schema); + f.addAll(sf.createFields(nvector[i])); + } } - @Override - public List createFields(SchemaField field, Object value) { - String externalVal = value.toString(); - String[] point = parseCommaSeparatedList(externalVal, dimension); - String[] nvector = NVectorUtil.latLongToNVector(point); - - List f = new ArrayList<>((dimension * 2) + 1); - - if (field.indexed()) { - for (int i = 0; i < dimension; i++) { - SchemaField sf = subField(field, i, schema); - f.addAll(sf.createFields(nvector[i])); - } - } - - if (field.stored()) { - f.add(createField(field.getName(), externalVal, StoredField.TYPE)); - } - return f; + if (field.stored()) { + f.add(createField(field.getName(), externalVal, StoredField.TYPE)); } - - @Override - public ValueSource getValueSource(SchemaField field, QParser parser) { - ArrayList vs = new ArrayList<>(dimension); - for (int i = 0; i < dimension; i++) { - SchemaField sub = subField(field, i, schema); - vs.add(sub.getType() - .getValueSource(sub, parser)); - } - return new NVectorValueSource(vs); + return f; + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser parser) { + ArrayList vs = new ArrayList<>(dimension); + for (int i = 0; i < dimension; i++) { + SchemaField sub = subField(field, i, schema); + vs.add(sub.getType().getValueSource(sub, parser)); } - - - /** - * Given a string containing dimension values encoded in it, separated by commas, - * return a String array of length dimension containing the values. - * - * @param externalVal The value to parse - * @param dimension The expected number of values for the point - * @return An array of the values that make up the point (aka vector) - * @throws SolrException if the dimension specified does not match the number found - */ - public static String[] parseCommaSeparatedList(String externalVal, int dimension) throws SolrException { - //TODO: Should we support sparse vectors? - String[] out = new String[dimension]; - int idx = externalVal.indexOf(','); - int end = idx; - int start = 0; - int i = 0; - if (idx == -1 && dimension == 1 && externalVal.length() > 0) {//we have a single point, dimension better be 1 - out[0] = externalVal.trim(); - i = 1; - } else if (idx > 0) {//if it is zero, that is an error - //Parse out a comma separated list of values, as in: 73.5,89.2,7773.4 - for (; i < dimension; i++) { - while (start < end && externalVal.charAt(start) == ' ') start++; - while (end > start && externalVal.charAt(end - 1) == ' ') end--; - if (start == end) { - break; - } - out[i] = externalVal.substring(start, end); - start = idx + 1; - end = externalVal.indexOf(',', start); - idx = end; - if (end == -1) { - end = externalVal.length(); - } - } + return new NVectorValueSource(vs); + } + + /** + * Given a string containing dimension values encoded in it, separated by commas, return a + * String array of length dimension containing the values. + * + * @param externalVal The value to parse + * @param dimension The expected number of values for the point + * @return An array of the values that make up the point (aka vector) + * @throws SolrException if the dimension specified does not match the number found + */ + public static String[] parseCommaSeparatedList(String externalVal, int dimension) + throws SolrException { + // TODO: Should we support sparse vectors? + String[] out = new String[dimension]; + int idx = externalVal.indexOf(','); + int end = idx; + int start = 0; + int i = 0; + if (idx == -1 + && dimension == 1 + && externalVal.length() > 0) { // we have a single point, dimension better be 1 + out[0] = externalVal.trim(); + i = 1; + } else if (idx > 0) { // if it is zero, that is an error + // Parse out a comma separated list of values, as in: 73.5,89.2,7773.4 + for (; i < dimension; i++) { + while (start < end && externalVal.charAt(start) == ' ') start++; + while (end > start && externalVal.charAt(end - 1) == ' ') end--; + if (start == end) { + break; } - if (i != dimension) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "incompatible dimension (" + dimension + - ") and values (" + externalVal + "). Only " + i + " values specified"); + out[i] = externalVal.substring(start, end); + start = idx + 1; + end = externalVal.indexOf(',', start); + idx = end; + if (end == -1) { + end = externalVal.length(); } - return out; + } } - - @Override - protected void checkSupportsDocValues() { - // DocValues supported only when enabled at the fieldType - if (!hasProperty(DOC_VALUES)) { - throw new UnsupportedOperationException("PointType can't have docValues=true in the field definition, use docValues=true in the fieldType definition, or in subFieldType/subFieldSuffix"); - } + if (i != dimension) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "incompatible dimension (" + + dimension + + ") and values (" + + externalVal + + "). Only " + + i + + " values specified"); } - - @Override - public UninvertingReader.Type getUninversionType(SchemaField sf) { - return null; - } - - @Override - public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { - writer.writeStr(name, f.stringValue(), true); - } - - @Override - public SortField getSortField(SchemaField field, boolean top) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Sorting not supported on NVector " + field.getName()); - } - - @Override - public boolean isPolyField() { - return true; + return out; + } + + @Override + protected void checkSupportsDocValues() { + // DocValues supported only when enabled at the fieldType + if (!hasProperty(DOC_VALUES)) { + throw new UnsupportedOperationException( + "PointType can't have docValues=true in the field definition, use docValues=true in the fieldType definition, or in subFieldType/subFieldSuffix"); } + } + + @Override + public UninvertingReader.Type getUninversionType(SchemaField sf) { + return null; + } + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + writer.writeStr(name, f.stringValue(), true); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Sorting not supported on NVector " + field.getName()); + } + + @Override + public boolean isPolyField() { + return true; + } } final class NVectorValueSource extends MultiValueSource { - private final List sources; - - public NVectorValueSource(List sources) { - this.sources = sources; + private final List sources; + + public NVectorValueSource(List sources) { + this.sources = sources; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) + throws IOException { + final FunctionValues x = sources.get(0).getValues(context, readerContext); + final FunctionValues y = sources.get(1).getValues(context, readerContext); + final FunctionValues z = sources.get(2).getValues(context, readerContext); + return new FunctionValues() { + + @Override + public void byteVal(int doc, byte[] vals) throws IOException { + vals[0] = x.byteVal(doc); + vals[1] = y.byteVal(doc); + vals[2] = z.byteVal(doc); + } + + @Override + public void shortVal(int doc, short[] vals) throws IOException { + vals[0] = x.shortVal(doc); + vals[1] = y.shortVal(doc); + vals[2] = z.shortVal(doc); + } + + @Override + public void intVal(int doc, int[] vals) throws IOException { + vals[0] = x.intVal(doc); + vals[1] = y.intVal(doc); + vals[2] = z.intVal(doc); + } + + @Override + public void longVal(int doc, long[] vals) throws IOException { + vals[0] = x.longVal(doc); + vals[1] = y.longVal(doc); + vals[2] = z.longVal(doc); + } + + @Override + public void floatVal(int doc, float[] vals) throws IOException { + vals[0] = x.floatVal(doc); + vals[1] = y.floatVal(doc); + vals[2] = z.floatVal(doc); + } + + @Override + public void doubleVal(int doc, double[] vals) throws IOException { + vals[0] = x.doubleVal(doc); + vals[1] = y.doubleVal(doc); + vals[2] = z.doubleVal(doc); + } + + @Override + public void strVal(int doc, String[] vals) throws IOException { + vals[0] = x.strVal(doc); + vals[1] = y.strVal(doc); + vals[2] = z.strVal(doc); + } + + @Override + public String toString(int doc) throws IOException { + return "nvector(" + x.toString(doc) + "," + y.toString(doc) + "," + z.toString(doc) + ")"; + } + }; + } + + @Override + public String description() { + StringBuilder sb = new StringBuilder(); + sb.append("nvector("); + boolean firstTime = true; + for (ValueSource source : sources) { + if (firstTime) { + firstTime = false; + } else { + sb.append(','); + } + sb.append(source); } + sb.append(")"); + return sb.toString(); + } - @Override - public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - final FunctionValues x = sources.get(0) - .getValues(context, readerContext); - final FunctionValues y = sources.get(1) - .getValues(context, readerContext); - final FunctionValues z = sources.get(2) - .getValues(context, readerContext); - return new FunctionValues() { - - @Override - public void byteVal(int doc, byte[] vals) throws IOException { - vals[0] = x.byteVal(doc); - vals[1] = y.byteVal(doc); - vals[2] = z.byteVal(doc); - } - - @Override - public void shortVal(int doc, short[] vals) throws IOException { - vals[0] = x.shortVal(doc); - vals[1] = y.shortVal(doc); - vals[2] = z.shortVal(doc); - } - - @Override - public void intVal(int doc, int[] vals) throws IOException { - vals[0] = x.intVal(doc); - vals[1] = y.intVal(doc); - vals[2] = z.intVal(doc); - } - - @Override - public void longVal(int doc, long[] vals) throws IOException { - vals[0] = x.longVal(doc); - vals[1] = y.longVal(doc); - vals[2] = z.longVal(doc); - } - - @Override - public void floatVal(int doc, float[] vals) throws IOException { - vals[0] = x.floatVal(doc); - vals[1] = y.floatVal(doc); - vals[2] = z.floatVal(doc); - } - - @Override - public void doubleVal(int doc, double[] vals) throws IOException { - vals[0] = x.doubleVal(doc); - vals[1] = y.doubleVal(doc); - vals[2] = z.doubleVal(doc); - } - - @Override - public void strVal(int doc, String[] vals) throws IOException { - vals[0] = x.strVal(doc); - vals[1] = y.strVal(doc); - vals[2] = z.strVal(doc); - } - - @Override - public String toString(int doc) throws IOException { - return "nvector(" + x.toString(doc) + "," + y.toString(doc) + "," + z.toString(doc) + ")"; - } - }; - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NVectorValueSource)) return false; - @Override - public String description() { - StringBuilder sb = new StringBuilder(); - sb.append("nvector("); - boolean firstTime = true; - for (ValueSource source : sources) { - if (firstTime) { - firstTime = false; - } else { - sb.append(','); - } - sb.append(source); - } - sb.append(")"); - return sb.toString(); - } + NVectorValueSource that = (NVectorValueSource) o; - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof NVectorValueSource)) return false; + return sources.equals(that.sources); + } - NVectorValueSource that = (NVectorValueSource) o; + @Override + public int hashCode() { + return sources.hashCode(); + } - return sources.equals(that.sources); - - } - - @Override - public int hashCode() { - return sources.hashCode(); - } - - @Override - public int dimension() { - return sources.size(); - } + @Override + public int dimension() { + return sources.size(); + } } diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java index 576da68b9df..72fda11b46f 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java @@ -17,6 +17,11 @@ package org.apache.solr.search.function.distance; +import static org.apache.solr.util.NVectorUtil.NVectorDist; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; @@ -25,84 +30,74 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.solr.common.SolrException; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; - -import static org.apache.solr.util.NVectorUtil.NVectorDist; - public class NVectorFunction extends ValueSource { - private final MultiValueSource nvector1; - private final MultiValueSource nvector2; - private final double radius; - - public NVectorFunction(MultiValueSource nvector1, MultiValueSource nvector2, double radius) { - this.nvector1 = nvector1; - this.nvector2 = nvector2; - if (nvector1.dimension() != 3 || nvector2.dimension() != 3) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Illegal dimension for value sources"); - } - this.radius = radius; - } - - @Override - public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { - - final FunctionValues nvector_v1 = nvector1.getValues(context, readerContext); - final FunctionValues nvector_v2 = nvector2.getValues(context, readerContext); - - return new DoubleDocValues(this) { - - @Override - public double doubleVal(int doc) throws IOException { - double[] nvector_dv1 = new double[nvector1.dimension()]; - double[] nvector_dv2 = new double[nvector2.dimension()]; - nvector_v1.doubleVal(doc, nvector_dv1); - nvector_v2.doubleVal(doc, nvector_dv2); - return NVectorDist(nvector_dv1, nvector_dv2, radius); - } - - @Override - public String toString(int doc) throws IOException { - return name() + - ',' + - nvector_v1.toString(doc) + - ',' + - nvector_v2.toString(doc) + - ')'; - } - }; - } - - protected String name() { - return "nvector"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NVectorFunction that = (NVectorFunction) o; - return Double.compare(that.radius, radius) == 0 && - nvector1.equals(that.nvector1) && nvector2.equals(that.nvector2); - } - - @Override - public int hashCode() { - return Objects.hash(nvector1, nvector2, radius); - } - - @Override - public void createWeight(Map context, IndexSearcher searcher) throws IOException { - nvector1.createWeight(context, searcher); - nvector2.createWeight(context, searcher); - } + private final MultiValueSource nvector1; + private final MultiValueSource nvector2; + private final double radius; - @Override - public String description() { - return name() + '(' + - nvector1 + ',' + nvector2 + - ')'; + public NVectorFunction(MultiValueSource nvector1, MultiValueSource nvector2, double radius) { + this.nvector1 = nvector1; + this.nvector2 = nvector2; + if (nvector1.dimension() != 3 || nvector2.dimension() != 3) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Illegal dimension for value sources"); } + this.radius = radius; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) + throws IOException { + + final FunctionValues nvector_v1 = nvector1.getValues(context, readerContext); + final FunctionValues nvector_v2 = nvector2.getValues(context, readerContext); + + return new DoubleDocValues(this) { + + @Override + public double doubleVal(int doc) throws IOException { + double[] nvector_dv1 = new double[nvector1.dimension()]; + double[] nvector_dv2 = new double[nvector2.dimension()]; + nvector_v1.doubleVal(doc, nvector_dv1); + nvector_v2.doubleVal(doc, nvector_dv2); + return NVectorDist(nvector_dv1, nvector_dv2, radius); + } + + @Override + public String toString(int doc) throws IOException { + return name() + ',' + nvector_v1.toString(doc) + ',' + nvector_v2.toString(doc) + ')'; + } + }; + } + + protected String name() { + return "nvector"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NVectorFunction that = (NVectorFunction) o; + return Double.compare(that.radius, radius) == 0 + && nvector1.equals(that.nvector1) + && nvector2.equals(that.nvector2); + } + + @Override + public int hashCode() { + return Objects.hash(nvector1, nvector2, radius); + } + + @Override + public void createWeight(Map context, IndexSearcher searcher) throws IOException { + nvector1.createWeight(context, searcher); + nvector2.createWeight(context, searcher); + } + + @Override + public String description() { + return name() + '(' + nvector1 + ',' + nvector2 + ')'; + } } diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java index afed202e20c..91317e09cee 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorValueSourceParser.java @@ -17,6 +17,9 @@ package org.apache.solr.search.function.distance; +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; + +import java.util.Arrays; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; import org.apache.lucene.queries.function.valuesource.MultiValueSource; @@ -27,33 +30,28 @@ import org.apache.solr.search.ValueSourceParser; import org.apache.solr.util.NVectorUtil; -import java.util.Arrays; - -import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; - public class NVectorValueSourceParser extends ValueSourceParser { - @Override - public ValueSource parse(FunctionQParser fp) throws SyntaxError { - double lat = fp.parseDouble(); - double lon = fp.parseDouble(); + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + double lat = fp.parseDouble(); + double lon = fp.parseDouble(); - ValueSource vs1 = fp.parseValueSource(); - if (!(vs1 instanceof MultiValueSource)) - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Field must a MultiValueSource"); - MultiValueSource nvector_vs1 = (MultiValueSource) vs1; + ValueSource vs1 = fp.parseValueSource(); + if (!(vs1 instanceof MultiValueSource)) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field must a MultiValueSource"); + MultiValueSource nvector_vs1 = (MultiValueSource) vs1; - double[] nvector = NVectorUtil.latLongToNVector(lat, lon); + double[] nvector = NVectorUtil.latLongToNVector(lat, lon); - MultiValueSource nvector_vs2 = new VectorValueSource( + MultiValueSource nvector_vs2 = + new VectorValueSource( Arrays.asList( new DoubleConstValueSource(nvector[0]), new DoubleConstValueSource(nvector[1]), - new DoubleConstValueSource(nvector[2]) - )); + new DoubleConstValueSource(nvector[2]))); - double radius = fp.hasMoreArguments() ? fp.parseDouble() : EARTH_MEAN_RADIUS_KM; + double radius = fp.hasMoreArguments() ? fp.parseDouble() : EARTH_MEAN_RADIUS_KM; - return new NVectorFunction(nvector_vs1, nvector_vs2, radius); - } + return new NVectorFunction(nvector_vs1, nvector_vs2, radius); + } } diff --git a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java index 0eb9e09dc99..69b113af9a5 100644 --- a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java +++ b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -20,89 +19,88 @@ public class FastInvTrig { - final static double pip2 = Math.PI / 2.0; - static final double SQRT2 = 1.0 / Math.sqrt(2.0); - - final static double[] TABLE; - final static int MAX_TERMS = 100; - final static int DEF_TERMS = 10; - - static { - TABLE = new double[MAX_TERMS]; - double factor = 1.0; - double divisor = 1.0; - for (int n = 0; n < MAX_TERMS; n++) { - TABLE[n] = factor / divisor; - divisor += 2; - factor *= (2 * n + 1.0) / ((n + 1) * 2); - } - } - - private static double asin2(double x, int n_terms) { - if (n_terms > MAX_TERMS) - throw new IllegalArgumentException("Too many terms"); - double acc = x; - double tempExp = x; - double x2 = x * x; - for (int n = 1; n < n_terms; n++) { - tempExp *= x2; - acc += TABLE[n] * tempExp; - } - return acc; - } - - // split domain for faster convergence i.e. fewer maclaurin terms required - // see https://www.wolframalpha.com/input/?i=arcsin%28sqrt%281-x%5E2%29%29-acos%28x%29 for x > 0 - // arcsin(sqrt(1-x^2)) = acos(x) for x > 0 - // arcsin(sqrt(1-x^2)) = acos(x) = pi/2 - arcsin(x) for x > 0 - // arcsin(sqrt(1-x^2)) = pi/2 - arcsin(x) for x > 0 - // arcsin(x) = pi/2 - arcsin(sqrt(1-x^2)) for x > 0 .... 1 - // - // see https://www.wolframalpha.com/input/?i=arcsin%28x%29+-+arcsin%28sqrt%281-x%5E2%29%29+%2B+pi%2F2+ for x < 0 - // arcsin(sqrt(1-x^2)) = pi-acos(x) for x < 0 - // arcsin(sqrt(1-x^2)) = pi-(pi/2 - arcsin(x)) for x < 0 - // arcsin(sqrt(1-x^2)) = pi/2 + arcsin(x) for x < 0 - // arcsin(x) = arcsin(sqrt(1-x^2)) - pi/2 for x < 0 .... 2 - - // within domain [-SQRT2 <= x <= SQRT2] use arcsin(x). outside this range use formulae above. - // so that convergence is faster - //This way we can transform input x into [-1/sqrt(2),1/sqrt(2)], where convergence is relatively fast. - - public static double asin(double x, int n_terms) { - if (x > SQRT2) - return pip2 - asin2(Math.sqrt(1 - (x * x)), n_terms); - else if (Math.abs(x) <= SQRT2) - return asin2(x, n_terms); - return asin2(Math.sqrt(1 - (x * x)), n_terms) - pip2; + static final double pip2 = Math.PI / 2.0; + static final double SQRT2 = 1.0 / Math.sqrt(2.0); + + static final double[] TABLE; + static final int MAX_TERMS = 100; + static final int DEF_TERMS = 10; + + static { + TABLE = new double[MAX_TERMS]; + double factor = 1.0; + double divisor = 1.0; + for (int n = 0; n < MAX_TERMS; n++) { + TABLE[n] = factor / divisor; + divisor += 2; + factor *= (2 * n + 1.0) / ((n + 1) * 2); } - - public static double asin(double x) { - return asin(x, DEF_TERMS); - } - - public static double acos(double x, int n_terms) { - return Math.abs(x) <= SQRT2 ? pip2 - asin2(x, n_terms) : asin2(Math.sqrt(1 - (x * x)), n_terms); - } - - public static double acos(double x) { - return acos(x, DEF_TERMS); + } + + private static double asin2(double x, int n_terms) { + if (n_terms > MAX_TERMS) throw new IllegalArgumentException("Too many terms"); + double acc = x; + double tempExp = x; + double x2 = x * x; + for (int n = 1; n < n_terms; n++) { + tempExp *= x2; + acc += TABLE[n] * tempExp; } - - //Following for completion for Inverse trigonometric functions - public static double atan(double x) { - return asin(x / Math.sqrt(1 + x * x)); - } - - public static double acot(double x) { - return pip2 - atan(x); - } - - public static double asec(double x) { - return acos(1 / x); - } - - public static double acsc(double x) { - return pip2 - asec(x); - } - + return acc; + } + + // split domain for faster convergence i.e. fewer maclaurin terms required + // see https://www.wolframalpha.com/input/?i=arcsin%28sqrt%281-x%5E2%29%29-acos%28x%29 for x > 0 + // arcsin(sqrt(1-x^2)) = acos(x) for x > 0 + // arcsin(sqrt(1-x^2)) = acos(x) = pi/2 - arcsin(x) for x > 0 + // arcsin(sqrt(1-x^2)) = pi/2 - arcsin(x) for x > 0 + // arcsin(x) = pi/2 - arcsin(sqrt(1-x^2)) for x > 0 .... 1 + // + // see + // https://www.wolframalpha.com/input/?i=arcsin%28x%29+-+arcsin%28sqrt%281-x%5E2%29%29+%2B+pi%2F2+ + // for x < 0 + // arcsin(sqrt(1-x^2)) = pi-acos(x) for x < 0 + // arcsin(sqrt(1-x^2)) = pi-(pi/2 - arcsin(x)) for x < 0 + // arcsin(sqrt(1-x^2)) = pi/2 + arcsin(x) for x < 0 + // arcsin(x) = arcsin(sqrt(1-x^2)) - pi/2 for x < 0 .... 2 + + // within domain [-SQRT2 <= x <= SQRT2] use arcsin(x). outside this range use formulae above. + // so that convergence is faster + // This way we can transform input x into [-1/sqrt(2),1/sqrt(2)], where convergence is relatively + // fast. + + public static double asin(double x, int n_terms) { + if (x > SQRT2) return pip2 - asin2(Math.sqrt(1 - (x * x)), n_terms); + else if (Math.abs(x) <= SQRT2) return asin2(x, n_terms); + return asin2(Math.sqrt(1 - (x * x)), n_terms) - pip2; + } + + public static double asin(double x) { + return asin(x, DEF_TERMS); + } + + public static double acos(double x, int n_terms) { + return Math.abs(x) <= SQRT2 ? pip2 - asin2(x, n_terms) : asin2(Math.sqrt(1 - (x * x)), n_terms); + } + + public static double acos(double x) { + return acos(x, DEF_TERMS); + } + + // Following for completion for Inverse trigonometric functions + public static double atan(double x) { + return asin(x / Math.sqrt(1 + x * x)); + } + + public static double acot(double x) { + return pip2 - atan(x); + } + + public static double asec(double x) { + return acos(1 / x); + } + + public static double acsc(double x) { + return pip2 - asec(x); + } } diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index 18722c53d68..615c32c7f04 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -21,40 +21,44 @@ public class NVectorUtil { - public static double[] latLongToNVector(double lat, double lon) { - double latRad = lat * (Math.PI / 180); - double lonRad = lon * (Math.PI / 180); - double x = Math.cos(latRad) * Math.cos(lonRad); - double y = Math.cos(latRad) * Math.sin(lonRad); - double z = Math.sin(latRad); - return new double[]{x, y, z}; - } - - public static String[] latLongToNVector(String lat, String lon) { - double[] nvec = latLongToNVector(Double.parseDouble(lat), Double.parseDouble(lon)); - return new String[]{Double.toString(nvec[0]), Double.toString(nvec[1]), Double.toString(nvec[2])}; - } - - public static String[] latLongToNVector(String[] latlon) { - return latLongToNVector(latlon[0], latlon[1]); - } - - public static double[] NVectorToLatLong(double[] n) { - return new double[]{Math.asin(n[2]) * (180 / Math.PI), Math.atan(n[1] / n[0]) * (180 / Math.PI)}; - } - - public static double[] NVectorToLatLong(String[] n) { - return NVectorToLatLong(new double[]{ - Double.parseDouble(n[0]), - Double.parseDouble(n[1]), - Double.parseDouble(n[2])}); - } - - public static double NVectorDist(double[] a, double[] b) { - return NVectorDist(a, b, EARTH_MEAN_RADIUS_KM); - } - - public static double NVectorDist(double[] a, double[] b, double radius) { - return radius * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - } + public static double[] latLongToNVector(double lat, double lon) { + double latRad = lat * (Math.PI / 180); + double lonRad = lon * (Math.PI / 180); + double x = Math.cos(latRad) * Math.cos(lonRad); + double y = Math.cos(latRad) * Math.sin(lonRad); + double z = Math.sin(latRad); + return new double[] {x, y, z}; + } + + public static String[] latLongToNVector(String lat, String lon) { + double[] nvec = latLongToNVector(Double.parseDouble(lat), Double.parseDouble(lon)); + return new String[] { + Double.toString(nvec[0]), Double.toString(nvec[1]), Double.toString(nvec[2]) + }; + } + + public static String[] latLongToNVector(String[] latlon) { + return latLongToNVector(latlon[0], latlon[1]); + } + + public static double[] NVectorToLatLong(double[] n) { + return new double[] { + Math.asin(n[2]) * (180 / Math.PI), Math.atan(n[1] / n[0]) * (180 / Math.PI) + }; + } + + public static double[] NVectorToLatLong(String[] n) { + return NVectorToLatLong( + new double[] { + Double.parseDouble(n[0]), Double.parseDouble(n[1]), Double.parseDouble(n[2]) + }); + } + + public static double NVectorDist(double[] a, double[] b) { + return NVectorDist(a, b, EARTH_MEAN_RADIUS_KM); + } + + public static double NVectorDist(double[] a, double[] b, double radius) { + return radius * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + } } diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java index be0949fdc30..685f17173a1 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java @@ -23,79 +23,80 @@ public class NVectorDistTest extends SolrTestCaseJ4 { - @BeforeClass - public static void beforeClass() throws Exception { - System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ - initCore("solrconfig-nvector.xml", "schema-nvector.xml"); - } + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + initCore("solrconfig-nvector.xml", "schema-nvector.xml"); + } - @Test - public void testNVector() throws Exception { - assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); - assertU(commit()); - assertJQ(req("defType", "lucene", "q", "*:*", "fl", "id,nvector*","sort","id asc"), - "/response/docs/[0]== {" + - "'id':'0'," + - "'nvector_0_d1':0.6152990562577377," + - "'nvector_1_d1':-0.005263047078845837," + - "'nvector_2_d1':0.7882762026750415," + - "'nvector':'52.02471051274793, -0.49007556238612354'}"); + @Test + public void testNVector() throws Exception { + assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); + assertU(commit()); + assertJQ( + req("defType", "lucene", "q", "*:*", "fl", "id,nvector*", "sort", "id asc"), + "/response/docs/[0]== {" + + "'id':'0'," + + "'nvector_0_d1':0.6152990562577377," + + "'nvector_1_d1':-0.005263047078845837," + + "'nvector_2_d1':0.7882762026750415," + + "'nvector':'52.02471051274793, -0.49007556238612354'}"); - assertJQ(req( + assertJQ( + req( "defType", "lucene", "q", "*:*", - "nvd","nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "nvd", "nvdist(52.01966071979866, -0.4983083573742952,nvector)", "fl", "dist:$nvd", - "sort" ,"$nvd asc"), - "/response/docs/[0]/dist==0.7953814512052634"); - } + "sort", "$nvd asc"), + "/response/docs/[0]/dist==0.7953814512052634"); + } - @Test - public void testNVectorRadiusFilter() throws Exception { - assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); - assertU(adoc("id", "1", "nvector", "51.927619, -0.186636")); - assertU(adoc("id", "2", "nvector", "51.480043, -0.196508")); - assertU(commit()); - assertJQ(req( + @Test + public void testNVectorRadiusFilter() throws Exception { + assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); + assertU(adoc("id", "1", "nvector", "51.927619, -0.186636")); + assertU(adoc("id", "2", "nvector", "51.480043, -0.196508")); + assertU(commit()); + assertJQ( + req( "defType", "lucene", "q", "{!frange u=30}nvdist(52.01966071979866, -0.4983083573742952,nvector)", "fl", "id,dist:nvdist(52.01966071979866, -0.4983083573742952,nvector)", - "sort","nvdist(52.01966071979866, -0.4983083573742952,nvector) asc"), - "/response/numFound==2", - "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7953814512052634", - "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==23.675588801562068" - ); + "sort", "nvdist(52.01966071979866, -0.4983083573742952,nvector) asc"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7953814512052634", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==23.675588801562068"); - assertJQ(req( + assertJQ( + req( "defType", "lucene", - "dist","nvdist(52.01966071979866, -0.4983083573742952,nvector)", + "dist", "nvdist(52.01966071979866, -0.4983083573742952,nvector)", "q", "{!frange u=30}$dist", "fl", "id,dist:$dist", - "sort","$dist asc"), - "/response/numFound==2", - "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7953814512052634", - "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==23.675588801562068" - ); + "sort", "$dist asc"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7953814512052634", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==23.675588801562068"); - assertJQ(req( + assertJQ( + req( "defType", "lucene", - "lat","52.01966071979866", - "lon","-0.4983083573742952", - "dist","nvdist($lat,$lon,nvector)", - "q","*:*", + "lat", "52.01966071979866", + "lon", "-0.4983083573742952", + "dist", "nvdist($lat,$lon,nvector)", + "q", "*:*", "fq", "{!frange u=30}$dist", "fl", "id,dist:$dist", - "sort","$dist asc" - ), - "/response/numFound==2", - "/response/docs/[0]/id=='0'", - "/response/docs/[0]/dist==0.7953814512052634", - "/response/docs/[1]/id=='1'", - "/response/docs/[1]/dist==23.675588801562068" - ); - } + "sort", "$dist asc"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7953814512052634", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==23.675588801562068"); + } } diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java index 9477d1e301f..8adf29e17d7 100644 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -17,57 +17,57 @@ package org.apache.solr.util; +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; + import org.apache.solr.SolrTestCase; import org.junit.Before; import org.junit.Test; -import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; - public class FastInvTrigTest extends SolrTestCase { - final static int num_points = 100000; - final static double EPSILON = 0.0001; - //static final Random r = new Random(); - private static final double TEN_METERS = 0.01; + static final int num_points = 100000; + static final double EPSILON = 0.0001; + // static final Random r = new Random(); + private static final double TEN_METERS = 0.01; - static double[][] points = new double[num_points][2]; + static double[][] points = new double[num_points][2]; - @Before - public void initAll() { - for (int i = 0; i < num_points; i++) { - points[i] = generateRandomPoint(); - } + @Before + public void initAll() { + for (int i = 0; i < num_points; i++) { + points[i] = generateRandomPoint(); } + } - public static double deg2rad(double deg) { - return deg * (Math.PI / 180); - } + public static double deg2rad(double deg) { + return deg * (Math.PI / 180); + } - public static double[] generateRandomPoint() { - double u = random().nextDouble(); - double v = random().nextDouble(); + public static double[] generateRandomPoint() { + double u = random().nextDouble(); + double v = random().nextDouble(); - double latitude = deg2rad(Math.toDegrees(Math.acos(u * 2 - 1)) - 90); - double longitude = deg2rad(360 * v - 180); - return new double[]{latitude, longitude}; - } + double latitude = deg2rad(Math.toDegrees(Math.acos(u * 2 - 1)) - 90); + double longitude = deg2rad(360 * v - 180); + return new double[] {latitude, longitude}; + } - @Test - public void acos() { - for (double i = -1; i <= 1; i = i + 0.00001) { - assertTrue(FastInvTrig.acos(i) - Math.acos(i) <= EPSILON); - } + @Test + public void acos() { + for (double i = -1; i <= 1; i = i + 0.00001) { + assertTrue(FastInvTrig.acos(i) - Math.acos(i) <= EPSILON); } + } - @Test - public void dist() { - double[] a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); + @Test + public void dist() { + double[] a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); - for (int i = 0; i < num_points; i++) { - double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); - double d1 = EARTH_MEAN_RADIUS_KM * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - double d2 = EARTH_MEAN_RADIUS_KM * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - assertEquals("Math.acos should be close to FastInvTrig.acos",d1,d2 , TEN_METERS); - } + for (int i = 0; i < num_points; i++) { + double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); + double d1 = EARTH_MEAN_RADIUS_KM * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + double d2 = EARTH_MEAN_RADIUS_KM * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + assertEquals("Math.acos should be close to FastInvTrig.acos", d1, d2, TEN_METERS); } -} \ No newline at end of file + } +} diff --git a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java index 54edce4aaba..f6c80943a05 100644 --- a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -18,40 +17,40 @@ package org.apache.solr.util; -import org.junit.Test; - import static org.junit.Assert.assertEquals; -public class NVectorUtilTest { - - @Test - public void latLongToNVector() { - double lat = 52.024535; - double lon = -0.490155; - double[] n = NVectorUtil.latLongToNVector(lat, lon); - double[] ll = NVectorUtil.NVectorToLatLong(n); - assertEquals(lat, ll[0],0.0001); - assertEquals(lon, ll[1],0.0001); - } +import org.junit.Test; - @Test - public void latLongToNVectorStr() { - String lat = "52.024535"; - String lon = "-0.490155"; - String[] n = NVectorUtil.latLongToNVector(lat, lon); - double[] ll = NVectorUtil.NVectorToLatLong(n); - assertEquals(Double.parseDouble(lat), ll[0],0.0001); - assertEquals(Double.parseDouble(lon), ll[1],0.0001); - } +public class NVectorUtilTest { - @Test - public void NVectorDist() { - double[] a = NVectorUtil.latLongToNVector(52.019819, -0.490155); - double[] b = NVectorUtil.latLongToNVector(52.019660, -0.498308); - double dist = NVectorUtil.NVectorDist(a, b); - assertEquals(0.5581762827572362, dist,0); - a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); - b = NVectorUtil.latLongToNVector(51.92756819110318, -0.18695373636718815); - assertEquals(23.400242809617353, NVectorUtil.NVectorDist(a, b),0); - } + @Test + public void latLongToNVector() { + double lat = 52.024535; + double lon = -0.490155; + double[] n = NVectorUtil.latLongToNVector(lat, lon); + double[] ll = NVectorUtil.NVectorToLatLong(n); + assertEquals(lat, ll[0], 0.0001); + assertEquals(lon, ll[1], 0.0001); + } + + @Test + public void latLongToNVectorStr() { + String lat = "52.024535"; + String lon = "-0.490155"; + String[] n = NVectorUtil.latLongToNVector(lat, lon); + double[] ll = NVectorUtil.NVectorToLatLong(n); + assertEquals(Double.parseDouble(lat), ll[0], 0.0001); + assertEquals(Double.parseDouble(lon), ll[1], 0.0001); + } + + @Test + public void NVectorDist() { + double[] a = NVectorUtil.latLongToNVector(52.019819, -0.490155); + double[] b = NVectorUtil.latLongToNVector(52.019660, -0.498308); + double dist = NVectorUtil.NVectorDist(a, b); + assertEquals(0.5581762827572362, dist, 0); + a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); + b = NVectorUtil.latLongToNVector(51.92756819110318, -0.18695373636718815); + assertEquals(23.400242809617353, NVectorUtil.NVectorDist(a, b), 0); + } } From acd5d108781a3db18a2df3bd94682577cd5e25f6 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 20 Jul 2022 15:03:31 +0100 Subject: [PATCH 09/20] add separator and use default locale to parse doubles --- .../org/apache/solr/schema/NVectorField.java | 440 +++++++++--------- .../org/apache/solr/util/NVectorUtil.java | 12 + 2 files changed, 241 insertions(+), 211 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/NVectorField.java b/solr/core/src/java/org/apache/solr/schema/NVectorField.java index 2866d856bc8..b4558c348a9 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/NVectorField.java @@ -18,9 +18,14 @@ package org.apache.solr.schema; import java.io.IOException; +import java.math.RoundingMode; +import java.text.NumberFormat; +import java.text.ParseException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; + import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -36,233 +41,246 @@ public class NVectorField extends CoordinateFieldType { - @Override - protected void init(IndexSchema schema, Map args) { - super.init(schema, args); - dimension = 3; - createSuffixCache(3); - } - - @Override - public List createFields(SchemaField field, Object value) { - String externalVal = value.toString(); - String[] point = parseCommaSeparatedList(externalVal, dimension); - String[] nvector = NVectorUtil.latLongToNVector(point); - - List f = new ArrayList<>((dimension * 2) + 1); - - if (field.indexed()) { - for (int i = 0; i < dimension; i++) { - SchemaField sf = subField(field, i, schema); - f.addAll(sf.createFields(nvector[i])); - } + String DEFAULT_SEPARATOR = ","; + String separator = DEFAULT_SEPARATOR; + + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + separator = args.getOrDefault("separator", DEFAULT_SEPARATOR); + dimension = 3; + createSuffixCache(3); } - if (field.stored()) { - f.add(createField(field.getName(), externalVal, StoredField.TYPE)); + @Override + public List createFields(SchemaField field, Object value) { + String externalVal = value.toString(); + String[] point = parseCommaSeparatedList(externalVal, dimension, separator); + String[] nvector; + try { + NumberFormat format = NumberFormat.getInstance(Locale.getDefault()); + format.setParseIntegerOnly(false); + nvector = NVectorUtil.latLongToNVector(point, format); + } catch (ParseException e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "format exception parsing: "+externalVal); + } + List f = new ArrayList<>((dimension * 2) + 1); + + if (field.indexed()) { + for (int i = 0; i < dimension; i++) { + SchemaField sf = subField(field, i, schema); + f.addAll(sf.createFields(nvector[i])); + } + } + + if (field.stored()) { + f.add(createField(field.getName(), externalVal, StoredField.TYPE)); + } + return f; } - return f; - } - - @Override - public ValueSource getValueSource(SchemaField field, QParser parser) { - ArrayList vs = new ArrayList<>(dimension); - for (int i = 0; i < dimension; i++) { - SchemaField sub = subField(field, i, schema); - vs.add(sub.getType().getValueSource(sub, parser)); + + @Override + public ValueSource getValueSource(SchemaField field, QParser parser) { + ArrayList vs = new ArrayList<>(dimension); + for (int i = 0; i < dimension; i++) { + SchemaField sub = subField(field, i, schema); + vs.add(sub.getType().getValueSource(sub, parser)); + } + return new NVectorValueSource(vs); } - return new NVectorValueSource(vs); - } - - /** - * Given a string containing dimension values encoded in it, separated by commas, return a - * String array of length dimension containing the values. - * - * @param externalVal The value to parse - * @param dimension The expected number of values for the point - * @return An array of the values that make up the point (aka vector) - * @throws SolrException if the dimension specified does not match the number found - */ - public static String[] parseCommaSeparatedList(String externalVal, int dimension) - throws SolrException { - // TODO: Should we support sparse vectors? - String[] out = new String[dimension]; - int idx = externalVal.indexOf(','); - int end = idx; - int start = 0; - int i = 0; - if (idx == -1 - && dimension == 1 - && externalVal.length() > 0) { // we have a single point, dimension better be 1 - out[0] = externalVal.trim(); - i = 1; - } else if (idx > 0) { // if it is zero, that is an error - // Parse out a comma separated list of values, as in: 73.5,89.2,7773.4 - for (; i < dimension; i++) { - while (start < end && externalVal.charAt(start) == ' ') start++; - while (end > start && externalVal.charAt(end - 1) == ' ') end--; - if (start == end) { - break; + + /** + * Given a string containing dimension values encoded in it, separated by commas, return a + * String array of length dimension containing the values. + * + * @param externalVal The value to parse + * @param dimension The expected number of values for the point + * @param separator The separator between values + * @return An array of the values that make up the point (aka vector) + * @throws SolrException if the dimension specified does not match the number found + */ + public static String[] parseCommaSeparatedList(String externalVal, int dimension, String separator) + throws SolrException { + char sep = separator.charAt(0); + String[] out = new String[dimension]; + int idx = externalVal.indexOf(sep); + int end = idx; + int start = 0; + int i = 0; + if (idx == -1 + && dimension == 1 + && externalVal.length() > 0) { // we have a single point, dimension better be 1 + out[0] = externalVal.trim(); + i = 1; + } else if (idx > 0) { // if it is zero, that is an error + // Parse out a comma separated list of values, as in: 73.5,89.2,7773.4 + for (; i < dimension; i++) { + while (start < end && externalVal.charAt(start) == ' ') start++; + while (end > start && externalVal.charAt(end - 1) == ' ') end--; + if (start == end) { + break; + } + out[i] = externalVal.substring(start, end); + start = idx + 1; + end = externalVal.indexOf(sep, start); + idx = end; + if (end == -1) { + end = externalVal.length(); + } + } } - out[i] = externalVal.substring(start, end); - start = idx + 1; - end = externalVal.indexOf(',', start); - idx = end; - if (end == -1) { - end = externalVal.length(); + if (i != dimension) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "incompatible dimension (" + + dimension + + ") and values (" + + externalVal + + "). Only " + + i + + " values specified"); } - } + return out; + } + + @Override + protected void checkSupportsDocValues() { + // DocValues supported only when enabled at the fieldType + if (!hasProperty(DOC_VALUES)) { + throw new UnsupportedOperationException( + "PointType can't have docValues=true in the field definition, use docValues=true in the fieldType definition, or in subFieldType/subFieldSuffix"); + } + } + + @Override + public UninvertingReader.Type getUninversionType(SchemaField sf) { + return null; } - if (i != dimension) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "incompatible dimension (" - + dimension - + ") and values (" - + externalVal - + "). Only " - + i - + " values specified"); + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + writer.writeStr(name, f.stringValue(), true); } - return out; - } - - @Override - protected void checkSupportsDocValues() { - // DocValues supported only when enabled at the fieldType - if (!hasProperty(DOC_VALUES)) { - throw new UnsupportedOperationException( - "PointType can't have docValues=true in the field definition, use docValues=true in the fieldType definition, or in subFieldType/subFieldSuffix"); + + @Override + public SortField getSortField(SchemaField field, boolean top) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Sorting not supported on NVector " + field.getName()); + } + + @Override + public boolean isPolyField() { + return true; } - } - - @Override - public UninvertingReader.Type getUninversionType(SchemaField sf) { - return null; - } - - @Override - public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { - writer.writeStr(name, f.stringValue(), true); - } - - @Override - public SortField getSortField(SchemaField field, boolean top) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Sorting not supported on NVector " + field.getName()); - } - - @Override - public boolean isPolyField() { - return true; - } } final class NVectorValueSource extends MultiValueSource { - private final List sources; - - public NVectorValueSource(List sources) { - this.sources = sources; - } - - @Override - public FunctionValues getValues(Map context, LeafReaderContext readerContext) - throws IOException { - final FunctionValues x = sources.get(0).getValues(context, readerContext); - final FunctionValues y = sources.get(1).getValues(context, readerContext); - final FunctionValues z = sources.get(2).getValues(context, readerContext); - return new FunctionValues() { - - @Override - public void byteVal(int doc, byte[] vals) throws IOException { - vals[0] = x.byteVal(doc); - vals[1] = y.byteVal(doc); - vals[2] = z.byteVal(doc); - } - - @Override - public void shortVal(int doc, short[] vals) throws IOException { - vals[0] = x.shortVal(doc); - vals[1] = y.shortVal(doc); - vals[2] = z.shortVal(doc); - } - - @Override - public void intVal(int doc, int[] vals) throws IOException { - vals[0] = x.intVal(doc); - vals[1] = y.intVal(doc); - vals[2] = z.intVal(doc); - } - - @Override - public void longVal(int doc, long[] vals) throws IOException { - vals[0] = x.longVal(doc); - vals[1] = y.longVal(doc); - vals[2] = z.longVal(doc); - } - - @Override - public void floatVal(int doc, float[] vals) throws IOException { - vals[0] = x.floatVal(doc); - vals[1] = y.floatVal(doc); - vals[2] = z.floatVal(doc); - } - - @Override - public void doubleVal(int doc, double[] vals) throws IOException { - vals[0] = x.doubleVal(doc); - vals[1] = y.doubleVal(doc); - vals[2] = z.doubleVal(doc); - } - - @Override - public void strVal(int doc, String[] vals) throws IOException { - vals[0] = x.strVal(doc); - vals[1] = y.strVal(doc); - vals[2] = z.strVal(doc); - } - - @Override - public String toString(int doc) throws IOException { - return "nvector(" + x.toString(doc) + "," + y.toString(doc) + "," + z.toString(doc) + ")"; - } - }; - } - - @Override - public String description() { - StringBuilder sb = new StringBuilder(); - sb.append("nvector("); - boolean firstTime = true; - for (ValueSource source : sources) { - if (firstTime) { - firstTime = false; - } else { - sb.append(','); - } - sb.append(source); + private final List sources; + + public NVectorValueSource(List sources) { + this.sources = sources; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) + throws IOException { + final FunctionValues x = sources.get(0).getValues(context, readerContext); + final FunctionValues y = sources.get(1).getValues(context, readerContext); + final FunctionValues z = sources.get(2).getValues(context, readerContext); + return new FunctionValues() { + + @Override + public void byteVal(int doc, byte[] vals) throws IOException { + vals[0] = x.byteVal(doc); + vals[1] = y.byteVal(doc); + vals[2] = z.byteVal(doc); + } + + @Override + public void shortVal(int doc, short[] vals) throws IOException { + vals[0] = x.shortVal(doc); + vals[1] = y.shortVal(doc); + vals[2] = z.shortVal(doc); + } + + @Override + public void intVal(int doc, int[] vals) throws IOException { + vals[0] = x.intVal(doc); + vals[1] = y.intVal(doc); + vals[2] = z.intVal(doc); + } + + @Override + public void longVal(int doc, long[] vals) throws IOException { + vals[0] = x.longVal(doc); + vals[1] = y.longVal(doc); + vals[2] = z.longVal(doc); + } + + @Override + public void floatVal(int doc, float[] vals) throws IOException { + vals[0] = x.floatVal(doc); + vals[1] = y.floatVal(doc); + vals[2] = z.floatVal(doc); + } + + @Override + public void doubleVal(int doc, double[] vals) throws IOException { + vals[0] = x.doubleVal(doc); + vals[1] = y.doubleVal(doc); + vals[2] = z.doubleVal(doc); + } + + @Override + public void strVal(int doc, String[] vals) throws IOException { + vals[0] = x.strVal(doc); + vals[1] = y.strVal(doc); + vals[2] = z.strVal(doc); + } + + @Override + public String toString(int doc) throws IOException { + return "nvector(" + x.toString(doc) + "," + y.toString(doc) + "," + z.toString(doc) + ")"; + } + }; } - sb.append(")"); - return sb.toString(); - } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof NVectorValueSource)) return false; + @Override + public String description() { + StringBuilder sb = new StringBuilder(); + sb.append("nvector("); + boolean firstTime = true; + for (ValueSource source : sources) { + if (firstTime) { + firstTime = false; + } else { + sb.append(','); + } + sb.append(source); + } + sb.append(")"); + return sb.toString(); + } - NVectorValueSource that = (NVectorValueSource) o; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NVectorValueSource)) return false; - return sources.equals(that.sources); - } + NVectorValueSource that = (NVectorValueSource) o; - @Override - public int hashCode() { - return sources.hashCode(); - } + return sources.equals(that.sources); + } - @Override - public int dimension() { - return sources.size(); - } + @Override + public int hashCode() { + return sources.hashCode(); + } + + @Override + public int dimension() { + return sources.size(); + } } diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index 615c32c7f04..7e60d1dddf8 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -17,6 +17,9 @@ package org.apache.solr.util; +import java.text.NumberFormat; +import java.text.ParseException; + import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; public class NVectorUtil { @@ -41,6 +44,13 @@ public static String[] latLongToNVector(String[] latlon) { return latLongToNVector(latlon[0], latlon[1]); } + public static String[] latLongToNVector(String[] point, NumberFormat formatter) throws ParseException { + double[] nvec = latLongToNVector(formatter.parse(point[0]).doubleValue(),formatter.parse(point[1]).doubleValue()); + return new String[] { + Double.toString(nvec[0]), Double.toString(nvec[1]), Double.toString(nvec[2]) + }; + } + public static double[] NVectorToLatLong(double[] n) { return new double[] { Math.asin(n[2]) * (180 / Math.PI), Math.atan(n[1] / n[0]) * (180 / Math.PI) @@ -61,4 +71,6 @@ public static double NVectorDist(double[] a, double[] b) { public static double NVectorDist(double[] a, double[] b, double radius) { return radius * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); } + + } From 4ace4cc9b08127fa44385aced64a7099acd4df5f Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 20 Jul 2022 15:18:42 +0100 Subject: [PATCH 10/20] tidy up --- .../solr/collection1/conf/schema-nvector.xml | 25 +++---------------- .../org/apache/solr/util/FastInvTrigTest.java | 1 - 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml index d2879423926..4cfc2fb494b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-nvector.xml @@ -38,39 +38,20 @@ - - - - - - - - - + - - - - + + - - - - - - - - id - I am your default sim diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java index 8adf29e17d7..d080980849b 100644 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java @@ -27,7 +27,6 @@ public class FastInvTrigTest extends SolrTestCase { static final int num_points = 100000; static final double EPSILON = 0.0001; - // static final Random r = new Random(); private static final double TEN_METERS = 0.01; static double[][] points = new double[num_points][2]; From 90a9173e95c65db48e43f9df4864b64dadfadcf4 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Wed, 20 Jul 2022 18:27:05 +0100 Subject: [PATCH 11/20] remove unused import --- .../org/apache/solr/schema/NVectorField.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/NVectorField.java b/solr/core/src/java/org/apache/solr/schema/NVectorField.java index b4558c348a9..591e928d7bc 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/NVectorField.java @@ -17,15 +17,6 @@ package org.apache.solr.schema; -import java.io.IOException; -import java.math.RoundingMode; -import java.text.NumberFormat; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; - import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -39,6 +30,14 @@ import org.apache.solr.uninverting.UninvertingReader; import org.apache.solr.util.NVectorUtil; +import java.io.IOException; +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + public class NVectorField extends CoordinateFieldType { String DEFAULT_SEPARATOR = ","; From 7168e5689f864d4c25a523883d8cd176516c8e80 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Thu, 28 Jul 2022 10:01:29 +0100 Subject: [PATCH 12/20] remove FastInvTrig as SloppyMath.asin faster at 40cm resolution --- .../org/apache/solr/util/FastInvTrig.java | 106 ------------------ .../org/apache/solr/util/FastInvTrigTest.java | 72 ------------ 2 files changed, 178 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/util/FastInvTrig.java delete mode 100644 solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java diff --git a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java b/solr/core/src/java/org/apache/solr/util/FastInvTrig.java deleted file mode 100644 index 69b113af9a5..00000000000 --- a/solr/core/src/java/org/apache/solr/util/FastInvTrig.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util; - -public class FastInvTrig { - - static final double pip2 = Math.PI / 2.0; - static final double SQRT2 = 1.0 / Math.sqrt(2.0); - - static final double[] TABLE; - static final int MAX_TERMS = 100; - static final int DEF_TERMS = 10; - - static { - TABLE = new double[MAX_TERMS]; - double factor = 1.0; - double divisor = 1.0; - for (int n = 0; n < MAX_TERMS; n++) { - TABLE[n] = factor / divisor; - divisor += 2; - factor *= (2 * n + 1.0) / ((n + 1) * 2); - } - } - - private static double asin2(double x, int n_terms) { - if (n_terms > MAX_TERMS) throw new IllegalArgumentException("Too many terms"); - double acc = x; - double tempExp = x; - double x2 = x * x; - for (int n = 1; n < n_terms; n++) { - tempExp *= x2; - acc += TABLE[n] * tempExp; - } - return acc; - } - - // split domain for faster convergence i.e. fewer maclaurin terms required - // see https://www.wolframalpha.com/input/?i=arcsin%28sqrt%281-x%5E2%29%29-acos%28x%29 for x > 0 - // arcsin(sqrt(1-x^2)) = acos(x) for x > 0 - // arcsin(sqrt(1-x^2)) = acos(x) = pi/2 - arcsin(x) for x > 0 - // arcsin(sqrt(1-x^2)) = pi/2 - arcsin(x) for x > 0 - // arcsin(x) = pi/2 - arcsin(sqrt(1-x^2)) for x > 0 .... 1 - // - // see - // https://www.wolframalpha.com/input/?i=arcsin%28x%29+-+arcsin%28sqrt%281-x%5E2%29%29+%2B+pi%2F2+ - // for x < 0 - // arcsin(sqrt(1-x^2)) = pi-acos(x) for x < 0 - // arcsin(sqrt(1-x^2)) = pi-(pi/2 - arcsin(x)) for x < 0 - // arcsin(sqrt(1-x^2)) = pi/2 + arcsin(x) for x < 0 - // arcsin(x) = arcsin(sqrt(1-x^2)) - pi/2 for x < 0 .... 2 - - // within domain [-SQRT2 <= x <= SQRT2] use arcsin(x). outside this range use formulae above. - // so that convergence is faster - // This way we can transform input x into [-1/sqrt(2),1/sqrt(2)], where convergence is relatively - // fast. - - public static double asin(double x, int n_terms) { - if (x > SQRT2) return pip2 - asin2(Math.sqrt(1 - (x * x)), n_terms); - else if (Math.abs(x) <= SQRT2) return asin2(x, n_terms); - return asin2(Math.sqrt(1 - (x * x)), n_terms) - pip2; - } - - public static double asin(double x) { - return asin(x, DEF_TERMS); - } - - public static double acos(double x, int n_terms) { - return Math.abs(x) <= SQRT2 ? pip2 - asin2(x, n_terms) : asin2(Math.sqrt(1 - (x * x)), n_terms); - } - - public static double acos(double x) { - return acos(x, DEF_TERMS); - } - - // Following for completion for Inverse trigonometric functions - public static double atan(double x) { - return asin(x / Math.sqrt(1 + x * x)); - } - - public static double acot(double x) { - return pip2 - atan(x); - } - - public static double asec(double x) { - return acos(1 / x); - } - - public static double acsc(double x) { - return pip2 - asec(x); - } -} diff --git a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java b/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java deleted file mode 100644 index d080980849b..00000000000 --- a/solr/core/src/test/org/apache/solr/util/FastInvTrigTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util; - -import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; - -import org.apache.solr.SolrTestCase; -import org.junit.Before; -import org.junit.Test; - -public class FastInvTrigTest extends SolrTestCase { - - static final int num_points = 100000; - static final double EPSILON = 0.0001; - private static final double TEN_METERS = 0.01; - - static double[][] points = new double[num_points][2]; - - @Before - public void initAll() { - for (int i = 0; i < num_points; i++) { - points[i] = generateRandomPoint(); - } - } - - public static double deg2rad(double deg) { - return deg * (Math.PI / 180); - } - - public static double[] generateRandomPoint() { - double u = random().nextDouble(); - double v = random().nextDouble(); - - double latitude = deg2rad(Math.toDegrees(Math.acos(u * 2 - 1)) - 90); - double longitude = deg2rad(360 * v - 180); - return new double[] {latitude, longitude}; - } - - @Test - public void acos() { - for (double i = -1; i <= 1; i = i + 0.00001) { - assertTrue(FastInvTrig.acos(i) - Math.acos(i) <= EPSILON); - } - } - - @Test - public void dist() { - double[] a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); - - for (int i = 0; i < num_points; i++) { - double[] b = NVectorUtil.latLongToNVector(points[i][0], points[i][1]); - double d1 = EARTH_MEAN_RADIUS_KM * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - double d2 = EARTH_MEAN_RADIUS_KM * Math.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); - assertEquals("Math.acos should be close to FastInvTrig.acos", d1, d2, TEN_METERS); - } - } -} From 50c486f92c8f25068ad24be5c0318cc471b4fb99 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Thu, 28 Jul 2022 10:08:10 +0100 Subject: [PATCH 13/20] use custom sorting on dotproduct as enough for comparison --- .../function/distance/NVectorFunction.java | 127 ++++++++++++++++-- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java index 72fda11b46f..e5b6d496a93 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java @@ -18,6 +18,7 @@ package org.apache.solr.search.function.distance; import static org.apache.solr.util.NVectorUtil.NVectorDist; +import static org.apache.solr.util.NVectorUtil.NVectorDotProduct; import java.io.IOException; import java.util.Map; @@ -27,11 +28,10 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.apache.lucene.queries.function.valuesource.MultiValueSource; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.*; import org.apache.solr.common.SolrException; public class NVectorFunction extends ValueSource { - private final MultiValueSource nvector1; private final MultiValueSource nvector2; private final double radius; @@ -48,11 +48,28 @@ public NVectorFunction(MultiValueSource nvector1, MultiValueSource nvector2, dou @Override public FunctionValues getValues(Map context, LeafReaderContext readerContext) - throws IOException { + throws IOException { + + final FunctionValues fv = getDotProductValues(context,readerContext); + + return new DoubleDocValues(this) { + + @Override + public double doubleVal(int doc) throws IOException { + double dotProduct = fv.doubleVal(doc); + return NVectorDist(dotProduct, radius); + } + + @Override + public String toString(int doc) throws IOException { + return fv.toString(doc); + } + }; + } + public FunctionValues getDotProductValues(Map context, LeafReaderContext readerContext) throws IOException { final FunctionValues nvector_v1 = nvector1.getValues(context, readerContext); final FunctionValues nvector_v2 = nvector2.getValues(context, readerContext); - return new DoubleDocValues(this) { @Override @@ -61,7 +78,7 @@ public double doubleVal(int doc) throws IOException { double[] nvector_dv2 = new double[nvector2.dimension()]; nvector_v1.doubleVal(doc, nvector_dv1); nvector_v2.doubleVal(doc, nvector_dv2); - return NVectorDist(nvector_dv1, nvector_dv2, radius); + return NVectorDotProduct(nvector_dv1, nvector_dv2); } @Override @@ -74,7 +91,6 @@ public String toString(int doc) throws IOException { protected String name() { return "nvector"; } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -84,20 +100,113 @@ public boolean equals(Object o) { && nvector1.equals(that.nvector1) && nvector2.equals(that.nvector2); } - @Override public int hashCode() { return Objects.hash(nvector1, nvector2, radius); } - @Override public void createWeight(Map context, IndexSearcher searcher) throws IOException { nvector1.createWeight(context, searcher); nvector2.createWeight(context, searcher); } - @Override public String description() { return name() + '(' + nvector1 + ',' + nvector2 + ')'; } + @Override + public SortField getSortField(boolean reverse){ return new NVectorValueSourceSortField(reverse); } + class NVectorValueSourceSortField extends SortField { + public NVectorValueSourceSortField(boolean reverse) { + super(description(), SortField.Type.REWRITEABLE, reverse); + } + + @Override + public SortField rewrite(IndexSearcher searcher) throws IOException { + Map context = ValueSource.newContext(searcher); + createWeight(context, searcher); + return new SortField(getField(), new NVectorValueSourceComparatorSource(context), getReverse()); + } + } + class NVectorValueSourceComparatorSource extends FieldComparatorSource { + private final Map context; + + public NVectorValueSourceComparatorSource(Map context) { + this.context = context; + } + + @Override + public FieldComparator newComparator( + String fieldname, int numHits, boolean enableSkipping, boolean reversed) { + return new NVectorValueSourceComparator(context, numHits); + } + } + //Please note: The comparisons are INVERTED here, for performance on sorting we compare + // with the dot product, which is negatively correlated to acos(dot_product). Converting the dot product to distance with d=R*acos(dot_product), for example: + // where d = distance, dp = dot_product + // d:19628.29698448594 dp:-0.9981573955675561 + //d:18583.651644725564 dp:-0.9748645959351536 + //d:18180.44788490305 dp:-0.9588220684898193 + //d:18052.6639699517 dp:-0.9529332319157707 + //d:17882.7920545206 dp:-0.9445116994940388 + //.... + //d:1930.2292960794525 dp:0.954454358625817 + //d:1843.0056018566079 dp:0.9584495044607074 + //d:455.63526860635756 dp:0.9974437510266485 + //d:336.9277320011129 dp:0.9986019397287412 + //d:0.0 dp:1.0 + // A HIGHER dot_product equates to a closer (LOWER) distance hence we invert + class NVectorValueSourceComparator extends SimpleFieldComparator { + private final double[] values; + private FunctionValues docVals; + private double bottom; + private final Map fcontext; + private double topValue; + + NVectorValueSourceComparator(Map fcontext, int numHits) { + this.fcontext = fcontext; + values = new double[numHits]; + } + + @Override + public int compare(int slot1, int slot2) { + return Double.compare(values[slot2],values[slot1]); + } + + @Override + public int compareBottom(int doc) throws IOException { + return Double.compare(docVals.doubleVal(doc),bottom); + } + + @Override + public void copy(int slot, int doc) throws IOException { + values[slot] = docVals.doubleVal(doc); + } + + @Override + public void doSetNextReader(LeafReaderContext context) throws IOException { + docVals = getDotProductValues(fcontext, context); + } + + @Override + public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override + public void setTopValue(final Double value) { + this.topValue = value; + } + + @Override + public Double value(int slot) { + return values[slot]; + } + + @Override + public int compareTop(int doc) throws IOException { + return Double.compare(docVals.doubleVal(doc),topValue); + } + } } + + From 0389ca4d9b990de01b23c1aa3fabaaa0f9f9790a Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Thu, 28 Jul 2022 10:11:01 +0100 Subject: [PATCH 14/20] Add dotproduct calculation, use SloppyMath.asin instead of FastInvTrig.acos, faster at 40cm resolution --- .../src/java/org/apache/solr/util/NVectorUtil.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index 7e60d1dddf8..e5f2ce82274 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -17,6 +17,8 @@ package org.apache.solr.util; +import org.apache.lucene.util.SloppyMath; + import java.text.NumberFormat; import java.text.ParseException; @@ -24,6 +26,8 @@ public class NVectorUtil { + private static final double pip2 = Math.PI/2; + public static double[] latLongToNVector(double lat, double lon) { double latRad = lat * (Math.PI / 180); double lonRad = lon * (Math.PI / 180); @@ -64,12 +68,20 @@ public static double[] NVectorToLatLong(String[] n) { }); } + public static double NVectorDotProduct(double[] a, double[] b) { + return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; + } + public static double NVectorDist(double[] a, double[] b) { return NVectorDist(a, b, EARTH_MEAN_RADIUS_KM); } public static double NVectorDist(double[] a, double[] b, double radius) { - return radius * FastInvTrig.acos(a[0] * b[0] + a[1] * b[1] + a[2] * b[2]); + return radius * (pip2 - SloppyMath.asin(a[0] * b[0] + a[1] * b[1] + a[2] * b[2])); + } + + public static double NVectorDist(double dotProduct, double radius){ + return radius * (pip2 - SloppyMath.asin(dotProduct)); } From 7b325ce04a14d4e2cb26347855489829557cd465 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Thu, 28 Jul 2022 10:11:37 +0100 Subject: [PATCH 15/20] more testing for sorting --- .../function/distance/NVectorDistTest.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java index 685f17173a1..390dcc8c579 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/NVectorDistTest.java @@ -21,6 +21,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Locale; + public class NVectorDistTest extends SolrTestCaseJ4 { @BeforeClass @@ -29,6 +31,14 @@ public static void beforeClass() throws Exception { initCore("solrconfig-nvector.xml", "schema-nvector.xml"); } + @Override + public void setUp() throws Exception { + Locale.setDefault(Locale.ENGLISH);//for parsing lat/log correctly in this test + super.setUp(); + assertU(delQ("*:*")); + assertU(commit()); + } + @Test public void testNVector() throws Exception { assertU(adoc("id", "0", "nvector", "52.02471051274793, -0.49007556238612354")); @@ -58,6 +68,73 @@ public void testNVectorRadiusFilter() throws Exception { assertU(adoc("id", "1", "nvector", "51.927619, -0.186636")); assertU(adoc("id", "2", "nvector", "51.480043, -0.196508")); assertU(commit()); + + assertJQ( + req( + "defType", "lucene", + "lat", "52.01966071979866", + "lon", "-0.4983083573742952", + "dist", "nvdist($lat,$lon,nvector)", + "q", "*:*", + "fl","id", + "sort", "$dist asc"), + "/response/numFound==3", + "/response/docs/[0]/id=='0'", + "/response/docs/[1]/id=='1'" + ); + + assertJQ( + req( + "defType", "lucene", + "lat", "52.01966071979866", + "lon", "-0.4983083573742952", + "dist", "nvdist($lat,$lon,nvector)", + "q", "*:*", + "fl","id", + "sort", "$dist desc"), + "/response/numFound==3", + "/response/docs/[0]/id=='2'", + "/response/docs/[1]/id=='1'" + ); + + assertJQ( + req( + "defType", "lucene", + "lat", "52.01966071979866", + "lon", "-0.4983083573742952", + "dist", "nvdist($lat,$lon,nvector)", + "q", "*:*", + "fl","id,dist:$dist", + "sort", "$dist asc"), + "/response/numFound==3", + "/response/docs/[0]/id=='0'", + "/response/docs/[0]/dist==0.7953814512052634", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==23.675588801593264", + "/response/docs/[2]/id=='2'", + "/response/docs/[2]/dist==63.49776326818523" + ); + + assertJQ( + req( + "defType", "lucene", + "lat", "52.01966071979866", + "lon", "-0.4983083573742952", + "dist", "nvdist($lat,$lon,nvector)", + "q", "*:*", + "fl","id,dist:$dist", + "sort", "$dist desc"), + "/response/numFound==3", + "/response/docs/[0]/id=='2'", + "/response/docs/[0]/dist==63.49776326818523", + "/response/docs/[1]/id=='1'", + "/response/docs/[1]/dist==23.675588801593264", + "/response/docs/[2]/id=='0'", + "/response/docs/[2]/dist==0.7953814512052634" + + + ); + assertJQ( req( "defType", "lucene", @@ -98,5 +175,6 @@ public void testNVectorRadiusFilter() throws Exception { "/response/docs/[0]/dist==0.7953814512052634", "/response/docs/[1]/id=='1'", "/response/docs/[1]/dist==23.675588801562068"); + } } From 0119d316dfd2d1caf1bc35635f82c7afaf1f7399 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Mon, 8 Aug 2022 12:08:16 +0100 Subject: [PATCH 16/20] psf for DEFAULT_SEPARATOR. use Math for radian calc --- solr/core/src/java/org/apache/solr/schema/NVectorField.java | 2 +- solr/core/src/java/org/apache/solr/util/NVectorUtil.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/NVectorField.java b/solr/core/src/java/org/apache/solr/schema/NVectorField.java index 591e928d7bc..171667d824b 100644 --- a/solr/core/src/java/org/apache/solr/schema/NVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/NVectorField.java @@ -40,7 +40,7 @@ public class NVectorField extends CoordinateFieldType { - String DEFAULT_SEPARATOR = ","; + private static final String DEFAULT_SEPARATOR = ","; String separator = DEFAULT_SEPARATOR; @Override diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index e5f2ce82274..065edc87116 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -29,8 +29,8 @@ public class NVectorUtil { private static final double pip2 = Math.PI/2; public static double[] latLongToNVector(double lat, double lon) { - double latRad = lat * (Math.PI / 180); - double lonRad = lon * (Math.PI / 180); + double latRad = Math.toRadians(lat); + double lonRad = Math.toRadians(lon); double x = Math.cos(latRad) * Math.cos(lonRad); double y = Math.cos(latRad) * Math.sin(lonRad); double z = Math.sin(latRad); @@ -77,7 +77,7 @@ public static double NVectorDist(double[] a, double[] b) { } public static double NVectorDist(double[] a, double[] b, double radius) { - return radius * (pip2 - SloppyMath.asin(a[0] * b[0] + a[1] * b[1] + a[2] * b[2])); + return NVectorDist(NVectorDotProduct(a, b), radius); } public static double NVectorDist(double dotProduct, double radius){ From 6e745ec8002079264801c33c943da090779d2c50 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Tue, 9 Aug 2022 14:44:01 +0100 Subject: [PATCH 17/20] lc method names --- .../solr/search/function/distance/NVectorFunction.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java index e5b6d496a93..e1ce5afb9f5 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/NVectorFunction.java @@ -17,8 +17,8 @@ package org.apache.solr.search.function.distance; -import static org.apache.solr.util.NVectorUtil.NVectorDist; -import static org.apache.solr.util.NVectorUtil.NVectorDotProduct; +import static org.apache.solr.util.NVectorUtil.nVectorDist; +import static org.apache.solr.util.NVectorUtil.nVectorDotProduct; import java.io.IOException; import java.util.Map; @@ -57,7 +57,7 @@ public FunctionValues getValues(Map context, LeafReaderContext r @Override public double doubleVal(int doc) throws IOException { double dotProduct = fv.doubleVal(doc); - return NVectorDist(dotProduct, radius); + return nVectorDist(dotProduct, radius); } @Override @@ -78,7 +78,7 @@ public double doubleVal(int doc) throws IOException { double[] nvector_dv2 = new double[nvector2.dimension()]; nvector_v1.doubleVal(doc, nvector_dv1); nvector_v2.doubleVal(doc, nvector_dv2); - return NVectorDotProduct(nvector_dv1, nvector_dv2); + return nVectorDotProduct(nvector_dv1, nvector_dv2); } @Override From d9b7fe906c0f4cebfb37a2617d64fe65f53becaf Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Tue, 9 Aug 2022 14:44:49 +0100 Subject: [PATCH 18/20] - javadoc - lc method names - use Math static method where appropriate --- .../org/apache/solr/util/NVectorUtil.java | 77 +++++++++++++++---- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java index 065edc87116..47fd2bbbf55 100644 --- a/solr/core/src/java/org/apache/solr/util/NVectorUtil.java +++ b/solr/core/src/java/org/apache/solr/util/NVectorUtil.java @@ -18,16 +18,29 @@ package org.apache.solr.util; import org.apache.lucene.util.SloppyMath; +import org.apache.solr.schema.NVectorField; +import org.apache.solr.search.function.distance.NVectorFunction; import java.text.NumberFormat; import java.text.ParseException; import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_KM; +/** + * NVectorUtil : This class contains helper methods used by {@link NVectorFunction} and {@link NVectorField} + * to convert between n-vectors and lat,lon as well as calculating dot product for sorting and + * calculating the great circle (surface) distance + */ public class NVectorUtil { private static final double pip2 = Math.PI/2; + /** + * + * @param lat the latitude + * @param lon the longitude + * @return the NVector as double[3] + */ public static double[] latLongToNVector(double lat, double lon) { double latRad = Math.toRadians(lat); double lonRad = Math.toRadians(lon); @@ -37,6 +50,12 @@ public static double[] latLongToNVector(double lat, double lon) { return new double[] {x, y, z}; } + + /** + * @param lat the latitude + * @param lon the longitude + * @return string rep of the n-vector + */ public static String[] latLongToNVector(String lat, String lon) { double[] nvec = latLongToNVector(Double.parseDouble(lat), Double.parseDouble(lon)); return new String[] { @@ -44,45 +63,71 @@ public static String[] latLongToNVector(String lat, String lon) { }; } - public static String[] latLongToNVector(String[] latlon) { - return latLongToNVector(latlon[0], latlon[1]); - } - - public static String[] latLongToNVector(String[] point, NumberFormat formatter) throws ParseException { + /** + * @param point string rep of lat,lon + * @param formatter for parsing the string into a double wrt the locale + * @return string rep of the n-vector + * @throws ParseException If the string for point cannot be parsed + */ + public static String[] latLongToNVector(String[] point, NumberFormat formatter) throws ParseException { double[] nvec = latLongToNVector(formatter.parse(point[0]).doubleValue(),formatter.parse(point[1]).doubleValue()); return new String[] { Double.toString(nvec[0]), Double.toString(nvec[1]), Double.toString(nvec[2]) }; } - public static double[] NVectorToLatLong(double[] n) { + /** + * @param n the nvector + * @return the lat lon for this n-vector + */ + public static double[] nVectorToLatLong(double[] n) { return new double[] { - Math.asin(n[2]) * (180 / Math.PI), Math.atan(n[1] / n[0]) * (180 / Math.PI) + Math.toDegrees(Math.asin(n[2])),Math.toDegrees(Math.atan(n[1] / n[0])) }; } - public static double[] NVectorToLatLong(String[] n) { - return NVectorToLatLong( + public static double[] nVectorToLatLong(String[] n) { + return nVectorToLatLong( new double[] { Double.parseDouble(n[0]), Double.parseDouble(n[1]), Double.parseDouble(n[2]) }); } - public static double NVectorDotProduct(double[] a, double[] b) { + /** + * @param a the first n-vector + * @param b the second n-vector + * @return scalar doc product of both n-vectors + */ + public static double nVectorDotProduct(double[] a, double[] b) { return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; } - public static double NVectorDist(double[] a, double[] b) { - return NVectorDist(a, b, EARTH_MEAN_RADIUS_KM); + /** + * @param a the first n-vector + * @param b the second n-vector + * @return the great circle (surface) distance between the two n-vectors + */ + public static double nVectorDist(double[] a, double[] b) { + return nVectorDist(a, b, EARTH_MEAN_RADIUS_KM); } - public static double NVectorDist(double[] a, double[] b, double radius) { - return NVectorDist(NVectorDotProduct(a, b), radius); + /** + * @param a the first n-vector + * @param b the second n-vector + * @param radius he radius of the ellipsoid + * @return the great circle (surface) distance between the two n-vectors + */ + public static double nVectorDist(double[] a, double[] b, double radius) { + return nVectorDist(nVectorDotProduct(a, b), radius); } - public static double NVectorDist(double dotProduct, double radius){ + /** + * @param dotProduct the scalar dot product of two n-vectors + * @param radius the radius of the ellipsoid + * @return the great circle (surface) distance between the two n-vectors + */ + public static double nVectorDist(double dotProduct, double radius){ return radius * (pip2 - SloppyMath.asin(dotProduct)); } - } From 83aae86a1dae279ba1123ce45c946859e8a25f65 Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Tue, 9 Aug 2022 14:45:28 +0100 Subject: [PATCH 19/20] - alow tolerance - lc method names --- .../src/test/org/apache/solr/util/NVectorUtilTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java index f6c80943a05..89a9b82e1d9 100644 --- a/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/NVectorUtilTest.java @@ -28,7 +28,7 @@ public void latLongToNVector() { double lat = 52.024535; double lon = -0.490155; double[] n = NVectorUtil.latLongToNVector(lat, lon); - double[] ll = NVectorUtil.NVectorToLatLong(n); + double[] ll = NVectorUtil.nVectorToLatLong(n); assertEquals(lat, ll[0], 0.0001); assertEquals(lon, ll[1], 0.0001); } @@ -38,7 +38,7 @@ public void latLongToNVectorStr() { String lat = "52.024535"; String lon = "-0.490155"; String[] n = NVectorUtil.latLongToNVector(lat, lon); - double[] ll = NVectorUtil.NVectorToLatLong(n); + double[] ll = NVectorUtil.nVectorToLatLong(n); assertEquals(Double.parseDouble(lat), ll[0], 0.0001); assertEquals(Double.parseDouble(lon), ll[1], 0.0001); } @@ -47,10 +47,10 @@ public void latLongToNVectorStr() { public void NVectorDist() { double[] a = NVectorUtil.latLongToNVector(52.019819, -0.490155); double[] b = NVectorUtil.latLongToNVector(52.019660, -0.498308); - double dist = NVectorUtil.NVectorDist(a, b); - assertEquals(0.5581762827572362, dist, 0); + double dist = NVectorUtil.nVectorDist(a, b); + assertEquals(0.5581762827572362, dist, 0.0001); a = NVectorUtil.latLongToNVector(52.02456414691066, -0.49013542948214134); b = NVectorUtil.latLongToNVector(51.92756819110318, -0.18695373636718815); - assertEquals(23.400242809617353, NVectorUtil.NVectorDist(a, b), 0); + assertEquals(23.400242809617353, NVectorUtil.nVectorDist(a, b), 0.0001); } } From f9952bb3143bb68c7e923aa68deb73452addb97d Mon Sep 17 00:00:00 2001 From: Dan Rosher Date: Tue, 9 Aug 2022 14:48:51 +0100 Subject: [PATCH 20/20] - add nvdist parser --- .../src/java/org/apache/solr/search/ValueSourceParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java index cd86b3acadd..3e35bdf2f9c 100644 --- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java @@ -81,6 +81,7 @@ import org.apache.solr.search.function.distance.SquaredEuclideanFunction; import org.apache.solr.search.function.distance.StringDistanceFunction; import org.apache.solr.search.function.distance.VectorDistanceFunction; +import org.apache.solr.search.function.distance.NVectorValueSourceParser; import org.apache.solr.search.join.ChildFieldValueSourceParser; import org.apache.solr.util.DateMathParser; import org.apache.solr.util.PayloadUtils; @@ -386,6 +387,8 @@ public ValueSource parse(FunctionQParser fp) throws SyntaxError { addParser("geodist", new GeoDistValueSourceParser()); + addParser("nvdist", new NVectorValueSourceParser()); + addParser( "hsin", new ValueSourceParser() {