From 8bbb1dcacea996f81b989e360578a64ba3b78bfb Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Wed, 2 Mar 2016 09:22:20 -0500 Subject: [PATCH 0001/1113] SOLR-8764: Remove deprecated methods and classes --- solr/CHANGES.txt | 2 + .../clustering/carrot2/CarrotParams.java | 10 +- .../carrot2/SolrResourceLocator.java | 4 +- .../solr/collection1/conf/solrconfig.xml | 5 - .../carrot2/CarrotClusteringEngineTest.java | 6 - .../dataimport/EntityProcessorWrapper.java | 12 -- .../TestContentStreamDataSource.java | 6 +- .../TestSolrEntityProcessorEndToEnd.java | 6 +- .../UIMAUpdateRequestProcessorTest.java | 6 +- .../solrj/embedded/JettySolrRunner.java | 120 +------------- .../java/org/apache/solr/core/NodeConfig.java | 20 --- .../java/org/apache/solr/core/SolrConfig.java | 8 +- .../java/org/apache/solr/core/SolrCore.java | 48 ------ .../solr/handler/MoreLikeThisHandler.java | 17 -- .../solr/handler/admin/AdminHandlers.java | 151 ------------------ .../handler/admin/LukeRequestHandler.java | 7 - .../apache/solr/internal/csv/CSVParser.java | 34 +--- .../apache/solr/internal/csv/CSVStrategy.java | 15 +- .../org/apache/solr/parser/CharStream.java | 16 -- .../apache/solr/parser/FastCharStream.java | 8 - .../solr/schema/AbstractSpatialFieldType.java | 23 +-- .../schema/RptWithGeometrySpatialField.java | 1 - .../org/apache/solr/schema/TrieField.java | 66 +------- .../apache/solr/search/SolrIndexSearcher.java | 5 - .../distance/GeoDistValueSourceParser.java | 2 - .../solr/update/UpdateShardHandler.java | 5 - .../UpdateRequestProcessorChain.java | 6 - .../org/apache/solr/util/DistanceUnits.java | 4 - .../solr/collection1/conf/schema_latest.xml | 2 +- .../solr/collection1/conf/solrconfig-sql.xml | 1 - .../apache/solr/TestSolrCoreProperties.java | 5 +- .../solr/cloud/BaseCdcrDistributedZkTest.java | 9 +- .../org/apache/solr/core/TestSolrXml.java | 4 - .../apache/solr/core/TestXIncludeConfig.java | 4 +- .../solr/handler/TestReplicationHandler.java | 7 +- .../handler/TestReplicationHandlerBackup.java | 8 +- .../apache/solr/handler/TestRestoreCore.java | 8 +- .../solr/schema/SpatialRPTFieldTypeTest.java | 56 +------ ...cExpirationUpdateProcessorFactoryTest.java | 11 +- .../SignatureUpdateProcessorFactoryTest.java | 12 +- ...elessScriptUpdateProcessorFactoryTest.java | 4 +- .../UniqFieldsUpdateProcessorFactoryTest.java | 3 +- .../UpdateRequestProcessorFactoryTest.java | 8 +- .../client/solrj/impl/HttpSolrClient.java | 5 - .../solrj/request/CollectionAdminRequest.java | 17 -- .../client/solrj/response/PivotField.java | 8 - .../solr/client/solrj/util/ClientUtils.java | 36 ----- .../solr/common/cloud/ClosableThread.java | 26 --- .../solr/common/cloud/ZkCmdExecutor.java | 5 - .../solr/common/cloud/ZkStateReader.java | 6 +- .../solr/common/util/IteratorChain.java | 86 ---------- .../solr/collection1/conf/solrconfig-sql.xml | 1 - .../solrj/beans/TestDocumentObjectBinder.java | 101 ++++-------- .../solr/common/util/IteratorChainTest.java | 104 ------------ .../java/org/apache/solr/SolrTestCaseHS.java | 13 +- solr/webapp/web/js/angular/app.js | 11 -- solr/webapp/web/js/scripts/app.js | 11 -- 57 files changed, 124 insertions(+), 1061 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java delete mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/ClosableThread.java delete mode 100644 solr/solrj/src/java/org/apache/solr/common/util/IteratorChain.java delete mode 100644 solr/solrj/src/test/org/apache/solr/common/util/IteratorChainTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 77511cd701b6..3120eeb45e72 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -349,6 +349,8 @@ Other Changes * SOLR-8758: Add a new SolrCloudTestCase class, using MiniSolrCloudCluster (Alan Woodward) + +* SOLR-8764: Remove all deprecated methods and classes from master prior to the 6.0 release. (Steve Rowe) ================== 5.5.1 ================== diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java index 42ff8da84024..71a22fe93430 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java @@ -46,14 +46,7 @@ public final class CarrotParams { public static String LANGUAGE_CODE_MAP = CARROT_PREFIX + "lcmap"; /** - * Use {@link #RESOURCES_DIR}. - */ - @Deprecated - public static String LEXICAL_RESOURCES_DIR = CARROT_PREFIX + "lexicalResourcesDir"; - - /** - * A replacement property pointing to Carrot2 resources - * (a more generic version of the deprecated {@link #LEXICAL_RESOURCES_DIR}). + * Points to Carrot2 resources */ public static String RESOURCES_DIR = CARROT_PREFIX + "resourcesDir"; @@ -72,7 +65,6 @@ public final class CarrotParams { NUM_DESCRIPTIONS, OUTPUT_SUB_CLUSTERS, - LEXICAL_RESOURCES_DIR, RESOURCES_DIR, LANGUAGE_CODE_MAP); diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrResourceLocator.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrResourceLocator.java index b0783022d7dc..2cc67993a2ec 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrResourceLocator.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrResourceLocator.java @@ -45,10 +45,8 @@ class SolrResourceLocator implements IResourceLocator { public SolrResourceLocator(SolrCore core, SolrParams initParams) { resourceLoader = core.getResourceLoader(); - @SuppressWarnings("deprecation") - String lexicalResourcesDir = initParams.get(CarrotParams.LEXICAL_RESOURCES_DIR); String resourcesDir = initParams.get(CarrotParams.RESOURCES_DIR); - carrot2ResourcesDir = firstNonNull(resourcesDir, lexicalResourcesDir, CarrotClusteringEngine.CARROT_RESOURCES_PREFIX); + carrot2ResourcesDir = firstNonNull(resourcesDir, CarrotClusteringEngine.CARROT_RESOURCES_PREFIX); } @SuppressWarnings("unchecked") diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml index 7b2bd46c81ef..be629d80c0d7 100644 --- a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml @@ -347,11 +347,6 @@ org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm clustering/custom - - lexical-resource-check-custom-resource-dir-deprecated - org.apache.solr.handler.clustering.carrot2.LexicalResourcesCheckClusteringAlgorithm - clustering/custom - custom-duplicating-tokenizer org.apache.solr.handler.clustering.carrot2.EchoTokensClusteringAlgorithm diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java index 302247f39e7b..752570de3c12 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java @@ -199,12 +199,6 @@ public void testLexicalResourcesFromSolrConfigCustomDir() throws Exception { "online,customsolrstopwordcustomdir,customsolrstoplabelcustomdir"); } - @Test - public void testLexicalResourcesFromSolrConfigCustomDirDeprecated() throws Exception { - checkLexicalResourcesFromSolrConfig("lexical-resource-check-custom-resource-dir-deprecated", - "online,customsolrstopwordcustomdir,customsolrstoplabelcustomdir"); - } - private void checkLexicalResourcesFromSolrConfig(String engineName, String wordsToCheck) throws IOException { ModifiableSolrParams params = new ModifiableSolrParams(); diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java index c88c5775b8cf..6370d24cf278 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorWrapper.java @@ -318,22 +318,10 @@ public void setDatasource(DataSource datasource) { this.datasource = datasource; } - /** @deprecated will be removed in Solr 6; use {@link #isInitialized()} */ - @Deprecated - public boolean isInitalized() { - return initialized; - } - public boolean isInitialized() { return initialized; } - /** @deprecated will be removed in Solr 6; use {@link #setInitialized(boolean)} */ - @Deprecated - public void setInitalized(boolean initialized) { - this.initialized = initialized; - } - public void setInitialized(boolean initialized) { this.initialized = initialized; } diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java index b0b30c402148..453f5713709d 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java @@ -32,6 +32,7 @@ import java.io.File; import java.nio.file.Files; import java.util.List; +import java.util.Properties; /** * Test for ContentStreamDataSource @@ -173,8 +174,9 @@ public void setUp() throws Exception { } private JettySolrRunner createJetty(SolrInstance instance) throws Exception { - System.setProperty("solr.data.dir", instance.getDataDir()); - JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), buildJettyConfig("/solr")); + Properties nodeProperties = new Properties(); + nodeProperties.setProperty("solr.data.dir", instance.getDataDir()); + JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr")); jetty.start(); return jetty; } diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java index 1253f1643e95..abefc493f6b1 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Properties; /** * End-to-end test of SolrEntityProcessor. "Real" test using embedded Solr @@ -340,8 +341,9 @@ public void tearDown() throws Exception { } private JettySolrRunner createJetty(SolrInstance instance) throws Exception { - JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), buildJettyConfig("/solr")); - jetty.setDataDir(instance.getDataDir()); + Properties nodeProperties = new Properties(); + nodeProperties.setProperty("solr.data.dir", instance.getDataDir()); + JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr")); jetty.start(); return jetty; } diff --git a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java index 95feb365fa0a..5879c78e212f 100644 --- a/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java +++ b/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java @@ -65,8 +65,7 @@ public void testProcessorConfiguration() { SolrCore core = h.getCore(); UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("uima"); assertNotNull(chained); - UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory) chained - .getFactories()[0]; + UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory)chained.getProcessors().get(0); assertNotNull(factory); UpdateRequestProcessor processor = factory.getInstance(req(), null, null); assertTrue(processor instanceof UIMAUpdateRequestProcessor); @@ -77,8 +76,7 @@ public void testMultiMap() { SolrCore core = h.getCore(); UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("uima-multi-map"); assertNotNull(chained); - UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory) chained - .getFactories()[0]; + UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory)chained.getProcessors().get(0); assertNotNull(factory); UpdateRequestProcessor processor = factory.getInstance(req(), null, null); assertTrue(processor instanceof UIMAUpdateRequestProcessor); diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java index 02954c4d4c35..88ea5677aef6 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java @@ -114,13 +114,6 @@ public void destroy() { } } - private static Properties defaultNodeProperties(String solrconfigFilename, String schemaFilename) { - Properties props = new Properties(); - props.setProperty("solrconfig", solrconfigFilename); - props.setProperty("schema", schemaFilename); - return props; - } - /** * Create a new JettySolrRunner. * @@ -134,82 +127,6 @@ public JettySolrRunner(String solrHome, String context, int port) { this(solrHome, JettyConfig.builder().setContext(context).setPort(port).build()); } - /** - * @deprecated use {@link #JettySolrRunner(String,Properties,JettyConfig)} - */ - @Deprecated - public JettySolrRunner(String solrHome, String context, int port, String solrConfigFilename, String schemaFileName) { - this(solrHome, defaultNodeProperties(solrConfigFilename, schemaFileName), JettyConfig.builder() - .setContext(context) - .setPort(port) - .build()); - } - - /** - * @deprecated use {@link #JettySolrRunner(String,Properties,JettyConfig)} - */ - @Deprecated - public JettySolrRunner(String solrHome, String context, int port, - String solrConfigFilename, String schemaFileName, boolean stopAtShutdown) { - this(solrHome, defaultNodeProperties(solrConfigFilename, schemaFileName), - JettyConfig.builder() - .setContext(context) - .setPort(port) - .stopAtShutdown(stopAtShutdown) - .build()); - } - - /** - * Constructor taking an ordered list of additional (servlet holder -> path spec) mappings - * to add to the servlet context - * @deprecated use {@link JettySolrRunner#JettySolrRunner(String,Properties,JettyConfig)} - */ - @Deprecated - public JettySolrRunner(String solrHome, String context, int port, - String solrConfigFilename, String schemaFileName, boolean stopAtShutdown, - SortedMap extraServlets) { - this(solrHome, defaultNodeProperties(solrConfigFilename, schemaFileName), - JettyConfig.builder() - .setContext(context) - .setPort(port) - .stopAtShutdown(stopAtShutdown) - .withServlets(extraServlets) - .build()); - } - - /** - * @deprecated use {@link #JettySolrRunner(String,Properties,JettyConfig)} - */ - @Deprecated - public JettySolrRunner(String solrHome, String context, int port, String solrConfigFilename, String schemaFileName, - boolean stopAtShutdown, SortedMap extraServlets, SSLConfig sslConfig) { - this(solrHome, defaultNodeProperties(solrConfigFilename, schemaFileName), - JettyConfig.builder() - .setContext(context) - .setPort(port) - .stopAtShutdown(stopAtShutdown) - .withServlets(extraServlets) - .withSSLConfig(sslConfig) - .build()); - } - - /** - * @deprecated use {@link #JettySolrRunner(String,Properties,JettyConfig)} - */ - @Deprecated - public JettySolrRunner(String solrHome, String context, int port, String solrConfigFilename, String schemaFileName, - boolean stopAtShutdown, SortedMap extraServlets, SSLConfig sslConfig, - SortedMap, String> extraRequestFilters) { - this(solrHome, defaultNodeProperties(solrConfigFilename, schemaFileName), - JettyConfig.builder() - .setContext(context) - .setPort(port) - .stopAtShutdown(stopAtShutdown) - .withServlets(extraServlets) - .withFilters(extraRequestFilters) - .withSSLConfig(sslConfig) - .build()); - } /** * Construct a JettySolrRunner @@ -552,42 +469,17 @@ public static void main(String[] args) { } /** - * @deprecated set properties in the Properties passed to the constructor - */ - @Deprecated - public void setShards(String shardList) { - nodeProperties.setProperty("shard", shardList); - } - - /** - * @deprecated set properties in the Properties passed to the constructor - */ - @Deprecated - public void setDataDir(String dataDir) { - nodeProperties.setProperty("solr.data.dir", dataDir); - } - - /** - * @deprecated set properties in the Properties passed to the constructor - */ - @Deprecated - public void setUlogDir(String ulogDir) { - nodeProperties.setProperty("solr.ulog.dir", ulogDir); - } - - /** - * @deprecated set properties in the Properties passed to the constructor + * @return the Solr home directory of this JettySolrRunner */ - @Deprecated - public void setCoreNodeName(String coreNodeName) { - nodeProperties.setProperty("coreNodeName", coreNodeName); + public String getSolrHome() { + return solrHome; } /** - * @return the Solr home directory of this JettySolrRunner + * @return this node's properties */ - public String getSolrHome() { - return solrHome; + public Properties getNodeProperties() { + return nodeProperties; } private void waitForLoadingCoresToFinish(long timeoutMs) { diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java index 546f27c5f0ad..0783355cfe42 100644 --- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java +++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java @@ -106,26 +106,6 @@ public UpdateShardHandlerConfig getUpdateShardHandlerConfig() { return updateShardHandlerConfig; } - @Deprecated - public int getDistributedConnectionTimeout() { - return updateShardHandlerConfig.getDistributedConnectionTimeout(); - } - - @Deprecated - public int getDistributedSocketTimeout() { - return updateShardHandlerConfig.getDistributedSocketTimeout(); - } - - @Deprecated - public int getMaxUpdateConnections() { - return updateShardHandlerConfig.getMaxUpdateConnections(); - } - - @Deprecated - public int getMaxUpdateConnectionsPerHost() { - return updateShardHandlerConfig.getMaxUpdateConnectionsPerHost(); - } - public int getCoreLoadThreadCount() { return coreLoadThreads; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index e66fc7e74045..0b4bac33678e 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -197,7 +197,6 @@ public SolrConfig(SolrResourceLoader loader, String name, InputSource is) if (hasDeprecatedIndexConfig) { throw new SolrException(ErrorCode.FORBIDDEN, " and configuration sections are discontinued. Use instead."); } else { - defaultIndexConfig = mainIndexConfig = null; indexConfigPrefix = "indexConfig"; } assertWarnOrFail("The config has been discontinued and NRT mode is always used by Solr." + @@ -214,7 +213,7 @@ public SolrConfig(SolrResourceLoader loader, String name, InputSource is) ); // Parse indexConfig section, using mainIndex as backup in case old config is used - indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig); + indexConfig = new SolrIndexConfig(this, "indexConfig", null); booleanQueryMaxClauseCount = getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount()); log.info("Using Lucene MatchVersion: " + luceneMatchVersion); @@ -466,11 +465,6 @@ public SolrRequestParsers getRequestParsers() { // DocSet public final float hashSetInverseLoadFactor; public final int hashDocSetMaxSize; - // default & main index configurations, deprecated as of 3.6 - @Deprecated - public final SolrIndexConfig defaultIndexConfig; - @Deprecated - public final SolrIndexConfig mainIndexConfig; // IndexConfig settings public final SolrIndexConfig indexConfig; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 5d7ebdce8795..06d7da305d8f 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -166,9 +166,6 @@ public final class SolrCore implements SolrInfoMBean, Closeable { public Date getStartTimeStamp() { return startTime; } - @Deprecated - public long getStartTime() { return startTime.getTime(); } - public long getStartNanoTime() { return startNanoTime; } @@ -641,56 +638,11 @@ private UpdateHandler createUpdateHandler(String className, UpdateHandler update return createReloadedUpdateHandler(className, "Update Handler", updateHandler); } - /** - * Creates a new core and register it in the list of cores. - * If a core with the same name already exists, it will be stopped and replaced by this one. - * - * @param dataDir the index directory - * @param config a solr config instance - * @param schema a solr schema instance - * - * @since solr 1.3 - * @deprecated will be removed in the next release - */ - public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) { - this(name, dataDir, config, schema, null, cd, null, null, null); - } - public SolrCore(CoreDescriptor cd, ConfigSet coreConfig) { this(cd.getName(), null, coreConfig.getSolrConfig(), coreConfig.getIndexSchema(), coreConfig.getProperties(), cd, null, null, null); } - /** - * Creates a new core that is to be loaded lazily. i.e. lazyLoad="true" in solr.xml - * - * @since solr 4.1 - * @deprecated will be removed in the next release - */ - public SolrCore(String name, CoreDescriptor coreDescriptor) { - this.coreDescriptor = coreDescriptor; - this.setName(name); - this.schema = null; - this.dataDir = null; - this.ulogDir = null; - this.solrConfig = null; - this.configSetProperties = null; - this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number. - this.slowQueryThresholdMillis = -1; - this.resourceLoader = null; - this.updateHandler = null; - this.isReloaded = true; - this.reqHandlers = null; - this.updateProcessorChains = null; - this.infoRegistry = null; - this.codec = null; - this.ruleExpiryLock = null; - this.memClassLoader = null; - this.directoryFactory = null; - this.solrCoreState = null; - this.restManager = null; - this.solrDelPolicy = null; - } /** * Creates a new core and register it in the list of cores. If a core with the diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index ad28911e4089..89b0180127b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -434,23 +434,6 @@ public DocListAndSet getMoreLikeThis( Reader reader, int start, int rows, List getMoreLikeThese( DocList docs, int rows, int flags ) throws IOException - { - IndexSchema schema = searcher.getSchema(); - NamedList mlt = new SimpleOrderedMap<>(); - DocIterator iterator = docs.iterator(); - while( iterator.hasNext() ) { - int id = iterator.nextDoc(); - - DocListAndSet sim = getMoreLikeThis( id, 0, rows, null, null, flags ); - String name = schema.printableUniqueKey( reader.document( id ) ); - - mlt.add(name, sim.docList); - } - return mlt; - } - public NamedList getMoreLikeTheseQuery(DocList docs) throws IOException { IndexSchema schema = searcher.getSchema(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java deleted file mode 100644 index 1965a5051245..000000000000 --- a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.handler.admin; - -import java.lang.invoke.MethodHandles; -import java.net.URL; -import java.util.Map; - -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.plugin.SolrCoreAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A special Handler that registers all standard admin handlers - * - * @since solr 1.3 - * @deprecated No need to register this request handler. All - * the plugins registered by this class are implicitly registered by the system - */ -@Deprecated -public class AdminHandlers extends RequestHandlerBase implements SolrCoreAware -{ - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - NamedList initArgs = null; - - private static class StandardHandler { - final String name; - final SolrRequestHandler handler; - - public StandardHandler( String n, SolrRequestHandler h ) - { - this.name = n; - this.handler = h; - } - } - - /** - * Save the args and pass them to each standard handler - */ - @Override - public void init(NamedList args) { - this.initArgs = args; - } - - @Override - public void inform(SolrCore core) - { - String path = null; - path = getPluginInfo().name; - if( path == null ) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "The AdminHandler is not registered with the current core." ); - } - if( !path.startsWith( "/" ) ) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "The AdminHandler needs to be registered to a path. Typically this is '/admin'" ); - } - // Remove the parent handler - core.registerRequestHandler(path, null); - if( !path.endsWith( "/" ) ) { - path += "/"; - } - - StandardHandler[] list = new StandardHandler[] { - new StandardHandler( "luke", new LukeRequestHandler() ), - new StandardHandler( "system", new SystemInfoHandler() ), - new StandardHandler( "mbeans", new SolrInfoMBeanHandler() ), - new StandardHandler( "plugins", new PluginInfoHandler() ), - new StandardHandler( "threads", new ThreadDumpHandler() ), - new StandardHandler( "properties", new PropertiesRequestHandler() ), - new StandardHandler( "logging", new LoggingHandler() ), - new StandardHandler( "file", new ShowFileRequestHandler() ) - }; - - for( StandardHandler handler : list ) { - if( core.getRequestHandler( path+handler.name ) == null ) { - handler.handler.init( initArgs ); - core.registerRequestHandler( path+handler.name, handler.handler ); - if( handler.handler instanceof SolrCoreAware ) { - ((SolrCoreAware)handler.handler).inform(core); - } - } - } - log.warn(" is deprecated . It is not required anymore"); - } - - - @Override - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "The AdminHandler should never be called directly" ); - } - - //////////////////////// SolrInfoMBeans methods ////////////////////// - - @Override - public String getDescription() { - return "Register Standard Admin Handlers"; - } - - @Override - public String getVersion() { - return getClass().getPackage().getSpecificationVersion(); - } - - @Override - public String getSource() { - return null; - } - - @Override - public Category getCategory() { - return Category.QUERYHANDLER; - } - - @Override - public URL[] getDocs() { - return null; - } - - @Override - public String getName() { - return this.getClass().getName(); - } - - @Override - public NamedList getStatistics() { - return null; - } -} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 4e497780e6e8..0ec6d79fecbb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -562,13 +562,6 @@ private static void populateFieldInfo(IndexSchema schema, typeusemap.put( ft.getTypeName(), v ); } - /** - * @deprecated use {@link #getIndexInfo(DirectoryReader)} since you now have to explicitly pass the "fl" prameter - * and this was always called with "false" anyway from CoreAdminHandler - */ - public static SimpleOrderedMap getIndexInfo(DirectoryReader reader, boolean detail) throws IOException { - return getIndexInfo(reader); - } // This method just gets the top-most level of information. This was conflated with getting detailed info // for *all* the fields, called from CoreAdminHandler etc. diff --git a/solr/core/src/java/org/apache/solr/internal/csv/CSVParser.java b/solr/core/src/java/org/apache/solr/internal/csv/CSVParser.java index 8d88c3727107..ad1aef71f6c5 100644 --- a/solr/core/src/java/org/apache/solr/internal/csv/CSVParser.java +++ b/solr/core/src/java/org/apache/solr/internal/csv/CSVParser.java @@ -111,41 +111,9 @@ Token reset() { * @param input a Reader containing "csv-formatted" input */ public CSVParser(Reader input) { - // note: must match default-CSV-strategy !! - this(input, ','); + this(input, CSVStrategy.DEFAULT_STRATEGY); } - /** - * Customized value delimiter parser. - * - * The parser follows the default {@link CSVStrategy} - * except for the delimiter setting. - * - * @param input a Reader based on "csv-formatted" input - * @param delimiter a Char used for value separation - * @deprecated use {@link #CSVParser(Reader,CSVStrategy)}. - */ - public CSVParser(Reader input, char delimiter) { - this(input, delimiter, '"', CSVStrategy.COMMENTS_DISABLED); - } - - /** - * Customized csv parser. - * - * The parser parses according to the given CSV dialect settings. - * Leading whitespaces are truncated, unicode escapes are - * not interpreted and empty lines are ignored. - * - * @param input a Reader based on "csv-formatted" input - * @param delimiter a Char used for value separation - * @param encapsulator a Char used as value encapsulation marker - * @param commentStart a Char used for comment identification - * @deprecated use {@link #CSVParser(Reader,CSVStrategy)}. - */ - public CSVParser(Reader input, char delimiter, char encapsulator, char commentStart) { - this(input, new CSVStrategy(delimiter, encapsulator, commentStart)); - } - /** * Customized CSV parser using the given {@link CSVStrategy} * diff --git a/solr/core/src/java/org/apache/solr/internal/csv/CSVStrategy.java b/solr/core/src/java/org/apache/solr/internal/csv/CSVStrategy.java index 9273908b0e8e..e27c9c372610 100644 --- a/solr/core/src/java/org/apache/solr/internal/csv/CSVStrategy.java +++ b/solr/core/src/java/org/apache/solr/internal/csv/CSVStrategy.java @@ -54,7 +54,7 @@ public class CSVStrategy implements Cloneable, Serializable { public CSVStrategy(char delimiter, char encapsulator, char commentStart) { - this(delimiter, encapsulator, commentStart, true, false, true); + this(delimiter, encapsulator, commentStart, ESCAPE_DISABLED, true, true, false, true); } /** @@ -89,19 +89,6 @@ public CSVStrategy( setIgnoreEmptyLines(ignoreEmptyLines); } - /** @deprecated Use {@link #CSVStrategy(char, char, char, char, boolean, boolean, boolean, boolean)} */ - public CSVStrategy( - char delimiter, - char encapsulator, - char commentStart, - boolean ignoreLeadingWhitespace, - boolean interpretUnicodeEscapes, - boolean ignoreEmptyLines) - { - this(delimiter, encapsulator, commentStart, CSVStrategy.ESCAPE_DISABLED, ignoreLeadingWhitespace, - true, interpretUnicodeEscapes, ignoreEmptyLines); - } - public void setDelimiter(char delimiter) { this.delimiter = delimiter; } public char getDelimiter() { return this.delimiter; } diff --git a/solr/core/src/java/org/apache/solr/parser/CharStream.java b/solr/core/src/java/org/apache/solr/parser/CharStream.java index 0400af47e050..9bec5a6e9d52 100644 --- a/solr/core/src/java/org/apache/solr/parser/CharStream.java +++ b/solr/core/src/java/org/apache/solr/parser/CharStream.java @@ -27,22 +27,6 @@ interface CharStream { */ char readChar() throws java.io.IOException; - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - int getColumn(); - - @Deprecated - /** - * Returns the line number of the character last read. - * @deprecated - * @see #getEndLine - */ - int getLine(); - /** * Returns the column number of the last character for current token (being * matched after the last call to BeginTOken). diff --git a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java index 6a283f20d09e..712a28335b55 100644 --- a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java +++ b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java @@ -105,14 +105,6 @@ public final void Done() { } } - @Override - public final int getColumn() { - return bufferStart + bufferPosition; - } - @Override - public final int getLine() { - return 1; - } @Override public final int getEndColumn() { return bufferStart + bufferPosition; diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index c86f69153da3..83fd44775f41 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -91,8 +91,6 @@ public abstract class AbstractSpatialFieldType extend private final Cache fieldStrategyCache = CacheBuilder.newBuilder().build(); protected DistanceUnits distanceUnits; - @Deprecated - protected String units; // for back compat; hopefully null protected final Set supportedScoreModes; @@ -120,30 +118,11 @@ protected void init(IndexSchema schema, Map args) { ctx = SpatialContextFactory.makeSpatialContext(argsWrap, schema.getResourceLoader().getClassLoader()); args.keySet().removeAll(argsWrap.getSeenKeys()); } - - final String unitsErrMsg = "units parameter is deprecated, please use distanceUnits instead for field types with class " + - getClass().getSimpleName(); - this.units = args.remove("units");//deprecated - if (units != null) { - if ("degrees".equals(units)) { - log.warn(unitsErrMsg); - } else { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, unitsErrMsg); - } - } final String distanceUnitsStr = args.remove("distanceUnits"); if (distanceUnitsStr == null) { - if (units != null) { - this.distanceUnits = DistanceUnits.BACKCOMPAT; - } else { - this.distanceUnits = ctx.isGeo() ? DistanceUnits.KILOMETERS : DistanceUnits.DEGREES; - } + this.distanceUnits = ctx.isGeo() ? DistanceUnits.KILOMETERS : DistanceUnits.DEGREES; } else { - // If both units and distanceUnits was specified - if (units != null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, unitsErrMsg); - } this.distanceUnits = parseDistanceUnits(distanceUnitsStr); if (this.distanceUnits == null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java index 55fcf73d868c..fe4cedc04c58 100644 --- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java +++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java @@ -68,7 +68,6 @@ protected void init(IndexSchema schema, Map args) { rptFieldType.argsParser = argsParser = newSpatialArgsParser(); this.ctx = rptFieldType.ctx; this.distanceUnits = rptFieldType.distanceUnits; - this.units = rptFieldType.units; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index b369e999a898..251eb8544d3a 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -117,23 +117,8 @@ public Object toObject(IndexableField f) { if (val != null) { return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val; } else { - // the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding: - final BytesRef bytes = f.binaryValue(); - if (bytes==null) return badFieldString(f); - switch (type) { - case INTEGER: - return toInt(bytes.bytes, bytes.offset); - case FLOAT: - return Float.intBitsToFloat(toInt(bytes.bytes, bytes.offset)); - case LONG: - return toLong(bytes.bytes, bytes.offset); - case DOUBLE: - return Double.longBitsToDouble(toLong(bytes.bytes, bytes.offset)); - case DATE: - return new Date(toLong(bytes.bytes, bytes.offset)); - default: - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); - } + // the old BinaryField encoding is no longer supported + return badFieldString(f); } } @@ -432,18 +417,6 @@ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal } } - @Deprecated - static int toInt(byte[] arr, int offset) { - return (arr[offset]<<24) | ((arr[offset+1]&0xff)<<16) | ((arr[offset+2]&0xff)<<8) | (arr[offset+3]&0xff); - } - - @Deprecated - static long toLong(byte[] arr, int offset) { - int high = (arr[offset]<<24) | ((arr[offset+1]&0xff)<<16) | ((arr[offset+2]&0xff)<<8) | (arr[offset+3]&0xff); - int low = (arr[offset+4]<<24) | ((arr[offset+5]&0xff)<<16) | ((arr[offset+6]&0xff)<<8) | (arr[offset+7]&0xff); - return (((long)high)<<32) | (low&0x0ffffffffL); - } - @Override public String storedToReadable(IndexableField f) { return toExternal(f); @@ -596,39 +569,8 @@ private void storedToIndexed(IndexableField f, final BytesRefBuilder bytes) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } } else { - // the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding: - final BytesRef bytesRef = f.binaryValue(); - if (bytesRef==null) - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: "+f.name()); - switch (type) { - case INTEGER: - LegacyNumericUtils.intToPrefixCoded(toInt(bytesRef.bytes, bytesRef.offset), 0, bytes); - break; - case FLOAT: { - // WARNING: Code Duplication! Keep in sync with o.a.l.util.LegacyNumericUtils! - // copied from LegacyNumericUtils to not convert to/from float two times - // code in next 2 lines is identical to: int v = LegacyNumericUtils.floatToSortableInt(Float.intBitsToFloat(toInt(arr))); - int v = toInt(bytesRef.bytes, bytesRef.offset); - if (v<0) v ^= 0x7fffffff; - LegacyNumericUtils.intToPrefixCoded(v, 0, bytes); - break; - } - case LONG: //fallthrough! - case DATE: - LegacyNumericUtils.longToPrefixCoded(toLong(bytesRef.bytes, bytesRef.offset), 0, bytes); - break; - case DOUBLE: { - // WARNING: Code Duplication! Keep in sync with o.a.l.util.LegacyNumericUtils! - // copied from LegacyNumericUtils to not convert to/from double two times - // code in next 2 lines is identical to: long v = LegacyNumericUtils.doubleToSortableLong(Double.longBitsToDouble(toLong(arr))); - long v = toLong(bytesRef.bytes, bytesRef.offset); - if (v<0) v ^= 0x7fffffffffffffffL; - LegacyNumericUtils.longToPrefixCoded(v, 0, bytes); - break; - } - default: - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); - } + // the old BinaryField encoding is no longer supported + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: "+f.name()); } } diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index da0c7bf5e665..833230977ccf 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -2392,11 +2392,6 @@ public long getOpenNanoTime() { return openNanoTime; } - @Deprecated - public long getOpenTime() { - return openTime.getTime(); - } - @Override public Explanation explain(Query query, int doc) throws IOException { return super.explain(QueryUtils.makeQueryable(query), doc); diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java index f7c05c3d30bb..1f4ea34ff955 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java @@ -135,8 +135,6 @@ public ValueSource parse(FunctionQParser fp) throws SyntaxError { SpatialStrategy strategy = ((SpatialStrategyMultiValueSource) mv2).strategy; DistanceUnits distanceUnits = ((SpatialStrategyMultiValueSource) mv2).distanceUnits; Point queryPoint = strategy.getSpatialContext().makePoint(constants[1], constants[0]); - if (distanceUnits == DistanceUnits.BACKCOMPAT) - distanceUnits = DistanceUnits.KILOMETERS; return strategy.makeDistanceValueSource(queryPoint, distanceUnits.multiplierFromDegreesToThisUnit()); } diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java index 5e805ca6821a..a44b8f87b766 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java @@ -60,11 +60,6 @@ public class UpdateShardHandler { private final UpdateShardHandlerConfig cfg; - @Deprecated - public UpdateShardHandler(NodeConfig cfg) { - this(cfg.getUpdateShardHandlerConfig()); - } - public UpdateShardHandler(UpdateShardHandlerConfig cfg) { this.cfg = cfg; clientConnectionManager = new PoolingClientConnectionManager(SchemeRegistryFactory.createSystemDefault()); diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index ca2e5ce6340f..4629425aa822 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -220,12 +220,6 @@ public UpdateRequestProcessor createProcessor(SolrQueryRequest req, return last; } - - @Deprecated - public UpdateRequestProcessorFactory[] getFactories() { - return chain.toArray(new UpdateRequestProcessorFactory[0]); - } - /** * Returns the underlying array of factories used in this chain. * Modifications to the array will affect future calls to diff --git a/solr/core/src/java/org/apache/solr/util/DistanceUnits.java b/solr/core/src/java/org/apache/solr/util/DistanceUnits.java index 7785bc9466c5..63f43e4652fb 100644 --- a/solr/core/src/java/org/apache/solr/util/DistanceUnits.java +++ b/solr/core/src/java/org/apache/solr/util/DistanceUnits.java @@ -41,10 +41,6 @@ public class DistanceUnits { DistanceUtils.MILES_TO_KM * DistanceUtils.KM_TO_DEG); public final static DistanceUnits DEGREES = new DistanceUnits(DEGREES_PARAM, 180.0/Math.PI, 1.0); - // Previously, distance based filtering was done with km, but scores were based on degrees - @Deprecated - public final static DistanceUnits BACKCOMPAT = new DistanceUnits("backcompat", DistanceUtils.EARTH_MEAN_RADIUS_KM, 1.0); - //volatile so other threads see when we replace when copy-on-write private static volatile Map instances = ImmutableMap.of( KILOMETERS_PARAM, KILOMETERS, diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index fc8f7c11dd1f..803d45eed770 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -718,7 +718,7 @@ http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4 --> + geo="true" distErrPct="0.025" maxDistErr="0.000009" /> '.esc() + "\n" + - ''.esc() + - '' - ); }; */ diff --git a/solr/webapp/web/js/scripts/app.js b/solr/webapp/web/js/scripts/app.js index 32382b07ec5c..a967cfe307a1 100644 --- a/solr/webapp/web/js/scripts/app.js +++ b/solr/webapp/web/js/scripts/app.js @@ -479,17 +479,6 @@ var solr_admin = function( app_config ) }, error : function() { - show_global_error - ( - '

Unable to load environment info from ' + system_url.esc() + '.

' + - '

This interface requires that you activate the admin request handlers in all SolrCores by adding the ' + - 'following configuration to your solrconfig.xml:

' + "\n" + - - '
' +
-                  ''.esc() + "\n" +
-                  ''.esc() +
-                  '
' - ); }, complete : function() { From 3cbc48ed3085ed2bac5096d6828a7f129003619d Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 2 Mar 2016 11:25:27 -0500 Subject: [PATCH 0002/1113] LUCENE-7059: always visit 1D points in sorted order; fix tie-break but in BKDWriter; fix BKDWriter to pass on maxMBSortInHeap to the OfflineSorter too --- .../org/apache/lucene/index/CheckIndex.java | 23 +- .../org/apache/lucene/index/PointValues.java | 4 +- .../lucene/index/PointValuesWriter.java | 1 + .../index/SlowCompositeReaderWrapper.java | 2 +- .../apache/lucene/search/PointInSetQuery.java | 13 - .../org/apache/lucene/util/OfflineSorter.java | 2 +- .../org/apache/lucene/util/bkd/BKDWriter.java | 41 ++- .../lucene/index/TestDuelingCodecs.java | 4 +- .../apache/lucene/index/TestPointValues.java | 62 +++++ .../lucene/search/TestPointQueries.java | 37 +-- .../facet/range/TestRangeFacetCounts.java | 8 +- .../apache/lucene/index/SorterTestBase.java | 6 +- .../lucene/document/TestBigIntegerPoint.java | 4 +- .../lucene/document/TestInetAddressPoint.java | 4 +- .../lucene/document/TestLatLonPoint.java | 2 +- .../TestLatLonPointDistanceQuery.java | 2 +- .../search/TestDocValuesRangeQuery.java | 4 +- .../asserting/AssertingPointFormat.java | 86 +++++- .../lucene/index/BasePointFormatTestCase.java | 262 ++++++++++-------- .../apache/lucene/util/LuceneTestCase.java | 111 ++++---- 20 files changed, 440 insertions(+), 238 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index 0c244617e525..9f711df4120c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -1706,8 +1706,12 @@ public static Status.PointsStatus testPoints(CodecReader reader, PrintStream inf lastMaxPacked.length = bytesPerDim; lastMinPacked.length = bytesPerDim; scratch.length = bytesPerDim; + byte[] lastPackedValue = new byte[dimCount*bytesPerDim]; values.intersect(fieldInfo.name, new PointValues.IntersectVisitor() { + + private int lastDocID = -1; + @Override public void visit(int docID) { throw new RuntimeException("codec called IntersectVisitor.visit without a packed value for docID=" + docID); @@ -1725,12 +1729,27 @@ public void visit(int docID, byte[] packedValue) { if (scratch.compareTo(lastMinPacked) < 0) { // This doc's point, in this dimension, is lower than the minimum value of the last cell checked: - throw new RuntimeException("packed value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); + throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); } if (scratch.compareTo(lastMaxPacked) > 0) { // This doc's point, in this dimension, is greater than the maximum value of the last cell checked: - throw new RuntimeException("packed value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); + throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-bounds of the last cell min=" + Arrays.toString(lastMinPackedValue) + " max=" + Arrays.toString(lastMaxPackedValue) + " dim=" + dim); + } + } + + // In the 1D case, PointValues must make a single in-order sweep through all values, and tie-break by + // increasing docID: + if (dimCount == 1) { + int cmp = StringHelper.compare(bytesPerDim, lastPackedValue, 0, packedValue, 0); + if (cmp > 0) { + throw new RuntimeException("packed points value " + Arrays.toString(packedValue) + " for docID=" + docID + " is out-of-order vs the previous document's value " + Arrays.toString(lastPackedValue)); + } else if (cmp == 0) { + if (docID < lastDocID) { + throw new RuntimeException("packed points value is the same, but docID=" + docID + " is out of order vs previous docID=" + lastDocID); + } } + System.arraycopy(packedValue, 0, lastPackedValue, 0, bytesPerDim); + lastDocID = docID; } status.totalValuePoints++; diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java index 108dde31050a..34d315fdde5a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java @@ -61,7 +61,9 @@ public interface IntersectVisitor { void visit(int docID) throws IOException; /** Called for all documents in a leaf cell that crosses the query. The consumer - * should scrutinize the packedValue to decide whether to accept it. */ + * should scrutinize the packedValue to decide whether to accept it. In the 1D case, + * values are visited in increasing order, and in the case of ties, in increasing + * docID order. */ void visit(int docID, byte[] packedValue) throws IOException; /** Called for non-leaf cells to test how the cell relates to the query, to diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java index 283f7bdbea0f..2fa8b4fa2b15 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java @@ -43,6 +43,7 @@ public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo packedValue = new byte[fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()]; } + // TODO: if exactly the same value is added to exactly the same doc, should we dedup? public void addPackedValue(int docID, BytesRef value) { if (value == null) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value cannot be null"); diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java index db9579cc13d4..e44c53cb54ea 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java @@ -236,7 +236,7 @@ public Bits getLiveDocs() { @Override public PointValues getPointValues() { ensureOpen(); - return MultiPointValues.get(in); + return null; } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 42f5bb19ff0b..3d6086c90858 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -172,10 +172,6 @@ public MergePointVisitor(PrefixCodedTerms sortedPackedPoints, DocIdSetBuilder re this.sortedPackedPoints = sortedPackedPoints; lastMaxPackedValue = new byte[bytesPerDim]; scratch.length = bytesPerDim; - resetIterator(); - } - - private void resetIterator() { this.iterator = sortedPackedPoints.iterator(); nextQueryPoint = iterator.next(); } @@ -211,15 +207,6 @@ public void visit(int docID, byte[] packedValue) { @Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - - // NOTE: this is messy ... we need it in cases where a single vistor (us) is shared across multiple leaf readers - // (e.g. SlowCompositeReaderWrapper), in which case we need to reset our iterator to re-start the merge sort. Maybe we should instead - // add an explicit .start() to IntersectVisitor, and clarify the semantics that in the 1D case all cells will be visited in order? - if (StringHelper.compare(bytesPerDim, lastMaxPackedValue, 0, minPackedValue, 0) > 0) { - resetIterator(); - } - System.arraycopy(maxPackedValue, 0, lastMaxPackedValue, 0, bytesPerDim); - while (nextQueryPoint != null) { scratch.bytes = minPackedValue; int cmpMin = nextQueryPoint.compareTo(scratch); diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java index b14610225f70..283dc1f90cb6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java @@ -120,7 +120,7 @@ public static BufferSize automatic() { final long minBufferSizeBytes = MIN_BUFFER_SIZE_MB*MB; if (sortBufferByteSize < minBufferSizeBytes || totalAvailableBytes > 10 * minBufferSizeBytes) { // lets see if we need/should to grow the heap - if (totalAvailableBytes/2 > minBufferSizeBytes){ // there is enough mem for a reasonable buffer + if (totalAvailableBytes/2 > minBufferSizeBytes) { // there is enough mem for a reasonable buffer sortBufferByteSize = totalAvailableBytes/2; // grow the heap } else { //heap seems smallish lets be conservative fall back to the free/2 diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 4397f2e1ab1c..9d6ad978878c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -102,6 +102,7 @@ public class BKDWriter implements Closeable { final TrackingDirectoryWrapper tempDir; final String tempFileNamePrefix; + final double maxMBSortInHeap; final byte[] scratchDiff; final byte[] scratchPackedValue; @@ -169,6 +170,8 @@ public BKDWriter(Directory tempDir, String tempFileNamePrefix, int numDims, int // We write first maxPointsSortInHeap in heap, then cutover to offline for additional points: heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength); + + this.maxMBSortInHeap = maxMBSortInHeap; } public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap) { @@ -550,7 +553,7 @@ private void sortHeapPointWriter(final HeapPointWriter writer, int start, int le //int[] swapCount = new int[1]; //int[] cmpCount = new int[1]; - //System.out.println("SORT length=" + length); + // System.out.println("SORT length=" + length); // All buffered points are still in heap; just do in-place sort: new IntroSorter() { @@ -623,13 +626,11 @@ protected int compare(int i, int j) { return cmp; } - // Tie-break - cmp = Integer.compare(writer.docIDs[i], writer.docIDs[j]); - if (cmp != 0) { - return cmp; - } + // Tie-break by docID: - return Long.compare(writer.ords[i], writer.ords[j]); + // No need to tie break on ord, for the case where the same doc has the same value in a given dimension indexed more than once: it + // can't matter at search time since we don't write ords into the index: + return Integer.compare(writer.docIDs[i], writer.docIDs[j]); } }.sort(start, start+length); //System.out.println("LEN=" + length + " SWAP=" + swapCount[0] + " CMP=" + cmpCount[0]); @@ -679,29 +680,23 @@ public int compare(BytesRef a, BytesRef b) { return cmp; } - // Tie-break by docID and then ord: - reader.reset(a.bytes, a.offset + packedBytesLength, a.length); - final int docIDA = reader.readVInt(); - final long ordA = reader.readVLong(); + // Tie-break by docID: + reader.reset(a.bytes, a.offset + packedBytesLength + Long.BYTES, a.length); + final int docIDA = reader.readInt(); - reader.reset(b.bytes, b.offset + packedBytesLength, b.length); - final int docIDB = reader.readVInt(); - final long ordB = reader.readVLong(); - - cmp = Integer.compare(docIDA, docIDB); - if (cmp != 0) { - return cmp; - } + reader.reset(b.bytes, b.offset + packedBytesLength + Long.BYTES, b.length); + final int docIDB = reader.readInt(); - // TODO: is this really necessary? If OfflineSorter is stable, we can safely return 0 here, and avoid writing ords? - return Long.compare(ordA, ordB); + // No need to tie break on ord, for the case where the same doc has the same value in a given dimension indexed more than once: it + // can't matter at search time since we don't write ords into the index: + return Integer.compare(docIDA, docIDB); } }; // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: IndexOutput[] lastWriter = new IndexOutput[1]; - OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, cmp) { + OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @Override @@ -753,7 +748,7 @@ private void checkMaxLeafNodeCount(int numLeaves) { /** Writes the BKD tree to the provided {@link IndexOutput} and returns the file offset where index was written. */ public long finish(IndexOutput out) throws IOException { - //System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + " heapWriter=" + heapWriter); + // System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + " heapWriter=" + heapPointWriter); // TODO: specialize the 1D case? it's much faster at indexing time (no partitioning on recruse...) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java index 62fe28aedb24..b79e638ba078 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java @@ -162,8 +162,8 @@ public void testCrazyReaderEquals() throws IOException { createRandomIndex(numdocs, leftWriter, seed); createRandomIndex(numdocs, rightWriter, seed); - leftReader = wrapReader(leftWriter.getReader()); - rightReader = wrapReader(rightWriter.getReader()); + leftReader = wrapReader(leftWriter.getReader(), false); + rightReader = wrapReader(rightWriter.getReader(), false); // check that our readers are valid TestUtil.checkReader(leftReader); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 3e1fe2d1ac83..7551d3c3f2d5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -17,8 +17,16 @@ package org.apache.lucene.index; +import java.io.IOException; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.PointFormat; +import org.apache.lucene.codecs.PointReader; +import org.apache.lucene.codecs.PointWriter; +import org.apache.lucene.codecs.lucene60.Lucene60PointReader; +import org.apache.lucene.codecs.lucene60.Lucene60PointWriter; import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; @@ -26,6 +34,9 @@ import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.index.PointValues; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; @@ -478,4 +489,55 @@ public void testInvalidDoublePointUsage() throws Exception { field.numericValue(); }); } + + public void testTieBreakByDocID() throws Exception { + Directory dir = newFSDirectory(createTempDir()); + IndexWriterConfig iwc = newIndexWriterConfig(); + IndexWriter w = new IndexWriter(dir, iwc); + Document doc = new Document(); + doc.add(new IntPoint("int", 17)); + for(int i=0;i<300000;i++) { + w.addDocument(doc); + if (false && random().nextInt(1000) == 17) { + w.commit(); + } + } + + IndexReader r = DirectoryReader.open(w); + + for(LeafReaderContext ctx : r.leaves()) { + PointValues points = ctx.reader().getPointValues(); + points.intersect("int", + new IntersectVisitor() { + + int lastDocID = -1; + + @Override + public void visit(int docID) { + if (docID < lastDocID) { + fail("docs out of order: docID=" + docID + " but lastDocID=" + lastDocID); + } + lastDocID = docID; + } + + @Override + public void visit(int docID, byte[] packedValue) { + visit(docID); + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + if (random().nextBoolean()) { + return Relation.CELL_CROSSES_QUERY; + } else { + return Relation.CELL_INSIDE_QUERY; + } + } + }); + } + + r.close(); + w.close(); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index 8a578fa5779f..fb3179298fb2 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -53,11 +53,13 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -360,7 +362,7 @@ private static void verifyLongs(long[] values, int[] ids) throws Exception { final IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); int numThreads = TestUtil.nextInt(random(), 2, 5); @@ -627,7 +629,7 @@ private void verifyBinary(byte[][][] docValues, int[] ids, int numBytesPerDim) t final IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); int numThreads = TestUtil.nextInt(random(), 2, 5); @@ -847,7 +849,7 @@ public void testMinMaxLong() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, 0L, true))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", 0L, true, Long.MAX_VALUE, true))); @@ -885,7 +887,7 @@ public void testBasicSortedSet() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("aaa"), @@ -950,7 +952,7 @@ public void testLongMinMaxNumeric() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, true))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, false))); @@ -976,7 +978,6 @@ public void testLongMinMaxSortedSet() throws Exception { IndexReader r = w.getReader(); - // We can't wrap with "exotic" readers because the query must see the RangeTreeDVFormat: IndexSearcher s = newSearcher(r, false); assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, true))); @@ -1003,7 +1004,7 @@ public void testSortedSetNoOrdsMatch() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("m"), true, toUTF8("n"), false))); assertEquals(2, s.count(BinaryPoint.newRangeQuery("value", (byte[]) null, true, null, true))); @@ -1040,7 +1041,7 @@ public void testNoDocs() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, true, 13L, false))); IOUtils.close(r, w, dir); @@ -1207,7 +1208,7 @@ public void testExactPoints() throws Exception { w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(1, s.count(IntPoint.newExactQuery("int", 42))); assertEquals(0, s.count(IntPoint.newExactQuery("int", 41))); @@ -1328,7 +1329,7 @@ public void testRandomPointInSetQuery() throws Exception { final IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); int numThreads = TestUtil.nextInt(random(), 2, 5); @@ -1467,7 +1468,7 @@ public void testBasicMultiDimPointInSetQuery() throws Exception { doc.add(new IntPoint("int", 17, 42)); w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(newMultiDimIntSetQuery("int", 2, 17, 41))); assertEquals(1, s.count(newMultiDimIntSetQuery("int", 2, 17, 42))); @@ -1490,7 +1491,7 @@ public void testBasicMultiValueMultiDimPointInSetQuery() throws Exception { doc.add(new IntPoint("int", 34, 79)); w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(newMultiDimIntSetQuery("int", 2, 17, 41))); assertEquals(1, s.count(newMultiDimIntSetQuery("int", 2, 17, 42))); @@ -1523,7 +1524,7 @@ public void testManyEqualValuesMultiDimPointInSetQuery() throws Exception { w.addDocument(doc); } IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(zeroCount, s.count(newMultiDimIntSetQuery("int", 2, 0, 0))); assertEquals(10000-zeroCount, s.count(newMultiDimIntSetQuery("int", 2, 1, 1))); @@ -1573,7 +1574,7 @@ public void testBasicPointInSetQuery() throws Exception { w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(IntPoint.newSetQuery("int", 16))); assertEquals(1, s.count(IntPoint.newSetQuery("int", 17))); assertEquals(3, s.count(IntPoint.newSetQuery("int", 17, 97, 42))); @@ -1634,7 +1635,7 @@ public void testBasicMultiValuedPointInSetQuery() throws Exception { w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(IntPoint.newSetQuery("int", 16))); assertEquals(1, s.count(IntPoint.newSetQuery("int", 17))); assertEquals(1, s.count(IntPoint.newSetQuery("int", 17, 97, 42))); @@ -1685,7 +1686,7 @@ public void testEmptyPointInSetQuery() throws Exception { w.addDocument(doc); IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(IntPoint.newSetQuery("int"))); assertEquals(0, s.count(LongPoint.newSetQuery("long"))); assertEquals(0, s.count(FloatPoint.newSetQuery("float"))); @@ -1719,7 +1720,7 @@ public void testPointInSetQueryManyEqualValues() throws Exception { } IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0))); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0, -7))); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 7, 0))); @@ -1777,7 +1778,7 @@ public void testPointInSetQueryManyEqualValuesWithBigGap() throws Exception { } IndexReader r = DirectoryReader.open(w); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0))); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 0, -7))); assertEquals(zeroCount, s.count(IntPoint.newSetQuery("int", 7, 0))); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index f7a1970ca422..c4233c04710b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -218,7 +218,7 @@ public void testMixedRangeAndNonRangeTaxonomy() throws Exception { final TaxonomyReader tr = new DirectoryTaxonomyReader(tw); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); if (VERBOSE) { System.out.println("TEST: searcher=" + s); @@ -375,7 +375,7 @@ public void testRandomLongs() throws Exception { } IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); FacetsConfig config = new FacetsConfig(); int numIters = atLeast(10); @@ -516,7 +516,7 @@ public void testRandomFloats() throws Exception { } IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); FacetsConfig config = new FacetsConfig(); int numIters = atLeast(10); @@ -671,7 +671,7 @@ public void testRandomDoubles() throws Exception { } IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); FacetsConfig config = new FacetsConfig(); int numIters = atLeast(10); diff --git a/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java b/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java index 93caa7016b86..0015c6c629fa 100644 --- a/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java +++ b/lucene/misc/src/test/org/apache/lucene/index/SorterTestBase.java @@ -174,7 +174,8 @@ private static Document doc(final int id, PositionsTokenStream positions) { doc.add(new Field(TERM_VECTORS_FIELD, Integer.toString(id), TERM_VECTORS_TYPE)); byte[] bytes = new byte[4]; NumericUtils.intToBytes(id, bytes, 0); - doc.add(new BinaryPoint(DIMENSIONAL_FIELD, bytes)); + // TODO: index time sorting doesn't yet support points + //doc.add(new BinaryPoint(DIMENSIONAL_FIELD, bytes)); return doc; } @@ -378,6 +379,8 @@ public void testTermVectors() throws Exception { } } + // TODO: index sorting doesn't yet support points + /* public void testPoints() throws Exception { PointValues values = sortedReader.getPointValues(); values.intersect(DIMENSIONAL_FIELD, @@ -398,4 +401,5 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { } }); } + */ } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index 3e8cf3db011c..f6d407de6d7b 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -40,7 +40,7 @@ public void testBasics() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", large))); assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", large.subtract(BigInteger.ONE), false, large.add(BigInteger.ONE), false))); assertEquals(1, searcher.count(BigIntegerPoint.newSetQuery("field", large))); @@ -65,7 +65,7 @@ public void testNegative() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", negative))); assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", negative.subtract(BigInteger.ONE), false, negative.add(BigInteger.ONE), false))); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index c9be31fc3126..9854001a2d02 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -40,7 +40,7 @@ public void testBasics() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 24))); assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), false, InetAddress.getByName("1.2.3.5"), false))); @@ -66,7 +66,7 @@ public void testBasicsV6() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 64))); assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("fec0::f66c"), false, InetAddress.getByName("fec0::f66e"), false))); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index b67dec83ccfe..1d3bfac30525 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -38,7 +38,7 @@ public void testBoxQuery() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(LatLonPoint.newBoxQuery("field", 18, 19, -66, -65))); reader.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java index 3d47b4464c88..e37d75eb441d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java @@ -55,7 +55,7 @@ public void testBasics() throws Exception { // search within 50km and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(LatLonPoint.newDistanceQuery("field", 18, -65, 50_000))); reader.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java index 1bcadd6b9300..47cd740d1d3c 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java @@ -60,7 +60,7 @@ public void testDuelNumericRangeQuery() throws IOException { } iw.commit(); final IndexReader reader = iw.getReader(); - final IndexSearcher searcher = newSearcher(reader); + final IndexSearcher searcher = newSearcher(reader, false); iw.close(); for (int i = 0; i < 100; ++i) { @@ -188,7 +188,7 @@ public void testApproximation() throws IOException { } iw.commit(); final IndexReader reader = iw.getReader(); - final IndexSearcher searcher = newSearcher(reader); + final IndexSearcher searcher = newSearcher(reader, false); iw.close(); for (int i = 0; i < 100; ++i) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java index 3411c7b4b162..15836de6ec0f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java @@ -17,6 +17,7 @@ package org.apache.lucene.codecs.asserting; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import org.apache.lucene.codecs.PointFormat; @@ -24,9 +25,13 @@ import org.apache.lucene.codecs.PointWriter; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; /** @@ -60,6 +65,83 @@ public PointWriter fieldsWriter(SegmentWriteState state) throws IOException { public PointReader fieldsReader(SegmentReadState state) throws IOException { return new AssertingPointReader(in.fieldsReader(state)); } + + /** Validates in the 1D case that all points are visited in order, and point values are in bounds of the last cell checked */ + static class AssertingIntersectVisitor implements IntersectVisitor { + final IntersectVisitor in; + final int numDims; + final int bytesPerDim; + final byte[] lastDocValue; + final byte[] lastMinPackedValue; + final byte[] lastMaxPackedValue; + private Relation lastCompareResult; + private int lastDocID = -1; + + public AssertingIntersectVisitor(int numDims, int bytesPerDim, IntersectVisitor in) { + this.in = in; + this.numDims = numDims; + this.bytesPerDim = bytesPerDim; + lastMaxPackedValue = new byte[numDims*bytesPerDim]; + lastMinPackedValue = new byte[numDims*bytesPerDim]; + if (numDims == 1) { + lastDocValue = new byte[bytesPerDim]; + } else { + lastDocValue = null; + } + } + + @Override + public void visit(int docID) throws IOException { + // This method, not filtering each hit, should only be invoked when the cell is inside the query shape: + assert lastCompareResult == Relation.CELL_INSIDE_QUERY; + in.visit(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + + // This method, to filter each doc's value, should only be invoked when the cell crosses the query shape: + assert lastCompareResult == PointValues.Relation.CELL_CROSSES_QUERY; + + // This doc's packed value should be contained in the last cell passed to compare: + for(int dim=0;dim= 0: "dim=" + dim + " of " + numDims; + } + + // TODO: we should assert that this "matches" whatever relation the last call to compare had returned + assert packedValue.length == numDims * bytesPerDim; + if (numDims == 1) { + int cmp = StringHelper.compare(bytesPerDim, lastDocValue, 0, packedValue, 0); + if (cmp < 0) { + // ok + } else if (cmp == 0) { + assert lastDocID <= docID: "doc ids are out of order when point values are the same!"; + } else { + // out of order! + assert false: "point values are out of order"; + } + System.arraycopy(packedValue, 0, lastDocValue, 0, bytesPerDim); + } + in.visit(docID, packedValue); + } + + @Override + public void grow(int count) { + in.grow(count); + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + for(int dim=0;dim= 0; - - if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) { - return Relation.CELL_OUTSIDE_QUERY; - } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) { - crosses = true; + @Override + public void visit(int docID, byte[] packedValue) { + //System.out.println("visit check docID=" + docID); + for(int dim=0;dim 0) { + //System.out.println(" no"); + return; + } } + + //System.out.println(" yes"); + hits.set(docBase+docID); } - if (crosses) { - return Relation.CELL_CROSSES_QUERY; - } else { - return Relation.CELL_INSIDE_QUERY; + @Override + public Relation compare(byte[] minPacked, byte[] maxPacked) { + boolean crosses = false; + for(int dim=0;dim= 0; + + if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) { + return Relation.CELL_OUTSIDE_QUERY; + } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) { + crosses = true; + } + } + + if (crosses) { + return Relation.CELL_CROSSES_QUERY; + } else { + return Relation.CELL_INSIDE_QUERY; + } } - } - }); + }); + } for(int docID=0;docID 0) { + System.arraycopy(leafMaxValues, dim*numBytesPerDim, maxValues, dim*numBytesPerDim, numBytesPerDim); + } + } + } + byte[] scratch = new byte[numBytesPerDim]; for(int dim=0;dim 0) { - //System.out.println(" query_outside_cell"); - return Relation.CELL_OUTSIDE_QUERY; - } else if (StringHelper.compare(numBytesPerDim, minPacked, dim*numBytesPerDim, queryMin[dim], 0) < 0 || - StringHelper.compare(numBytesPerDim, maxPacked, dim*numBytesPerDim, queryMax[dim], 0) > 0) { - crosses = true; + //System.out.println("visit check docID=" + docID + " id=" + idValues.get(docID)); + for(int dim=0;dim 0) { + //System.out.println(" query_outside_cell"); + return Relation.CELL_OUTSIDE_QUERY; + } else if (StringHelper.compare(numBytesPerDim, minPacked, dim*numBytesPerDim, queryMin[dim], 0) < 0 || + StringHelper.compare(numBytesPerDim, maxPacked, dim*numBytesPerDim, queryMax[dim], 0) > 0) { + crosses = true; + } + } + + if (crosses) { + //System.out.println(" query_crosses_cell"); + return Relation.CELL_CROSSES_QUERY; + } else { + //System.out.println(" cell_inside_query"); + return Relation.CELL_INSIDE_QUERY; + } } - } - }); + }); + } BitSet expected = new BitSet(); for(int ord=0;ord allFields = new ArrayList<>(); - for (FieldInfo fi : ar.getFieldInfos()) { - allFields.add(fi.name); - } - Collections.shuffle(allFields, random); - final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size()); - final Set fields = new HashSet<>(allFields.subList(0, end)); - // will create no FC insanity as ParallelLeafReader has own cache key: - if (VERBOSE) { - System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with ParallelLeafReader(SlowCompositeReaderWapper)"); + if (allowSlowCompositeReader) { + final LeafReader ar = SlowCompositeReaderWrapper.wrap(r); + final List allFields = new ArrayList<>(); + for (FieldInfo fi : ar.getFieldInfos()) { + allFields.add(fi.name); + } + Collections.shuffle(allFields, random); + final int end = allFields.isEmpty() ? 0 : random.nextInt(allFields.size()); + final Set fields = new HashSet<>(allFields.subList(0, end)); + // will create no FC insanity as ParallelLeafReader has own cache key: + if (VERBOSE) { + System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with ParallelLeafReader(SlowCompositeReaderWapper)"); + } + r = new ParallelLeafReader( + new FieldFilterLeafReader(ar, fields, false), + new FieldFilterLeafReader(ar, fields, true) + ); } - r = new ParallelLeafReader( - new FieldFilterLeafReader(ar, fields, false), - new FieldFilterLeafReader(ar, fields, true) - ); break; case 4: // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use @@ -1701,7 +1709,9 @@ public static IndexReader wrapReader(IndexReader r) throws IOException { } } if (wasOriginallyAtomic) { - r = SlowCompositeReaderWrapper.wrap(r); + if (allowSlowCompositeReader) { + r = SlowCompositeReaderWrapper.wrap(r); + } } else if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) { // prevent cache insanity caused by e.g. ParallelCompositeReader, to fix we wrap one more time: r = new FCInvisibleMultiReader(r); @@ -2588,40 +2598,45 @@ public void assertFieldInfosEquals(String info, IndexReader leftReader, IndexRea } // naive silly memory heavy uninversion!! maps docID -> packed values (a Set because a given doc can be multi-valued) - private Map> uninvert(String fieldName, PointValues points) throws IOException { + private Map> uninvert(String fieldName, IndexReader reader) throws IOException { final Map> docValues = new HashMap<>(); - points.intersect(fieldName, new PointValues.IntersectVisitor() { - @Override - public void visit(int docID) { - throw new UnsupportedOperationException(); - } + for(LeafReaderContext ctx : reader.leaves()) { - @Override - public void visit(int docID, byte[] packedValue) throws IOException { - if (docValues.containsKey(docID) == false) { - docValues.put(docID, new HashSet()); - } - docValues.get(docID).add(new BytesRef(packedValue.clone())); - } + PointValues points = ctx.reader().getPointValues(); + if (points == null) { + continue; + } + + points.intersect(fieldName, + new PointValues.IntersectVisitor() { + @Override + public void visit(int docID) { + throw new UnsupportedOperationException(); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + int topDocID = ctx.docBase + docID; + if (docValues.containsKey(topDocID) == false) { + docValues.put(topDocID, new HashSet()); + } + docValues.get(topDocID).add(new BytesRef(packedValue.clone())); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + // We pretend our query shape is so hairy that it crosses every single cell: + return PointValues.Relation.CELL_CROSSES_QUERY; + } + }); + } - @Override - public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - // We pretend our query shape is so hairy that it crosses every single cell: - return PointValues.Relation.CELL_CROSSES_QUERY; - } - }); return docValues; } public void assertPointsEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException { - assertPointsEquals(info, - MultiFields.getMergedFieldInfos(leftReader), - MultiPointValues.get(leftReader), - MultiFields.getMergedFieldInfos(rightReader), - MultiPointValues.get(rightReader)); - } - - public void assertPointsEquals(String info, FieldInfos fieldInfos1, PointValues points1, FieldInfos fieldInfos2, PointValues points2) throws IOException { + FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(leftReader); + FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(rightReader); for(FieldInfo fieldInfo1 : fieldInfos1) { if (fieldInfo1.getPointDimensionCount() != 0) { FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name); @@ -2631,8 +2646,8 @@ public void assertPointsEquals(String info, FieldInfos fieldInfos1, PointValues assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes()); assertEquals(info + " field=" + fieldInfo1.name, - uninvert(fieldInfo1.name, points1), - uninvert(fieldInfo1.name, points2)); + uninvert(fieldInfo1.name, leftReader), + uninvert(fieldInfo1.name, rightReader)); } } From e1033d965414b34b990070bb87c509364a7f4194 Mon Sep 17 00:00:00 2001 From: thelabdude Date: Wed, 2 Mar 2016 11:22:27 -0700 Subject: [PATCH 0003/1113] SOLR-8145: Fix position of OOM killer script when starting Solr in the background --- solr/bin/solr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/solr/bin/solr b/solr/bin/solr index 904c2c389e1f..9cf8ae53d767 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -1420,8 +1420,9 @@ function launch_solr() { exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}" else # run Solr in the background - nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar \ - "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" "${SOLR_JETTY_CONFIG[@]}" \ + nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS \ + "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \ + -jar start.jar "${SOLR_JETTY_CONFIG[@]}" \ 1>"$SOLR_LOGS_DIR/solr-$SOLR_PORT-console.log" 2>&1 & echo $! > "$SOLR_PID_DIR/solr-$SOLR_PORT.pid" # no lsof on cygwin though From ddd019fac0d9eff352a4a17a62d9a9654f7bdc86 Mon Sep 17 00:00:00 2001 From: thelabdude Date: Wed, 2 Mar 2016 11:37:23 -0700 Subject: [PATCH 0004/1113] SOLR-8145: mention fix in solr/CHANGES.txt --- solr/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3120eeb45e72..eb4195850ca4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -233,6 +233,9 @@ Bug Fixes * SOLR-8771: Multi-threaded core shutdown creates executor per core. (Mike Drob via Mark Miller) +* SOLR-8145: Fix position of OOM killer script when starting Solr in the background (Jurian Broertjes via + Timothy Potter) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been From 25cc48bbb8cc11c2f7a2c9da30d675c9ae5926ca Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 2 Mar 2016 17:19:24 -0500 Subject: [PATCH 0005/1113] LUCENE-7059: remove MultiPointValues --- .../apache/lucene/index/MultiPointValues.java | 172 ------------------ 1 file changed, 172 deletions(-) delete mode 100644 lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java deleted file mode 100644 index dcc33da4aedb..000000000000 --- a/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.index; - - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.util.StringHelper; - -/** Merges multiple {@link PointValues} into a single one. */ -public class MultiPointValues extends PointValues { - - private final List subs; - private final List docBases; - - private MultiPointValues(List subs, List docBases) { - this.subs = subs; - this.docBases = docBases; - } - - /** Returns a {@link PointValues} merging all point values from the provided reader. */ - public static PointValues get(IndexReader r) { - final List leaves = r.leaves(); - final int size = leaves.size(); - if (size == 0) { - return null; - } else if (size == 1) { - return leaves.get(0).reader().getPointValues(); - } - - List values = new ArrayList<>(); - List docBases = new ArrayList<>(); - for (int i = 0; i < size; i++) { - LeafReaderContext context = leaves.get(i); - PointValues v = context.reader().getPointValues(); - if (v != null) { - values.add(v); - docBases.add(context.docBase); - } - } - - if (values.isEmpty()) { - return null; - } - - return new MultiPointValues(values, docBases); - } - - /** Finds all documents and points matching the provided visitor */ - public void intersect(String fieldName, IntersectVisitor visitor) throws IOException { - for(int i=0;i 0) { - b.append(", "); - } - b.append("docBase="); - b.append(docBases.get(i)); - b.append(" sub=" + subs.get(i)); - } - b.append(')'); - return b.toString(); - } - - @Override - public byte[] getMinPackedValue(String fieldName) throws IOException { - byte[] result = null; - for(int i=0;i 0) { - System.arraycopy(maxPackedValue, offset, result, offset, bytesPerDim); - } - } - } - } - - return result; - } - - @Override - public int getNumDimensions(String fieldName) throws IOException { - for(int i=0;i Date: Wed, 2 Mar 2016 16:51:16 -0500 Subject: [PATCH 0006/1113] LUCENE-7061: fix remaining api issues with XYZPoint classes Squashed commit of the following: commit 0261e28dd29b1c2a1dcbd5e796966b2cdf2f4b82 Author: Robert Muir Date: Wed Mar 2 14:51:49 2016 -0500 Add note about comparison order and test extreme values commit e1f7bc244cd980e931d584c00ba73f8ac521c3fd Author: Mike McCandless Date: Wed Mar 2 14:25:50 2016 -0500 add explicit test to verify rect query is inclusive; make test fail fast by default commit c9be9139ec2f9553ce05fb56b2667be77b8176b6 Author: Robert Muir Date: Wed Mar 2 14:03:47 2016 -0500 oops commit 351d0838bbc87dc7c6d83476bd9cb7ce6c38fc3c Author: Robert Muir Date: Wed Mar 2 13:53:42 2016 -0500 clean up pointrangequery: remove nulls and inclusives commit 0796057a8041ddf43341611b477502fa2307f0b1 Merge: 742ee02 e3198ca Author: Robert Muir Date: Wed Mar 2 13:26:48 2016 -0500 Merge branch 'master' into unfuck_points commit 742ee02aaf55439463daddbd3ea16c5e8df31f01 Author: Robert Muir Date: Wed Mar 2 13:16:08 2016 -0500 Remove nulls and inclusives from points apis --- .../apache/lucene/document/BinaryPoint.java | 82 ++-- .../apache/lucene/document/DoublePoint.java | 84 ++-- .../apache/lucene/document/FloatPoint.java | 84 ++-- .../org/apache/lucene/document/IntPoint.java | 84 ++-- .../org/apache/lucene/document/LongPoint.java | 84 ++-- .../apache/lucene/search/PointInSetQuery.java | 11 +- .../apache/lucene/search/PointRangeQuery.java | 170 ++------ .../index/TestDemoParallelLeafReader.java | 2 +- .../lucene/search/TestPointQueries.java | 410 +++++++++--------- .../TestUsageTrackingFilterCachingPolicy.java | 2 +- .../demo/facet/DistanceFacetsExample.java | 8 +- .../lucene/demo/facet/RangeFacetsExample.java | 3 +- .../lucene/facet/range/DoubleRange.java | 33 +- .../facet/range/DoubleRangeFacetCounts.java | 4 +- .../apache/lucene/facet/range/LongRange.java | 25 +- .../lucene/facet/range/LongRangeCounter.java | 14 +- .../facet/range/TestRangeFacetCounts.java | 20 +- .../search/highlight/HighlighterTest.java | 2 +- .../lucene/document/BigIntegerPoint.java | 97 ++--- .../lucene/document/InetAddressPoint.java | 66 ++- .../apache/lucene/document/LatLonPoint.java | 11 +- .../lucene/document/TestBigIntegerPoint.java | 12 +- .../lucene/document/TestInetAddressPoint.java | 4 +- .../lucene/document/TestLatLonPoint.java | 2 +- .../search/TestDocValuesRangeQuery.java | 26 +- .../lucene/search/TestLatLonPointQueries.java | 8 +- .../spatial/util/BaseGeoPointTestCase.java | 74 +++- .../suggest/document/TestSuggestField.java | 2 +- 28 files changed, 636 insertions(+), 788 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java index b8b53d5b2157..e139a877264a 100644 --- a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.util.Arrays; import java.util.Comparator; @@ -25,7 +24,6 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.StringHelper; /** @@ -37,10 +35,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, byte[])} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, byte[][]) newSetQuery(String, byte[]...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, byte[], byte[])} for matching a 1D range. + *
  • {@link #newRangeQuery(String, byte[][], byte[][])} for matching points/ranges in n-dimensional space. *
*/ public final class BinaryPoint extends Field { @@ -133,7 +131,7 @@ public BinaryPoint(String name, byte[] packedPoint, FieldType type) { * Create a query for matching an exact binary value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, byte[][], byte[][])} instead. * * @param field field name. must not be {@code null}. * @param value binary value @@ -141,56 +139,39 @@ public BinaryPoint(String name, byte[] packedPoint, FieldType type) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, byte[] value) { - if (value == null) { - throw new IllegalArgumentException("value cannot be null"); - } - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for binary values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. - *

- * You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. - *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * {@link #newRangeQuery(String, byte[][], byte[][])} instead. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null. + * @param lowerValue lower portion of the range (inclusive). must not be {@code null} + * @param upperValue upper portion of the range (inclusive). must not be {@code null} + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, + * or if {@code upperValue} is null * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, byte[] lowerValue, boolean lowerInclusive, byte[] upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, new byte[][] {lowerValue}, new boolean[] {lowerInclusive}, new byte[][] {upperValue}, new boolean[] {upperInclusive}); + public static Query newRangeQuery(String field, byte[] lowerValue, byte[] upperValue) { + PointRangeQuery.checkArgs(field, lowerValue, upperValue); + return newRangeQuery(field, new byte[][] {lowerValue}, new byte[][] {upperValue}); } /** - * Create a multidimensional range query for binary values. - *

- * You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. - *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Create a range query for n-dimensional binary values. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be null. + * @param upperValue upper portion of the range (inclusive). must not be null. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, byte[][] lowerValue, boolean[] lowerInclusive, byte[][] upperValue, boolean[] upperInclusive) { - PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, lowerValue, lowerInclusive, upperValue, upperInclusive) { + public static Query newRangeQuery(String field, byte[][] lowerValue, byte[][] upperValue) { + return new PointRangeQuery(field, lowerValue, upperValue) { @Override protected String toString(int dimension, byte[] value) { assert value != null; @@ -212,13 +193,13 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, byte[]... valuesIn) throws IOException { + public static Query newSetQuery(String field, byte[]... values) { // Make sure all byte[] have the same length int bytesPerDim = -1; - for(byte[] value : valuesIn) { + for(byte[] value : values) { if (bytesPerDim == -1) { bytesPerDim = value.length; } else if (value.length != bytesPerDim) { @@ -232,9 +213,8 @@ public static Query newSetQuery(String field, byte[]... valuesIn) throws IOExcep } // Don't unexpectedly change the user's incoming values array: - byte[][] values = valuesIn.clone(); - - Arrays.sort(values, + byte[][] sortedValues = values.clone(); + Arrays.sort(sortedValues, new Comparator() { @Override public int compare(byte[] a, byte[] b) { @@ -242,21 +222,21 @@ public int compare(byte[] a, byte[] b) { } }); - final BytesRef value = new BytesRef(new byte[bytesPerDim]); + final BytesRef encoded = new BytesRef(new byte[bytesPerDim]); return new PointInSetQuery(field, 1, bytesPerDim, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - value.bytes = values[upto]; + encoded.bytes = sortedValues[upto]; upto++; - return value; + return encoded; } } }) { diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java index 58ec364f58d9..9dbd96e4e3e6 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java @@ -16,14 +16,12 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.util.Arrays; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.NumericUtils; /** @@ -35,10 +33,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, double)} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, double...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, double, double)} for matching a 1D range. + *
  • {@link #newRangeQuery(String, double[], double[])} for matching points/ranges in n-dimensional space. *
*/ public final class DoublePoint extends Field { @@ -126,13 +124,11 @@ public String toString() { } /** Encode n-dimensional double point into binary encoding */ - private static byte[][] encode(Double value[]) { + private static byte[][] encode(double value[]) { byte[][] encoded = new byte[value.length][]; for (int i = 0; i < value.length; i++) { - if (value[i] != null) { - encoded[i] = new byte[Double.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } + encoded[i] = new byte[Double.BYTES]; + encodeDimension(value[i], encoded[i], 0); } return encoded; } @@ -155,7 +151,7 @@ public static double decodeDimension(byte value[], int offset) { * Create a query for matching an exact double value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, double[], double[])} instead. * * @param field field name. must not be {@code null}. * @param value double value @@ -163,60 +159,51 @@ public static double decodeDimension(byte value[], int offset) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, double value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for double values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, double[], double[])} instead. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. + * by setting {@code lowerValue = Double.NEGATIVE_INFINITY} or {@code upperValue = Double.POSITIVE_INFINITY}. *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Range comparisons are consistent with {@link Double#compareTo(Double)}. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. + * @param lowerValue lower portion of the range (inclusive). + * @param upperValue upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null. * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, Double lowerValue, boolean lowerInclusive, Double upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, - new Double[] { lowerValue }, - new boolean[] { lowerInclusive }, - new Double[] { upperValue }, - new boolean[] { upperInclusive }); + public static Query newRangeQuery(String field, double lowerValue, double upperValue) { + return newRangeQuery(field, new double[] { lowerValue }, new double[] { upperValue }); } /** - * Create a multidimensional range query for double values. + * Create a range query for n-dimensional double values. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. + * by setting {@code lowerValue[i] = Double.NEGATIVE_INFINITY} or {@code upperValue[i] = Double.POSITIVE_INFINITY}. *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Range comparisons are consistent with {@link Double#compareTo(Double)}. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, Double[] lowerValue, boolean lowerInclusive[], Double[] upperValue, boolean upperInclusive[]) { + public static Query newRangeQuery(String field, double[] lowerValue, double[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, DoublePoint.encode(lowerValue), lowerInclusive, DoublePoint.encode(upperValue), upperInclusive) { + return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { @Override protected String toString(int dimension, byte[] value) { - return Double.toString(DoublePoint.decodeDimension(value, 0)); + return Double.toString(decodeDimension(value, 0)); } }; } @@ -225,30 +212,29 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, double... valuesIn) throws IOException { + public static Query newSetQuery(String field, double... values) { // Don't unexpectedly change the user's incoming values array: - double[] values = valuesIn.clone(); - - Arrays.sort(values); + double[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[Double.BYTES]); + final BytesRef encoded = new BytesRef(new byte[Double.BYTES]); return new PointInSetQuery(field, 1, Double.BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - encodeDimension(values[upto], value.bytes, 0); + encodeDimension(sortedValues[upto], encoded.bytes, 0); upto++; - return value; + return encoded; } } }) { diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java index 7829f80d712f..0b82abe11281 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java @@ -16,14 +16,12 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.util.Arrays; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.NumericUtils; /** @@ -35,10 +33,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, float)} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, float...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, float, float)} for matching a 1D range. + *
  • {@link #newRangeQuery(String, float[], float[])} for matching points/ranges in n-dimensional space. *
*/ public final class FloatPoint extends Field { @@ -126,13 +124,11 @@ public String toString() { } /** Encode n-dimensional float values into binary encoding */ - private static byte[][] encode(Float value[]) { + private static byte[][] encode(float value[]) { byte[][] encoded = new byte[value.length][]; for (int i = 0; i < value.length; i++) { - if (value[i] != null) { - encoded[i] = new byte[Float.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } + encoded[i] = new byte[Float.BYTES]; + encodeDimension(value[i], encoded[i], 0); } return encoded; } @@ -155,7 +151,7 @@ public static float decodeDimension(byte value[], int offset) { * Create a query for matching an exact float value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, float[], float[])} instead. * * @param field field name. must not be {@code null}. * @param value float value @@ -163,60 +159,51 @@ public static float decodeDimension(byte value[], int offset) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, float value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for float values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, float[], float[])} instead. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. + * by setting {@code lowerValue = Float.NEGATIVE_INFINITY} or {@code upperValue = Float.POSITIVE_INFINITY}. *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Range comparisons are consistent with {@link Float#compareTo(Float)}. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. + * @param lowerValue lower portion of the range (inclusive). + * @param upperValue upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null. * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, Float lowerValue, boolean lowerInclusive, Float upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, - new Float[] { lowerValue }, - new boolean[] { lowerInclusive }, - new Float[] { upperValue }, - new boolean[] { upperInclusive }); + public static Query newRangeQuery(String field, float lowerValue, float upperValue) { + return newRangeQuery(field, new float[] { lowerValue }, new float[] { upperValue }); } /** - * Create a multidimensional range query for float values. + * Create a range query for n-dimensional float values. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. + * by setting {@code lowerValue[i] = Float.NEGATIVE_INFINITY} or {@code upperValue[i] = Float.POSITIVE_INFINITY}. *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Range comparisons are consistent with {@link Float#compareTo(Float)}. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, Float[] lowerValue, boolean lowerInclusive[], Float[] upperValue, boolean upperInclusive[]) { + public static Query newRangeQuery(String field, float[] lowerValue, float[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, FloatPoint.encode(lowerValue), lowerInclusive, FloatPoint.encode(upperValue), upperInclusive) { + return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { @Override protected String toString(int dimension, byte[] value) { - return Float.toString(FloatPoint.decodeDimension(value, 0)); + return Float.toString(decodeDimension(value, 0)); } }; } @@ -225,30 +212,29 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, float... valuesIn) throws IOException { + public static Query newSetQuery(String field, float... values) { // Don't unexpectedly change the user's incoming values array: - float[] values = valuesIn.clone(); - - Arrays.sort(values); + float[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[Float.BYTES]); + final BytesRef encoded = new BytesRef(new byte[Float.BYTES]); return new PointInSetQuery(field, 1, Float.BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - encodeDimension(values[upto], value.bytes, 0); + encodeDimension(sortedValues[upto], encoded.bytes, 0); upto++; - return value; + return encoded; } } }) { diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java index f27df755863e..effcb62945ac 100644 --- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java @@ -16,14 +16,12 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.util.Arrays; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.NumericUtils; /** @@ -35,10 +33,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, int)} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, int...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, int, int)} for matching a 1D range. + *
  • {@link #newRangeQuery(String, int[], int[])} for matching points/ranges in n-dimensional space. *
*/ public final class IntPoint extends Field { @@ -126,13 +124,11 @@ public String toString() { } /** Encode n-dimensional integer values into binary encoding */ - private static byte[][] encode(Integer value[]) { + private static byte[][] encode(int value[]) { byte[][] encoded = new byte[value.length][]; for (int i = 0; i < value.length; i++) { - if (value[i] != null) { - encoded[i] = new byte[Integer.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } + encoded[i] = new byte[Integer.BYTES]; + encodeDimension(value[i], encoded[i], 0); } return encoded; } @@ -155,7 +151,7 @@ public static int decodeDimension(byte value[], int offset) { * Create a query for matching an exact integer value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, int[], int[])} instead. * * @param field field name. must not be {@code null}. * @param value exact value @@ -163,60 +159,51 @@ public static int decodeDimension(byte value[], int offset) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, int value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for integer values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, int[], int[])} instead. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. + * by setting {@code lowerValue = Integer.MIN_VALUE} or {@code upperValue = Integer.MAX_VALUE}. *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue + 1} or {@code upperValue - 1} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. + * @param lowerValue lower portion of the range (inclusive). + * @param upperValue upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null. * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, Integer lowerValue, boolean lowerInclusive, Integer upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, - new Integer[] { lowerValue }, - new boolean[] { lowerInclusive }, - new Integer[] { upperValue }, - new boolean[] { upperInclusive }); + public static Query newRangeQuery(String field, int lowerValue, int upperValue) { + return newRangeQuery(field, new int[] { lowerValue }, new int[] { upperValue }); } /** - * Create a multidimensional range query for integer values. + * Create a range query for n-dimensional integer values. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. + * by setting {@code lowerValue[i] = Integer.MIN_VALUE} or {@code upperValue[i] = Integer.MAX_VALUE}. *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue[i] + 1} or {@code upperValue[i] - 1} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, Integer[] lowerValue, boolean lowerInclusive[], Integer[] upperValue, boolean upperInclusive[]) { + public static Query newRangeQuery(String field, int[] lowerValue, int[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, IntPoint.encode(lowerValue), lowerInclusive, IntPoint.encode(upperValue), upperInclusive) { + return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { @Override protected String toString(int dimension, byte[] value) { - return Integer.toString(IntPoint.decodeDimension(value, 0)); + return Integer.toString(decodeDimension(value, 0)); } }; } @@ -225,30 +212,29 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, int... valuesIn) throws IOException { + public static Query newSetQuery(String field, int... values) { // Don't unexpectedly change the user's incoming values array: - int[] values = valuesIn.clone(); - - Arrays.sort(values); + int[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[Integer.BYTES]); + final BytesRef encoded = new BytesRef(new byte[Integer.BYTES]); return new PointInSetQuery(field, 1, Integer.BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - encodeDimension(values[upto], value.bytes, 0); + encodeDimension(sortedValues[upto], encoded.bytes, 0); upto++; - return value; + return encoded; } } }) { diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java index 5b3708942857..a1d05d1deff9 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java @@ -16,14 +16,12 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.util.Arrays; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.NumericUtils; /** @@ -35,10 +33,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, long)} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, long...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, long, long)} for matching a 1D range. + *
  • {@link #newRangeQuery(String, long[], long[])} for matching points/ranges in n-dimensional space. *
*/ public final class LongPoint extends Field { @@ -126,13 +124,11 @@ public String toString() { } /** Encode n-dimensional long values into binary encoding */ - private static byte[][] encode(Long value[]) { + private static byte[][] encode(long value[]) { byte[][] encoded = new byte[value.length][]; for (int i = 0; i < value.length; i++) { - if (value[i] != null) { - encoded[i] = new byte[Long.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } + encoded[i] = new byte[Long.BYTES]; + encodeDimension(value[i], encoded[i], 0); } return encoded; } @@ -155,7 +151,7 @@ public static long decodeDimension(byte value[], int offset) { * Create a query for matching an exact long value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, long[], long[])} instead. * * @param field field name. must not be {@code null}. * @param value exact value @@ -163,60 +159,51 @@ public static long decodeDimension(byte value[], int offset) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, long value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for long values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, long[], long[])} instead. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. + * by setting {@code lowerValue = Long.MIN_VALUE} or {@code upperValue = Long.MAX_VALUE}. *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue + 1} or {@code upperValue - 1} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. + * @param lowerValue lower portion of the range (inclusive). + * @param upperValue upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null. * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, Long lowerValue, boolean lowerInclusive, Long upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, - new Long[] { lowerValue }, - new boolean[] { lowerInclusive }, - new Long[] { upperValue }, - new boolean[] { upperInclusive }); + public static Query newRangeQuery(String field, long lowerValue, long upperValue) { + return newRangeQuery(field, new long[] { lowerValue }, new long[] { upperValue }); } /** - * Create a multidimensional range query for long values. + * Create a range query for n-dimensional long values. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. + * by setting {@code lowerValue[i] = Long.MIN_VALUE} or {@code upperValue[i] = Long.MAX_VALUE}. *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue[i] + 1} or {@code upperValue[i] - 1} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, Long[] lowerValue, boolean lowerInclusive[], Long[] upperValue, boolean upperInclusive[]) { + public static Query newRangeQuery(String field, long[] lowerValue, long[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, LongPoint.encode(lowerValue), lowerInclusive, LongPoint.encode(upperValue), upperInclusive) { + return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { @Override protected String toString(int dimension, byte[] value) { - return Long.toString(LongPoint.decodeDimension(value, 0)); + return Long.toString(decodeDimension(value, 0)); } }; } @@ -225,30 +212,29 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, long... valuesIn) throws IOException { + public static Query newSetQuery(String field, long... values) { // Don't unexpectedly change the user's incoming values array: - long[] values = valuesIn.clone(); - - Arrays.sort(values); + long[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[Long.BYTES]); + final BytesRef encoded = new BytesRef(new byte[Long.BYTES]); return new PointInSetQuery(field, 1, Long.BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - encodeDimension(values[upto], value.bytes, 0); + encodeDimension(sortedValues[upto], encoded.bytes, 0); upto++; - return value; + return encoded; } } }) { diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 3d6086c90858..f5ba12dc0db6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -64,9 +64,18 @@ public abstract class PointInSetQuery extends Query { final String field; final int numDims; final int bytesPerDim; + + /** + * Iterator of encoded point values. + */ + // TODO: if we want to stream, maybe we should use jdk stream class? + public static abstract class Stream implements BytesRefIterator { + @Override + public abstract BytesRef next(); + }; /** The {@code packedPoints} iterator must be in sorted order. */ - protected PointInSetQuery(String field, int numDims, int bytesPerDim, BytesRefIterator packedPoints) throws IOException { + protected PointInSetQuery(String field, int numDims, int bytesPerDim, Stream packedPoints) { this.field = field; if (bytesPerDim < 1 || bytesPerDim > PointValues.MAX_NUM_BYTES) { throw new IllegalArgumentException("bytesPerDim must be > 0 and <= " + PointValues.MAX_NUM_BYTES + "; got " + bytesPerDim); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 189ba433f317..85c486e7ddea 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.search; - import java.io.IOException; import java.util.Arrays; import java.util.Objects; @@ -33,7 +32,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.DocIdSetBuilder; -import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.StringHelper; /** @@ -57,72 +55,49 @@ public abstract class PointRangeQuery extends Query { final String field; final int numDims; + final int bytesPerDim; final byte[][] lowerPoint; - final boolean[] lowerInclusive; final byte[][] upperPoint; - final boolean[] upperInclusive; - // This is null only in the "fully open range" case - final Integer bytesPerDim; /** * Expert: create a multidimensional range query for point values. - *

- * You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. - *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. * * @param field field name. must not be {@code null}. - * @param lowerPoint lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperPoint upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. + * @param lowerPoint lower portion of the range (inclusive). + * @param upperPoint upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} */ - protected PointRangeQuery(String field, - byte[][] lowerPoint, boolean[] lowerInclusive, - byte[][] upperPoint, boolean[] upperInclusive) { + protected PointRangeQuery(String field, byte[][] lowerPoint, byte[][] upperPoint) { checkArgs(field, lowerPoint, upperPoint); this.field = field; - numDims = lowerPoint.length; + if (lowerPoint.length == 0) { + throw new IllegalArgumentException("lowerPoint has length of zero"); + } + this.numDims = lowerPoint.length; + if (upperPoint.length != numDims) { throw new IllegalArgumentException("lowerPoint has length=" + numDims + " but upperPoint has different length=" + upperPoint.length); } - if (lowerInclusive.length != numDims) { - throw new IllegalArgumentException("lowerInclusive has length=" + lowerInclusive.length + " but expected=" + numDims); - } - if (upperInclusive.length != numDims) { - throw new IllegalArgumentException("upperInclusive has length=" + upperInclusive.length + " but expected=" + numDims); - } this.lowerPoint = lowerPoint; - this.lowerInclusive = lowerInclusive; this.upperPoint = upperPoint; - this.upperInclusive = upperInclusive; - - int bytesPerDim = -1; - for(byte[] value : lowerPoint) { - if (value != null) { - if (bytesPerDim == -1) { - bytesPerDim = value.length; - } else if (value.length != bytesPerDim) { - throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + value.length); - } - } + + if (lowerPoint[0] == null) { + throw new IllegalArgumentException("lowerPoint[0] is null"); } - for(byte[] value : upperPoint) { - if (value != null) { - if (bytesPerDim == -1) { - bytesPerDim = value.length; - } else if (value.length != bytesPerDim) { - throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + value.length); - } + this.bytesPerDim = lowerPoint[0].length; + for (int i = 0; i < numDims; i++) { + if (lowerPoint[i] == null) { + throw new IllegalArgumentException("lowerPoint[" + i + "] is null"); + } + if (upperPoint[i] == null) { + throw new IllegalArgumentException("upperPoint[" + i + "] is null"); + } + if (lowerPoint[i].length != bytesPerDim) { + throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + lowerPoint[i].length); + } + if (upperPoint[i].length != bytesPerDim) { + throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + upperPoint[i].length); } - } - if (bytesPerDim == -1) { - this.bytesPerDim = null; - } else { - this.bytesPerDim = bytesPerDim; } } @@ -166,55 +141,18 @@ public Scorer scorer(LeafReaderContext context) throws IOException { if (fieldInfo.getPointDimensionCount() != numDims) { throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() + " but this query has numDims=" + numDims); } - if (bytesPerDim != null && bytesPerDim.intValue() != fieldInfo.getPointNumBytes()) { + if (bytesPerDim != fieldInfo.getPointNumBytes()) { throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + fieldInfo.getPointNumBytes() + " but this query has bytesPerDim=" + bytesPerDim); } int bytesPerDim = fieldInfo.getPointNumBytes(); - byte[] packedLowerIncl = new byte[numDims * bytesPerDim]; - byte[] packedUpperIncl = new byte[numDims * bytesPerDim]; - - byte[] minValue = new byte[bytesPerDim]; - byte[] maxValue = new byte[bytesPerDim]; - Arrays.fill(maxValue, (byte) 0xff); - - byte[] one = new byte[bytesPerDim]; - one[bytesPerDim-1] = 1; + byte[] packedLower = new byte[numDims * bytesPerDim]; + byte[] packedUpper = new byte[numDims * bytesPerDim]; - // Carefully pack lower and upper bounds, taking care of per-dim inclusive: + // Carefully pack lower and upper bounds for(int dim=0;dim 0) { + if (StringHelper.compare(bytesPerDim, packedValue, offset, packedUpper, offset) > 0) { // Doc's value is too high, in this dimension return; } @@ -260,13 +198,13 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { for(int dim=0;dim 0 || - StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedLowerIncl, offset) < 0) { + if (StringHelper.compare(bytesPerDim, minPackedValue, offset, packedUpper, offset) > 0 || + StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedLower, offset) < 0) { return Relation.CELL_OUTSIDE_QUERY; } - crosses |= StringHelper.compare(bytesPerDim, minPackedValue, offset, packedLowerIncl, offset) < 0 || - StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedUpperIncl, offset) > 0; + crosses |= StringHelper.compare(bytesPerDim, minPackedValue, offset, packedLower, offset) < 0 || + StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedUpper, offset) > 0; } if (crosses) { @@ -287,8 +225,6 @@ public int hashCode() { int hash = super.hashCode(); hash = 31 * hash + Arrays.hashCode(lowerPoint); hash = 31 * hash + Arrays.hashCode(upperPoint); - hash = 31 * hash + Arrays.hashCode(lowerInclusive); - hash = 31 * hash + Arrays.hashCode(upperInclusive); hash = 31 * hash + numDims; hash = 31 * hash + Objects.hashCode(bytesPerDim); return hash; @@ -301,9 +237,7 @@ public boolean equals(Object other) { return q.numDims == numDims && q.bytesPerDim == bytesPerDim && Arrays.equals(lowerPoint, q.lowerPoint) && - Arrays.equals(lowerInclusive, q.lowerInclusive) && - Arrays.equals(upperPoint, q.upperPoint) && - Arrays.equals(upperInclusive, q.upperInclusive); + Arrays.equals(upperPoint, q.upperPoint); } return false; @@ -323,31 +257,11 @@ public String toString(String field) { sb.append(','); } - if (lowerInclusive[i]) { - sb.append('['); - } else { - sb.append('{'); - } - - if (lowerPoint[i] == null) { - sb.append('*'); - } else { - sb.append(toString(i, lowerPoint[i])); - } - + sb.append('['); + sb.append(toString(i, lowerPoint[i])); sb.append(" TO "); - - if (upperPoint[i] == null) { - sb.append('*'); - } else { - sb.append(toString(i, upperPoint[i])); - } - - if (upperInclusive[i]) { - sb.append(']'); - } else { - sb.append('}'); - } + sb.append(toString(i, upperPoint[i])); + sb.append(']'); } return sb.toString(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java index 0f265512632a..0034cee0c453 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java @@ -1351,7 +1351,7 @@ private static void testPointRangeQuery(IndexSearcher s) throws IOException { max = x; } - TopDocs hits = s.search(LongPoint.newRangeQuery("number", min, true, max, true), 100); + TopDocs hits = s.search(LongPoint.newRangeQuery("number", min, max), 100); for(ScoreDoc scoreDoc : hits.scoreDocs) { long value = Long.parseLong(s.doc(scoreDoc.doc).get("text").split(" ")[1]); assertTrue(value >= min); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index fb3179298fb2..19096c760035 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -63,7 +63,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; @@ -118,8 +117,8 @@ public void testBasicInts() throws Exception { DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(2, s.count(IntPoint.newRangeQuery("point", -8, false, 1, false))); - assertEquals(3, s.count(IntPoint.newRangeQuery("point", -7, true, 3, true))); + assertEquals(2, s.count(IntPoint.newRangeQuery("point", -8, 1))); + assertEquals(3, s.count(IntPoint.newRangeQuery("point", -7, 3))); assertEquals(1, s.count(IntPoint.newExactQuery("point", -7))); assertEquals(0, s.count(IntPoint.newExactQuery("point", -6))); w.close(); @@ -145,8 +144,8 @@ public void testBasicFloats() throws Exception { DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(2, s.count(FloatPoint.newRangeQuery("point", -8.0f, false, 1.0f, false))); - assertEquals(3, s.count(FloatPoint.newRangeQuery("point", -7.0f, true, 3.0f, true))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", -8.0f, 1.0f))); + assertEquals(3, s.count(FloatPoint.newRangeQuery("point", -7.0f, 3.0f))); assertEquals(1, s.count(FloatPoint.newExactQuery("point", -7.0f))); assertEquals(0, s.count(FloatPoint.newExactQuery("point", -6.0f))); w.close(); @@ -172,8 +171,8 @@ public void testBasicLongs() throws Exception { DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(2, s.count(LongPoint.newRangeQuery("point", -8L, false, 1L, false))); - assertEquals(3, s.count(LongPoint.newRangeQuery("point", -7L, true, 3L, true))); + assertEquals(2, s.count(LongPoint.newRangeQuery("point", -8L, 1L))); + assertEquals(3, s.count(LongPoint.newRangeQuery("point", -7L, 3L))); assertEquals(1, s.count(LongPoint.newExactQuery("point", -7L))); assertEquals(0, s.count(LongPoint.newExactQuery("point", -6L))); w.close(); @@ -199,18 +198,140 @@ public void testBasicDoubles() throws Exception { DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(2, s.count(DoublePoint.newRangeQuery("point", -8.0, false, 1.0, false))); - assertEquals(3, s.count(DoublePoint.newRangeQuery("point", -7.0, true, 3.0, true))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", -8.0, 1.0))); + assertEquals(3, s.count(DoublePoint.newRangeQuery("point", -7.0, 3.0))); assertEquals(1, s.count(DoublePoint.newExactQuery("point", -7.0))); assertEquals(0, s.count(DoublePoint.newExactQuery("point", -6.0))); w.close(); r.close(); dir.close(); } + + public void testCrazyDoubles() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + doc.add(new DoublePoint("point", Double.NEGATIVE_INFINITY)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", -0.0D)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", +0.0D)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", Double.MIN_VALUE)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", Double.MAX_VALUE)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", Double.POSITIVE_INFINITY)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new DoublePoint("point", Double.NaN)); + w.addDocument(doc); + + DirectoryReader r = DirectoryReader.open(w); + IndexSearcher s = new IndexSearcher(r); + + // exact queries + assertEquals(1, s.count(DoublePoint.newExactQuery("point", Double.NEGATIVE_INFINITY))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", -0.0D))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", +0.0D))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", Double.MIN_VALUE))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", Double.MAX_VALUE))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", Double.POSITIVE_INFINITY))); + assertEquals(1, s.count(DoublePoint.newExactQuery("point", Double.NaN))); + + // set query + double set[] = new double[] { Double.MAX_VALUE, Double.NaN, +0.0D, Double.NEGATIVE_INFINITY, Double.MIN_VALUE, -0.0D, Double.POSITIVE_INFINITY }; + assertEquals(7, s.count(DoublePoint.newSetQuery("point", set))); + + // ranges + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", Double.NEGATIVE_INFINITY, -0.0D))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", -0.0D, 0.0D))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", 0.0D, Double.MIN_VALUE))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", Double.MIN_VALUE, Double.MAX_VALUE))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", Double.MAX_VALUE, Double.POSITIVE_INFINITY))); + assertEquals(2, s.count(DoublePoint.newRangeQuery("point", Double.POSITIVE_INFINITY, Double.NaN))); + + w.close(); + r.close(); + dir.close(); + } + + public void testCrazyFloats() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + + Document doc = new Document(); + doc.add(new FloatPoint("point", Float.NEGATIVE_INFINITY)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", -0.0F)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", +0.0F)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", Float.MIN_VALUE)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", Float.MAX_VALUE)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", Float.POSITIVE_INFINITY)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new FloatPoint("point", Float.NaN)); + w.addDocument(doc); + + DirectoryReader r = DirectoryReader.open(w); + IndexSearcher s = new IndexSearcher(r); + + // exact queries + assertEquals(1, s.count(FloatPoint.newExactQuery("point", Float.NEGATIVE_INFINITY))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", -0.0F))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", +0.0F))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", Float.MIN_VALUE))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", Float.MAX_VALUE))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", Float.POSITIVE_INFINITY))); + assertEquals(1, s.count(FloatPoint.newExactQuery("point", Float.NaN))); + + // set query + float set[] = new float[] { Float.MAX_VALUE, Float.NaN, +0.0F, Float.NEGATIVE_INFINITY, Float.MIN_VALUE, -0.0F, Float.POSITIVE_INFINITY }; + assertEquals(7, s.count(FloatPoint.newSetQuery("point", set))); + + // ranges + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", Float.NEGATIVE_INFINITY, -0.0F))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", -0.0F, 0.0F))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", 0.0F, Float.MIN_VALUE))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", Float.MIN_VALUE, Float.MAX_VALUE))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", Float.MAX_VALUE, Float.POSITIVE_INFINITY))); + assertEquals(2, s.count(FloatPoint.newRangeQuery("point", Float.POSITIVE_INFINITY, Float.NaN))); + + w.close(); + r.close(); + dir.close(); + } public void testAllEqual() throws Exception { int numValues = atLeast(10000); - long value = randomValue(false); + long value = randomValue(); long[] values = new long[numValues]; if (VERBOSE) { @@ -256,7 +377,7 @@ private void doTestRandomLongs(int count) throws Exception { // Identical to old value values[ord] = values[random().nextInt(ord)]; } else { - values[ord] = randomValue(false); + values[ord] = randomValue(); } ids[ord] = id; @@ -393,21 +514,19 @@ private void _run() throws Exception { NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); for (int iter=0;iter= lower && values[id] <= upper; if (hits.get(docID) != expected) { // We do exact quantized comparison so the bbox query should never disagree: fail(Thread.currentThread().getName() + ": iter=" + iter + " id=" + id + " docID=" + docID + " value=" + values[id] + " (range: " + lower + " TO " + upper + ") expected " + expected + " but got: " + hits.get(docID) + " deleted?=" + deleted.get(id) + " query=" + query); @@ -662,32 +771,20 @@ private void _run() throws Exception { for (int iter=0;iter 0) { + if (StringHelper.compare(bytesPerDim, lower[dim], 0, upper[dim], 0) > 0) { byte[] x = lower[dim]; lower[dim] = upper[dim]; upper[dim] = x; } - - includeLower[dim] = random().nextBoolean(); - includeUpper[dim] = random().nextBoolean(); } if (VERBOSE) { @@ -695,13 +792,12 @@ private void _run() throws Exception { for(int dim=0;dim= lower) && (upper == null || value <= upper); - } - static String bytesToString(byte[] bytes) { if (bytes == null) { return "null"; @@ -792,28 +871,16 @@ static String bytesToString(byte[] bytes) { return new BytesRef(bytes).toString(); } - private static boolean matches(int bytesPerDim, byte[][] lower, boolean[] includeLower, byte[][] upper, boolean[] includeUpper, byte[][] value) { + private static boolean matches(int bytesPerDim, byte[][] lower, byte[][] upper, byte[][] value) { int numDims = lower.length; for(int dim=0;dim 0 || (cmp == 0 && includeUpper[dim] == false)) { + if (StringHelper.compare(bytesPerDim, value[dim], 0, upper[dim], 0) > 0) { // Value is above the upper bound, on this dim return false; } @@ -822,13 +889,9 @@ private static boolean matches(int bytesPerDim, byte[][] lower, boolean[] includ return true; } - private static Long randomValue(boolean allowNull) { + private static long randomValue() { if (valueRange == 0) { - if (allowNull && random().nextInt(10) == 1) { - return null; - } else { - return random().nextLong(); - } + return random().nextLong(); } else { return valueMid + TestUtil.nextInt(random(), -valueRange, valueRange); } @@ -851,9 +914,9 @@ public void testMinMaxLong() throws Exception { IndexSearcher s = newSearcher(r, false); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, 0L, true))); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", 0L, true, Long.MAX_VALUE, true))); - assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, true))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, 0L))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", 0L, Long.MAX_VALUE))); + assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); IOUtils.close(r, w, dir); } @@ -889,51 +952,15 @@ public void testBasicSortedSet() throws Exception { IndexSearcher s = newSearcher(r, false); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("aaa"), - true, - toUTF8("bbb"), - true))); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("c", 3), - true, - toUTF8("e", 3), - true))); - assertEquals(2, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("a", 3), - true, - toUTF8("z", 3), - true))); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - null, - true, - toUTF8("abc"), - true))); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("a", 3), - true, - toUTF8("abc"), - true))); - assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("a", 3), - true, - toUTF8("abc"), - false))); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("def"), - true, - null, - false))); - assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", - toUTF8(("def")), - true, - toUTF8("z", 3), - true))); - assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", - toUTF8("def"), - false, - toUTF8("z", 3), - true))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("aaa"), toUTF8("bbb")))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("c", 3), toUTF8("e", 3)))); + assertEquals(2, s.count(BinaryPoint.newRangeQuery("value", toUTF8("a", 3), toUTF8("z", 3)))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("", 3), toUTF8("abc")))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("a", 3), toUTF8("abc")))); + assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("a", 3), toUTF8("abb")))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8("def"), toUTF8("zzz")))); + assertEquals(1, s.count(BinaryPoint.newRangeQuery("value", toUTF8(("def")), toUTF8("z", 3)))); + assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("deg"), toUTF8("z", 3)))); IOUtils.close(r, w, dir); } @@ -954,12 +981,10 @@ public void testLongMinMaxNumeric() throws Exception { IndexSearcher s = newSearcher(r, false); - assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, true))); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, false))); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, false, Long.MAX_VALUE, true))); - assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, false, Long.MAX_VALUE, false))); - - assertEquals(2, s.count(BinaryPoint.newRangeQuery("value", (byte[]) null, true, null, true))); + assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE-1))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE+1, Long.MAX_VALUE))); + assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE+1, Long.MAX_VALUE-1))); IOUtils.close(r, w, dir); } @@ -980,12 +1005,10 @@ public void testLongMinMaxSortedSet() throws Exception { IndexSearcher s = newSearcher(r, false); - assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, true))); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, true, Long.MAX_VALUE, false))); - assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, false, Long.MAX_VALUE, true))); - assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, false, Long.MAX_VALUE, false))); - - assertEquals(2, s.count(LongPoint.newRangeQuery("value", (Long) null, true, null, true))); + assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE-1))); + assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE+1, Long.MAX_VALUE))); + assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE+1, Long.MAX_VALUE-1))); IOUtils.close(r, w, dir); } @@ -1004,10 +1027,8 @@ public void testSortedSetNoOrdsMatch() throws Exception { IndexReader r = w.getReader(); - IndexSearcher s = newSearcher(r, false); - assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("m"), true, toUTF8("n"), false))); - - assertEquals(2, s.count(BinaryPoint.newRangeQuery("value", (byte[]) null, true, null, true))); + IndexSearcher s = newSearcher(r,false); + assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("m"), toUTF8("m")))); IOUtils.close(r, w, dir); } @@ -1027,7 +1048,7 @@ public void testNumericNoValuesMatch() throws Exception { IndexReader r = w.getReader(); IndexSearcher s = new IndexSearcher(r); - assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, true, 13L, false))); + assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, 13L))); IOUtils.close(r, w, dir); } @@ -1042,7 +1063,7 @@ public void testNoDocs() throws Exception { IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); - assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, true, 13L, false))); + assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, 13L))); IOUtils.close(r, w, dir); } @@ -1061,41 +1082,16 @@ public void testWrongNumDims() throws Exception { // no wrapping, else the exc might happen in executor thread: IndexSearcher s = new IndexSearcher(r); byte[][] point = new byte[2][]; + point[0] = new byte[8]; + point[1] = new byte[8]; IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - s.count(BinaryPoint.newMultiRangeQuery("value", point, new boolean[] {true, true}, point, new boolean[] {true, true})); + s.count(BinaryPoint.newRangeQuery("value", point, point)); }); assertEquals("field=\"value\" was indexed with numDims=1 but this query has numDims=2", expected.getMessage()); IOUtils.close(r, w, dir); } - /** ensure good exception when boolean[]s for inclusive have wrong length */ - public void testWrongNumBooleans() throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setCodec(getCodec()); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - Document doc = new Document(); - doc.add(new LongPoint("value", 1L, 2L)); - w.addDocument(doc); - - IndexReader r = w.getReader(); - - // no wrapping, else the exc might happen in executor thread: - IndexSearcher s = new IndexSearcher(r); - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - s.count(LongPoint.newMultiRangeQuery("value", new Long[] { 1L, 2L }, new boolean[] {true}, new Long[] { 1L, 2L }, new boolean[] {true, true})); - }); - assertEquals("lowerInclusive has length=1 but expected=2", expected.getMessage()); - - expected = expectThrows(IllegalArgumentException.class, () -> { - s.count(LongPoint.newMultiRangeQuery("value", new Long[] { 1L, 2L }, new boolean[] {true, true}, new Long[] { 1L, 2L }, new boolean[] {true})); - }); - assertEquals("upperInclusive has length=1 but expected=2", expected.getMessage()); - - IOUtils.close(r, w, dir); - } - public void testWrongNumBytes() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); @@ -1112,7 +1108,7 @@ public void testWrongNumBytes() throws Exception { byte[][] point = new byte[1][]; point[0] = new byte[10]; IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - s.count(BinaryPoint.newMultiRangeQuery("value", point, new boolean[] {true}, point, new boolean[] {true})); + s.count(BinaryPoint.newRangeQuery("value", point, point)); }); assertEquals("field=\"value\" was indexed with bytesPerDim=8 but this query has bytesPerDim=10", expected.getMessage()); @@ -1228,31 +1224,25 @@ public void testExactPoints() throws Exception { public void testToString() throws Exception { // ints - assertEquals("field:[1 TO 2}", IntPoint.newRangeQuery("field", 1, true, 2, false).toString()); - assertEquals("field:{-2 TO 1]", IntPoint.newRangeQuery("field", -2, false, 1, true).toString()); - assertEquals("field:[* TO 2}", IntPoint.newRangeQuery("field", null, true, 2, false).toString()); + assertEquals("field:[1 TO 2]", IntPoint.newRangeQuery("field", 1, 2).toString()); + assertEquals("field:[-2 TO 1]", IntPoint.newRangeQuery("field", -2, 1).toString()); // longs - assertEquals("field:[1099511627776 TO 2199023255552}", LongPoint.newRangeQuery("field", 1L<<40, true, 1L<<41, false).toString()); - assertEquals("field:{-5 TO 6]", LongPoint.newRangeQuery("field", -5L, false, 6L, true).toString()); - assertEquals("field:[* TO 2}", LongPoint.newRangeQuery("field", null, true, 2L, false).toString()); + assertEquals("field:[1099511627776 TO 2199023255552]", LongPoint.newRangeQuery("field", 1L<<40, 1L<<41).toString()); + assertEquals("field:[-5 TO 6]", LongPoint.newRangeQuery("field", -5L, 6L).toString()); // floats - assertEquals("field:[1.3 TO 2.5}", FloatPoint.newRangeQuery("field", 1.3F, true, 2.5F, false).toString()); - assertEquals("field:{-2.9 TO 1.0]", FloatPoint.newRangeQuery("field", -2.9F, false, 1.0F, true).toString()); - assertEquals("field:{-2.9 TO *]", FloatPoint.newRangeQuery("field", -2.9F, false, null, true).toString()); + assertEquals("field:[1.3 TO 2.5]", FloatPoint.newRangeQuery("field", 1.3F, 2.5F).toString()); + assertEquals("field:[-2.9 TO 1.0]", FloatPoint.newRangeQuery("field", -2.9F, 1.0F).toString()); // doubles - assertEquals("field:[1.3 TO 2.5}", DoublePoint.newRangeQuery("field", 1.3, true, 2.5, false).toString()); - assertEquals("field:{-2.9 TO 1.0]", DoublePoint.newRangeQuery("field", -2.9, false, 1.0, true).toString()); - assertEquals("field:{-2.9 TO *]", DoublePoint.newRangeQuery("field", -2.9, false, null, true).toString()); + assertEquals("field:[1.3 TO 2.5]", DoublePoint.newRangeQuery("field", 1.3, 2.5).toString()); + assertEquals("field:[-2.9 TO 1.0]", DoublePoint.newRangeQuery("field", -2.9, 1.0).toString()); // n-dimensional double - assertEquals("field:[1.3 TO 2.5},{-2.9 TO 1.0]", DoublePoint.newMultiRangeQuery("field", - new Double[] { 1.3, -2.9 }, - new boolean[] { true, false }, - new Double[] { 2.5, 1.0 }, - new boolean[] { false, true }).toString()); + assertEquals("field:[1.3 TO 2.5],[-2.9 TO 1.0]", DoublePoint.newRangeQuery("field", + new double[] { 1.3, -2.9 }, + new double[] { 2.5, 1.0 }).toString()); } @@ -1430,7 +1420,7 @@ public int compare(byte[] a, byte[] b) { return new PointInSetQuery(field, numDims, Integer.BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { @@ -1818,7 +1808,7 @@ public void testInvalidPointInSetQuery() throws Exception { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { new PointInSetQuery("foo", 3, 4, - new BytesRefIterator() { + new PointInSetQuery.Stream() { @Override public BytesRef next() { return new BytesRef(new byte[3]); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index 2edebb3d7248..9c7ada8c68f7 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -26,7 +26,7 @@ public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase { public void testCostlyFilter() { assertTrue(UsageTrackingQueryCachingPolicy.isCostly(new PrefixQuery(new Term("field", "prefix")))); - assertTrue(UsageTrackingQueryCachingPolicy.isCostly(IntPoint.newRangeQuery("intField", 1, true, 1000, true))); + assertTrue(UsageTrackingQueryCachingPolicy.isCostly(IntPoint.newRangeQuery("intField", 1, 1000))); assertFalse(UsageTrackingQueryCachingPolicy.isCostly(new TermQuery(new Term("field", "value")))); } diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java index d9e944179cee..fff3bff9f6d1 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java @@ -179,7 +179,7 @@ public static Query getBoundingBoxQuery(double originLat, double originLng, doub BooleanQuery.Builder f = new BooleanQuery.Builder(); // Add latitude range filter: - f.add(DoublePoint.newRangeQuery("latitude", Math.toDegrees(minLat), true, Math.toDegrees(maxLat), true), + f.add(DoublePoint.newRangeQuery("latitude", Math.toDegrees(minLat), Math.toDegrees(maxLat)), BooleanClause.Occur.FILTER); // Add longitude range filter: @@ -187,13 +187,13 @@ public static Query getBoundingBoxQuery(double originLat, double originLng, doub // The bounding box crosses the international date // line: BooleanQuery.Builder lonF = new BooleanQuery.Builder(); - lonF.add(DoublePoint.newRangeQuery("longitude", Math.toDegrees(minLng), true, null, true), + lonF.add(DoublePoint.newRangeQuery("longitude", Math.toDegrees(minLng), Double.POSITIVE_INFINITY), BooleanClause.Occur.SHOULD); - lonF.add(DoublePoint.newRangeQuery("longitude", null, true, Math.toDegrees(maxLng), true), + lonF.add(DoublePoint.newRangeQuery("longitude", Double.NEGATIVE_INFINITY, Math.toDegrees(maxLng)), BooleanClause.Occur.SHOULD); f.add(lonF.build(), BooleanClause.Occur.MUST); } else { - f.add(DoublePoint.newRangeQuery("longitude", Math.toDegrees(minLng), true, Math.toDegrees(maxLng), true), + f.add(DoublePoint.newRangeQuery("longitude", Math.toDegrees(minLng), Math.toDegrees(maxLng)), BooleanClause.Occur.FILTER); } diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java index 96d7c17cf9f0..8fc794909e30 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/RangeFacetsExample.java @@ -105,8 +105,7 @@ public TopDocs drillDown(LongRange range) throws IOException { // documents ("browse only"): DrillDownQuery q = new DrillDownQuery(getConfig()); - q.add("timestamp", LongPoint.newRangeQuery("timestamp", range.min, range.minInclusive, range.max, range.maxInclusive)); - + q.add("timestamp", LongPoint.newRangeQuery("timestamp", range.min, range.max)); return searcher.search(q, 10); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 362dd7b7e97e..c246d7489dd4 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -38,42 +38,29 @@ * * @lucene.experimental */ public final class DoubleRange extends Range { - final double minIncl; - final double maxIncl; - - /** Minimum. */ + /** Minimum (inclusive). */ public final double min; - /** Maximum. */ + /** Maximum (inclusive. */ public final double max; - /** True if the minimum value is inclusive. */ - public final boolean minInclusive; - - /** True if the maximum value is inclusive. */ - public final boolean maxInclusive; - /** Create a DoubleRange. */ public DoubleRange(String label, double minIn, boolean minInclusive, double maxIn, boolean maxInclusive) { super(label); - this.min = minIn; - this.max = maxIn; - this.minInclusive = minInclusive; - this.maxInclusive = maxInclusive; // TODO: if DoubleDocValuesField used // LegacyNumericUtils.doubleToSortableLong format (instead of // Double.doubleToRawLongBits) we could do comparisons // in long space - if (Double.isNaN(min)) { + if (Double.isNaN(minIn)) { throw new IllegalArgumentException("min cannot be NaN"); } if (!minInclusive) { minIn = Math.nextUp(minIn); } - if (Double.isNaN(max)) { + if (Double.isNaN(maxIn)) { throw new IllegalArgumentException("max cannot be NaN"); } if (!maxInclusive) { @@ -85,24 +72,24 @@ public DoubleRange(String label, double minIn, boolean minInclusive, double maxI failNoMatch(); } - this.minIncl = minIn; - this.maxIncl = maxIn; + this.min = minIn; + this.max = maxIn; } /** True if this range accepts the provided value. */ public boolean accept(double value) { - return value >= minIncl && value <= maxIncl; + return value >= min && value <= max; } LongRange toLongRange() { return new LongRange(label, - LegacyNumericUtils.doubleToSortableLong(minIncl), true, - LegacyNumericUtils.doubleToSortableLong(maxIncl), true); + LegacyNumericUtils.doubleToSortableLong(min), true, + LegacyNumericUtils.doubleToSortableLong(max), true); } @Override public String toString() { - return "DoubleRange(" + minIncl + " to " + maxIncl + ")"; + return "DoubleRange(" + min + " to " + max + ")"; } private static class ValueSourceQuery extends Query { diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java index 8892725bb837..a39ea7e8de9c 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java @@ -88,8 +88,8 @@ private void count(ValueSource valueSource, List matchingDocs) thr for(int i=0;i= minIncl && value <= maxIncl; + return value >= min && value <= max; } @Override public String toString() { - return "LongRange(" + minIncl + " to " + maxIncl + ")"; + return "LongRange(" + min + " to " + max + ")"; } private static class ValueSourceQuery extends Query { diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeCounter.java b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeCounter.java index 8c0b123007d1..5c625f088671 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeCounter.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeCounter.java @@ -49,17 +49,17 @@ public LongRangeCounter(LongRange[] ranges) { endsMap.put(Long.MAX_VALUE, 2); for(LongRange range : ranges) { - Integer cur = endsMap.get(range.minIncl); + Integer cur = endsMap.get(range.min); if (cur == null) { - endsMap.put(range.minIncl, 1); + endsMap.put(range.min, 1); } else { - endsMap.put(range.minIncl, cur.intValue() | 1); + endsMap.put(range.min, cur.intValue() | 1); } - cur = endsMap.get(range.maxIncl); + cur = endsMap.get(range.max); if (cur == null) { - endsMap.put(range.maxIncl, 2); + endsMap.put(range.max, 2); } else { - endsMap.put(range.maxIncl, cur.intValue() | 2); + endsMap.put(range.max, cur.intValue() | 2); } } @@ -276,7 +276,7 @@ static void indent(StringBuilder sb, int depth) { /** Recursively assigns range outputs to each node. */ void addOutputs(int index, LongRange range) { - if (start >= range.minIncl && end <= range.maxIncl) { + if (start >= range.min && end <= range.max) { // Our range is fully included in the incoming // range; add to our output list: if (outputs == null) { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index c4233c04710b..e7e5d5720883 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -280,7 +280,7 @@ protected boolean scoreSubDocsAtOnce() { // Third search, drill down on "less than or equal to 10": ddq = new DrillDownQuery(config); - ddq.add("field", LongPoint.newRangeQuery("field", 0L, true, 10L, true)); + ddq.add("field", LongPoint.newRangeQuery("field", 0L, 10L)); dsr = ds.search(null, ddq, 10); assertEquals(11, dsr.hits.totalHits); @@ -460,9 +460,9 @@ public void testRandomLongs() throws Exception { Query fastMatchQuery; if (random().nextBoolean()) { if (random().nextBoolean()) { - fastMatchQuery = LongPoint.newRangeQuery("field", minValue, true, maxValue, true); + fastMatchQuery = LongPoint.newRangeQuery("field", minValue, maxValue); } else { - fastMatchQuery = LongPoint.newRangeQuery("field", minAcceptedValue, true, maxAcceptedValue, true); + fastMatchQuery = LongPoint.newRangeQuery("field", minAcceptedValue, maxAcceptedValue); } } else { fastMatchQuery = null; @@ -484,7 +484,7 @@ public void testRandomLongs() throws Exception { // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (random().nextBoolean()) { - ddq.add("field", LongPoint.newRangeQuery("field", range.min, range.minInclusive, range.max, range.maxInclusive)); + ddq.add("field", LongPoint.newRangeQuery("field", range.min, range.max)); } else { ddq.add("field", range.getQuery(fastMatchQuery, vs)); } @@ -615,9 +615,9 @@ public void testRandomFloats() throws Exception { Query fastMatchQuery; if (random().nextBoolean()) { if (random().nextBoolean()) { - fastMatchQuery = FloatPoint.newRangeQuery("field", minValue, true, maxValue, true); + fastMatchQuery = FloatPoint.newRangeQuery("field", minValue, maxValue); } else { - fastMatchQuery = FloatPoint.newRangeQuery("field", minAcceptedValue, true, maxAcceptedValue, true); + fastMatchQuery = FloatPoint.newRangeQuery("field", minAcceptedValue, maxAcceptedValue); } } else { fastMatchQuery = null; @@ -639,7 +639,7 @@ public void testRandomFloats() throws Exception { // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (random().nextBoolean()) { - ddq.add("field", FloatPoint.newRangeQuery("field", (float) range.min, range.minInclusive, (float) range.max, range.maxInclusive)); + ddq.add("field", FloatPoint.newRangeQuery("field", (float) range.min, (float) range.max)); } else { ddq.add("field", range.getQuery(fastMatchQuery, vs)); } @@ -754,9 +754,9 @@ public void testRandomDoubles() throws Exception { Query fastMatchFilter; if (random().nextBoolean()) { if (random().nextBoolean()) { - fastMatchFilter = DoublePoint.newRangeQuery("field", minValue, true, maxValue, true); + fastMatchFilter = DoublePoint.newRangeQuery("field", minValue, maxValue); } else { - fastMatchFilter = DoublePoint.newRangeQuery("field", minAcceptedValue, true, maxAcceptedValue, true); + fastMatchFilter = DoublePoint.newRangeQuery("field", minAcceptedValue, maxAcceptedValue); } } else { fastMatchFilter = null; @@ -778,7 +778,7 @@ public void testRandomDoubles() throws Exception { // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (random().nextBoolean()) { - ddq.add("field", DoublePoint.newRangeQuery("field", range.min, range.minInclusive, range.max, range.maxInclusive)); + ddq.add("field", DoublePoint.newRangeQuery("field", range.min, range.max)); } else { ddq.add("field", range.getQuery(fastMatchFilter, vs)); } diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java index edc91f6e9d3e..187f4a546f52 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java @@ -583,7 +583,7 @@ public void testExternalReader() throws Exception { public void testDimensionalRangeQuery() throws Exception { // doesn't currently highlight, but make sure it doesn't cause exception either - query = IntPoint.newRangeQuery(NUMERIC_FIELD_NAME, 2, true, 6, true); + query = IntPoint.newRangeQuery(NUMERIC_FIELD_NAME, 2, 6); searcher = newSearcher(reader); hits = searcher.search(query, 100); int maxNumFragmentsRequired = 2; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java index cb9f5c142d3a..9f765ab1865e 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; @@ -24,7 +23,6 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.NumericUtils; /** @@ -36,16 +34,22 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact 1D point. - *
  • {@link #newRangeQuery newRangeQuery()} for matching a 1D range. - *
  • {@link #newMultiRangeQuery newMultiRangeQuery()} for matching points/ranges in n-dimensional space. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, BigInteger)} for matching an exact 1D point. + *
  • {@link #newSetQuery(String, BigInteger...)} for matching a set of 1D values. + *
  • {@link #newRangeQuery(String, BigInteger, BigInteger)} for matching a 1D range. + *
  • {@link #newRangeQuery(String, BigInteger[], BigInteger[])} for matching points/ranges in n-dimensional space. *
*/ public class BigIntegerPoint extends Field { /** The number of bytes per dimension: 128 bits. */ public static final int BYTES = 16; + + /** A constant holding the minimum value a BigIntegerPoint can have, -2127. */ + public static final BigInteger MIN_VALUE = BigInteger.ONE.shiftLeft(BYTES * 8 - 1).negate(); + + /** A constant holding the maximum value a BigIntegerPoint can have, 2127-1. */ + public static final BigInteger MAX_VALUE = BigInteger.ONE.shiftLeft(BYTES * 8 - 1).subtract(BigInteger.ONE); private static FieldType getType(int numDims) { FieldType type = new FieldType(); @@ -128,10 +132,8 @@ public String toString() { private static byte[][] encode(BigInteger value[]) { byte[][] encoded = new byte[value.length][]; for (int i = 0; i < value.length; i++) { - if (value[i] != null) { - encoded[i] = new byte[BYTES]; - encodeDimension(value[i], encoded[i], 0); - } + encoded[i] = new byte[BYTES]; + encodeDimension(value[i], encoded[i], 0); } return encoded; } @@ -154,65 +156,61 @@ public static BigInteger decodeDimension(byte value[], int offset) { * Create a query for matching an exact big integer value. *

* This is for simple one-dimension points, for multidimensional points use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, BigInteger[], BigInteger[])} instead. * * @param field field name. must not be {@code null}. - * @param value exact value - * @throws IllegalArgumentException if {@code field} is null. + * @param value exact value. must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null or {@code value} is null. * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, BigInteger value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** * Create a range query for big integer values. *

* This is for simple one-dimension ranges, for multidimensional ranges use - * {@link #newMultiRangeQuery newMultiRangeQuery()} instead. + * {@link #newRangeQuery(String, BigInteger[], BigInteger[])} instead. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. + * by setting {@code lowerValue = BigIntegerPoint.MIN_VALUE} + * or {@code upperValue = BigIntegerPoint.MAX_VALUE}. *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue.add(BigInteger.ONE)} + * or {@code upperValue.subtract(BigInteger.ONE)} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null. + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, {@code lowerValue} is null, or {@code upperValue} is null. * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, BigInteger lowerValue, boolean lowerInclusive, BigInteger upperValue, boolean upperInclusive) { - return newMultiRangeQuery(field, - new BigInteger[] { lowerValue }, - new boolean[] { lowerInclusive }, - new BigInteger[] { upperValue }, - new boolean[] { upperInclusive }); + public static Query newRangeQuery(String field, BigInteger lowerValue, BigInteger upperValue) { + PointRangeQuery.checkArgs(field, lowerValue, upperValue); + return newRangeQuery(field, new BigInteger[] { lowerValue }, new BigInteger[] { upperValue }); } /** - * Create a multidimensional range query for big integer values. + * Create a range query for n-dimensional big integer values. *

* You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting a {@code lowerValue} element or {@code upperValue} element to {@code null}. + * by setting {@code lowerValue[i] = BigIntegerPoint.MIN_VALUE} + * or {@code upperValue[i] = BigIntegerPoint.MAX_VALUE}. *

- * By setting a dimension's inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. + * Ranges are inclusive. For exclusive ranges, pass {@code lowerValue[i].add(BigInteger.ONE)} + * or {@code upperValue[i].subtract(BigInteger.ONE)} * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} values mean "open" for that dimension. - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} values mean "open" for that dimension. - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + * @param lowerValue lower portion of the range (inclusive). must not be {@code null}. + * @param upperValue upper portion of the range (inclusive). must not be {@code null}. + * @throws IllegalArgumentException if {@code field} is null, if {@code lowerValue} is null, if {@code upperValue} is null, + * or if {@code lowerValue.length != upperValue.length} * @return a query matching documents within this range. */ - public static Query newMultiRangeQuery(String field, BigInteger[] lowerValue, boolean lowerInclusive[], BigInteger[] upperValue, boolean upperInclusive[]) { + public static Query newRangeQuery(String field, BigInteger[] lowerValue, BigInteger[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, BigIntegerPoint.encode(lowerValue), lowerInclusive, BigIntegerPoint.encode(upperValue), upperInclusive) { + return new PointRangeQuery(field, BigIntegerPoint.encode(lowerValue), BigIntegerPoint.encode(upperValue)) { @Override protected String toString(int dimension, byte[] value) { return BigIntegerPoint.decodeDimension(value, 0).toString(); @@ -224,30 +222,29 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, BigInteger... valuesIn) throws IOException { + public static Query newSetQuery(String field, BigInteger... values) { // Don't unexpectedly change the user's incoming values array: - BigInteger[] values = valuesIn.clone(); - - Arrays.sort(values); + BigInteger[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[BYTES]); + final BytesRef encoded = new BytesRef(new byte[BYTES]); return new PointInSetQuery(field, 1, BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - encodeDimension(values[upto], value.bytes, 0); + encodeDimension(sortedValues[upto], encoded.bytes, 0); upto++; - return value; + return encoded; } } }) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java index 1a73dc1b9f50..a0623b367927 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.document; -import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; @@ -25,7 +24,6 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; /** * An indexed 128-bit {@code InetAddress} field. @@ -36,10 +34,10 @@ *

* This field defines static factory methods for creating common queries: *

    - *
  • {@link #newExactQuery newExactQuery()} for matching an exact network address. - *
  • {@link #newPrefixQuery newPrefixQuery()} for matching a network based on CIDR prefix. - *
  • {@link #newRangeQuery newRangeQuery()} for matching arbitrary network address ranges. - *
  • {@link #newSetQuery newSetQuery()} for matching a set of 1D values. + *
  • {@link #newExactQuery(String, InetAddress)} for matching an exact network address. + *
  • {@link #newPrefixQuery(String, InetAddress, int)} for matching a network based on CIDR prefix. + *
  • {@link #newRangeQuery(String, InetAddress, InetAddress)} for matching arbitrary network address ranges. + *
  • {@link #newSetQuery(String, InetAddress...)} for matching a set of 1D values. *
*

* This field supports both IPv4 and IPv6 addresses: IPv4 addresses are converted @@ -149,7 +147,7 @@ public static InetAddress decode(byte value[]) { * @return a query matching documents with this exact value */ public static Query newExactQuery(String field, InetAddress value) { - return newRangeQuery(field, value, true, value, true); + return newRangeQuery(field, value, value); } /** @@ -162,6 +160,9 @@ public static Query newExactQuery(String field, InetAddress value) { * @return a query matching documents with addresses contained within this network */ public static Query newPrefixQuery(String field, InetAddress value, int prefixLength) { + if (value == null) { + throw new IllegalArgumentException("InetAddress cannot be null"); + } if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) { throw new IllegalArgumentException("illegal prefixLength '" + prefixLength + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges"); } @@ -173,7 +174,7 @@ public static Query newPrefixQuery(String field, InetAddress value, int prefixLe upper[i >> 3] |= 1 << (i & 7); } try { - return newRangeQuery(field, InetAddress.getByAddress(lower), true, InetAddress.getByAddress(upper), true); + return newRangeQuery(field, InetAddress.getByAddress(lower), InetAddress.getByAddress(upper)); } catch (UnknownHostException e) { throw new AssertionError(e); // values are coming from InetAddress } @@ -181,31 +182,21 @@ public static Query newPrefixQuery(String field, InetAddress value, int prefixLe /** * Create a range query for network addresses. - *

- * You can have half-open ranges (which are in fact </≤ or >/≥ queries) - * by setting the {@code lowerValue} or {@code upperValue} to {@code null}. - *

- * By setting inclusive ({@code lowerInclusive} or {@code upperInclusive}) to false, it will - * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too. * * @param field field name. must not be {@code null}. - * @param lowerValue lower portion of the range. {@code null} means "open". - * @param lowerInclusive {@code true} if the lower portion of the range is inclusive, {@code false} if it should be excluded. - * @param upperValue upper portion of the range. {@code null} means "open". - * @param upperInclusive {@code true} if the upper portion of the range is inclusive, {@code false} if it should be excluded. - * @throws IllegalArgumentException if {@code field} is null. + * @param lowerValue lower portion of the range (inclusive). must not be null. + * @param upperValue upper portion of the range (inclusive). must not be null. + * @throws IllegalArgumentException if {@code field} is null, {@code lowerValue} is null, + * or {@code upperValue} is null * @return a query matching documents within this range. */ - public static Query newRangeQuery(String field, InetAddress lowerValue, boolean lowerInclusive, InetAddress upperValue, boolean upperInclusive) { + public static Query newRangeQuery(String field, InetAddress lowerValue, InetAddress upperValue) { + PointRangeQuery.checkArgs(field, lowerValue, upperValue); byte[][] lowerBytes = new byte[1][]; - if (lowerValue != null) { - lowerBytes[0] = encode(lowerValue); - } + lowerBytes[0] = encode(lowerValue); byte[][] upperBytes = new byte[1][]; - if (upperValue != null) { - upperBytes[0] = encode(upperValue); - } - return new PointRangeQuery(field, lowerBytes, new boolean[] { lowerInclusive }, upperBytes, new boolean[] { upperInclusive }) { + upperBytes[0] = encode(upperValue); + return new PointRangeQuery(field, lowerBytes, upperBytes) { @Override protected String toString(int dimension, byte[] value) { return decode(value).getHostAddress(); // for ranges, the range itself is already bracketed @@ -217,31 +208,30 @@ protected String toString(int dimension, byte[] value) { * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. * * @param field field name. must not be {@code null}. - * @param valuesIn all values to match + * @param values all values to match */ - public static Query newSetQuery(String field, InetAddress... valuesIn) throws IOException { + public static Query newSetQuery(String field, InetAddress... values) { // Don't unexpectedly change the user's incoming values array: - InetAddress[] values = valuesIn.clone(); - - Arrays.sort(values); + InetAddress[] sortedValues = values.clone(); + Arrays.sort(sortedValues); - final BytesRef value = new BytesRef(new byte[BYTES]); + final BytesRef encoded = new BytesRef(new byte[BYTES]); return new PointInSetQuery(field, 1, BYTES, - new BytesRefIterator() { + new PointInSetQuery.Stream() { int upto; @Override public BytesRef next() { - if (upto == values.length) { + if (upto == sortedValues.length) { return null; } else { - value.bytes = encode(values[upto]); - assert value.bytes.length == value.length; + encoded.bytes = encode(sortedValues[upto]); + assert encoded.bytes.length == encoded.length; upto++; - return value; + return encoded; } } }) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index b0902f533ad3..aeb0a0f3e380 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -233,12 +233,17 @@ public static Query newBoxQuery(String field, double minLatitude, double maxLati // E.g.: maxLon = -179, minLon = 179 byte[][] leftOpen = new byte[2][]; leftOpen[0] = lower[0]; - // leave longitude open (null) + // leave longitude open + leftOpen[1] = new byte[Integer.BYTES]; + NumericUtils.intToBytes(Integer.MIN_VALUE, leftOpen[1], 0); Query left = newBoxInternal(field, leftOpen, upper); q.add(new BooleanClause(left, BooleanClause.Occur.SHOULD)); + byte[][] rightOpen = new byte[2][]; rightOpen[0] = upper[0]; - // leave longitude open (null) + // leave longitude open + rightOpen[1] = new byte[Integer.BYTES]; + NumericUtils.intToBytes(Integer.MAX_VALUE, rightOpen[1], 0); Query right = newBoxInternal(field, lower, rightOpen); q.add(new BooleanClause(right, BooleanClause.Occur.SHOULD)); return new ConstantScoreQuery(q.build()); @@ -248,7 +253,7 @@ public static Query newBoxQuery(String field, double minLatitude, double maxLati } private static Query newBoxInternal(String field, byte[][] min, byte[][] max) { - return new PointRangeQuery(field, min, new boolean[] { true, true }, max, new boolean[] { false, false }) { + return new PointRangeQuery(field, min, max) { @Override protected String toString(int dimension, byte[] value) { if (dimension == 0) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index f6d407de6d7b..500c2a320612 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -42,7 +42,7 @@ public void testBasics() throws Exception { IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", large))); - assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", large.subtract(BigInteger.ONE), false, large.add(BigInteger.ONE), false))); + assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", large.subtract(BigInteger.ONE), large.add(BigInteger.ONE)))); assertEquals(1, searcher.count(BigIntegerPoint.newSetQuery("field", large))); assertEquals(0, searcher.count(BigIntegerPoint.newSetQuery("field", large.subtract(BigInteger.ONE)))); assertEquals(0, searcher.count(BigIntegerPoint.newSetQuery("field"))); @@ -67,7 +67,7 @@ public void testNegative() throws Exception { IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", negative))); - assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", negative.subtract(BigInteger.ONE), false, negative.add(BigInteger.ONE), false))); + assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", negative.subtract(BigInteger.ONE), negative.add(BigInteger.ONE)))); reader.close(); writer.close(); @@ -87,12 +87,10 @@ public void testToString() throws Exception { assertEquals("BigIntegerPoint ", new BigIntegerPoint("field", BigInteger.ONE).toString()); assertEquals("BigIntegerPoint ", new BigIntegerPoint("field", BigInteger.ONE, BigInteger.valueOf(-2)).toString()); assertEquals("field:[1 TO 1]", BigIntegerPoint.newExactQuery("field", BigInteger.ONE).toString()); - assertEquals("field:{1 TO 17]", BigIntegerPoint.newRangeQuery("field", BigInteger.ONE, false, BigInteger.valueOf(17), true).toString()); - assertEquals("field:{1 TO 17],[0 TO 42}", BigIntegerPoint.newMultiRangeQuery("field", + assertEquals("field:[1 TO 17]", BigIntegerPoint.newRangeQuery("field", BigInteger.ONE, BigInteger.valueOf(17)).toString()); + assertEquals("field:[1 TO 17],[0 TO 42]", BigIntegerPoint.newRangeQuery("field", new BigInteger[] {BigInteger.ONE, BigInteger.ZERO}, - new boolean[] {false, true}, - new BigInteger[] {BigInteger.valueOf(17), BigInteger.valueOf(42)}, - new boolean[] {true, false}).toString()); + new BigInteger[] {BigInteger.valueOf(17), BigInteger.valueOf(42)}).toString()); assertEquals("field:{1}", BigIntegerPoint.newSetQuery("field", BigInteger.ONE).toString()); } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index 9854001a2d02..d4ddb3adbfde 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -43,7 +43,7 @@ public void testBasics() throws Exception { IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 24))); - assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), false, InetAddress.getByName("1.2.3.5"), false))); + assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")))); assertEquals(1, searcher.count(InetAddressPoint.newSetQuery("field", InetAddress.getByName("1.2.3.4")))); assertEquals(0, searcher.count(InetAddressPoint.newSetQuery("field", InetAddress.getByName("1.2.3.3")))); assertEquals(0, searcher.count(InetAddressPoint.newSetQuery("field"))); @@ -69,7 +69,7 @@ public void testBasicsV6() throws Exception { IndexSearcher searcher = newSearcher(reader, false); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 64))); - assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("fec0::f66c"), false, InetAddress.getByName("fec0::f66e"), false))); + assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("fec0::f66c"), InetAddress.getByName("fec0::f66e")))); reader.close(); writer.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 1d3bfac30525..0ef948d3f6dd 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -51,7 +51,7 @@ public void testToString() throws Exception { assertEquals("LatLonPoint ",(new LatLonPoint("field", 18.313694, -65.227444)).toString()); // looks crazy due to lossiness - assertEquals("field:[17.99999997485429 TO 18.999999999068677},[-65.9999999217689 TO -64.99999998137355}", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); + assertEquals("field:[17.99999997485429 TO 18.999999999068677],[-65.9999999217689 TO -64.99999998137355]", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); // distance query does not quantize inputs assertEquals("field:18.0,19.0 +/- 25.0 meters", LatLonPoint.newDistanceQuery("field", 18, 19, 25).toString()); diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java index 47cd740d1d3c..f2ec17e96a79 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java @@ -56,7 +56,7 @@ public void testDuelNumericRangeQuery() throws IOException { iw.addDocument(doc); } if (random().nextBoolean()) { - iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, true, 10L, true)); + iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L)); } iw.commit(); final IndexReader reader = iw.getReader(); @@ -64,12 +64,10 @@ public void testDuelNumericRangeQuery() throws IOException { iw.close(); for (int i = 0; i < 100; ++i) { - final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); - final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); - final boolean minInclusive = random().nextBoolean(); - final boolean maxInclusive = random().nextBoolean(); - final Query q1 = LongPoint.newRangeQuery("idx", min, minInclusive, max, maxInclusive); - final Query q2 = DocValuesRangeQuery.newLongRange("dv", min, max, minInclusive, maxInclusive); + final Long min = TestUtil.nextLong(random(), -100, 1000); + final Long max = TestUtil.nextLong(random(), -100, 1000); + final Query q1 = LongPoint.newRangeQuery("idx", min, max); + final Query q2 = DocValuesRangeQuery.newLongRange("dv", min, max, true, true); assertSameMatches(searcher, q1, q2, false); } @@ -184,7 +182,7 @@ public void testApproximation() throws IOException { iw.addDocument(doc); } if (random().nextBoolean()) { - iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, true, 10L, true)); + iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L)); } iw.commit(); final IndexReader reader = iw.getReader(); @@ -192,23 +190,21 @@ public void testApproximation() throws IOException { iw.close(); for (int i = 0; i < 100; ++i) { - final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); - final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); - final boolean minInclusive = random().nextBoolean(); - final boolean maxInclusive = random().nextBoolean(); + final Long min = TestUtil.nextLong(random(), -100, 1000); + final Long max = TestUtil.nextLong(random(), -100, 1000); BooleanQuery.Builder ref = new BooleanQuery.Builder(); - ref.add(LongPoint.newRangeQuery("idx", min, minInclusive, max, maxInclusive), Occur.FILTER); + ref.add(LongPoint.newRangeQuery("idx", min, max), Occur.FILTER); ref.add(new TermQuery(new Term("f", "a")), Occur.MUST); BooleanQuery.Builder bq1 = new BooleanQuery.Builder(); - bq1.add(DocValuesRangeQuery.newLongRange("dv1", min, max, minInclusive, maxInclusive), Occur.FILTER); + bq1.add(DocValuesRangeQuery.newLongRange("dv1", min, max, true, true), Occur.FILTER); bq1.add(new TermQuery(new Term("f", "a")), Occur.MUST); assertSameMatches(searcher, ref.build(), bq1.build(), true); BooleanQuery.Builder bq2 = new BooleanQuery.Builder(); - bq2.add(DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), minInclusive, maxInclusive), Occur.FILTER); + bq2.add(DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), true, true), Occur.FILTER); bq2.add(new TermQuery(new Term("f", "a")), Occur.MUST); assertSameMatches(searcher, ref.build(), bq2.build(), true); diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 8eb1be0f7e33..9f9bba2c2362 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -66,15 +66,15 @@ protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointL if (rect.minLon < rect.maxLon) { return pointLatEnc >= rectLatMinEnc && - pointLatEnc < rectLatMaxEnc && + pointLatEnc <= rectLatMaxEnc && pointLonEnc >= rectLonMinEnc && - pointLonEnc < rectLonMaxEnc; + pointLonEnc <= rectLonMaxEnc; } else { // Rect crosses dateline: return pointLatEnc >= rectLatMinEnc && - pointLatEnc < rectLatMaxEnc && + pointLatEnc <= rectLatMaxEnc && (pointLonEnc >= rectLonMinEnc || - pointLonEnc < rectLonMaxEnc); + pointLonEnc <= rectLonMaxEnc); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 8313616f3d2b..ac636ec97be2 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -519,6 +519,9 @@ public void collect(int doc) { boolean fail = false; + // Change to false to see all wrong hits: + boolean failFast = true; + for(int docID=0;docID Date: Wed, 2 Mar 2016 21:05:32 -0500 Subject: [PATCH 0007/1113] LUCENE-7060: Spatial4j 0.6 upgrade. Package com.spatial4j.core -> org.locationtech.spatial4j (cherry picked from commit 569b6ca) --- lucene/CHANGES.txt | 3 + lucene/benchmark/conf/spatial.alg | 2 +- lucene/benchmark/ivy.xml | 2 +- .../byTask/feeds/SpatialDocMaker.java | 8 +- .../byTask/feeds/SpatialFileQueryMaker.java | 4 +- lucene/ivy-versions.properties | 3 +- lucene/licenses/spatial4j-0.5-tests.jar.sha1 | 1 - lucene/licenses/spatial4j-0.5.jar.sha1 | 1 - lucene/licenses/spatial4j-0.6-tests.jar.sha1 | 1 + lucene/licenses/spatial4j-0.6.jar.sha1 | 1 + lucene/licenses/spatial4j-NOTICE.txt | 136 +++++++++++++++++- lucene/spatial-extras/ivy.xml | 4 +- .../lucene/spatial/SpatialStrategy.java | 12 +- .../bbox/BBoxOverlapRatioValueSource.java | 2 +- .../bbox/BBoxSimilarityValueSource.java | 4 +- .../lucene/spatial/bbox/BBoxStrategy.java | 10 +- .../lucene/spatial/bbox/BBoxValueSource.java | 2 +- .../composite/CompositeSpatialStrategy.java | 4 +- .../composite/IntersectsRPTVerifyQuery.java | 4 +- .../prefix/AbstractPrefixTreeQuery.java | 2 +- .../AbstractVisitingPrefixTreeQuery.java | 4 +- .../prefix/ContainsPrefixTreeQuery.java | 4 +- .../spatial/prefix/HeatmapFacetCounter.java | 10 +- .../prefix/IntersectsPrefixTreeQuery.java | 4 +- .../prefix/NumberRangePrefixTreeStrategy.java | 4 +- .../PointPrefixTreeFieldCacheProvider.java | 2 +- .../prefix/PrefixTreeFacetCounter.java | 6 +- .../spatial/prefix/PrefixTreeStrategy.java | 10 +- .../prefix/RecursivePrefixTreeStrategy.java | 4 +- .../prefix/TermQueryPrefixTreeStrategy.java | 6 +- .../spatial/prefix/WithinPrefixTreeQuery.java | 16 +-- .../lucene/spatial/prefix/tree/Cell.java | 6 +- .../prefix/tree/DateRangePrefixTree.java | 2 +- .../prefix/tree/FilterCellIterator.java | 4 +- .../prefix/tree/GeohashPrefixTree.java | 10 +- .../spatial/prefix/tree/LegacyCell.java | 6 +- .../spatial/prefix/tree/LegacyPrefixTree.java | 8 +- .../prefix/tree/NumberRangePrefixTree.java | 16 +-- .../prefix/tree/PackedQuadPrefixTree.java | 12 +- .../spatial/prefix/tree/QuadPrefixTree.java | 10 +- .../prefix/tree/SpatialPrefixTree.java | 4 +- .../prefix/tree/SpatialPrefixTreeFactory.java | 4 +- .../spatial/prefix/tree/TreeCellIterator.java | 4 +- .../lucene/spatial/query/SpatialArgs.java | 12 +- .../spatial/query/SpatialArgsParser.java | 10 +- .../spatial/query/SpatialOperation.java | 8 +- .../serialized/SerializedDVStrategy.java | 10 +- .../lucene/spatial/spatial4j/Geo3dShape.java | 14 +- .../util/DistanceToShapeValueSource.java | 10 +- .../spatial/util/ShapeAreaValueSource.java | 6 +- .../lucene/spatial/util/ShapeFieldCache.java | 2 +- .../ShapeFieldCacheDistanceValueSource.java | 6 +- .../spatial/util/ShapeFieldCacheProvider.java | 2 +- .../util/ShapePredicateValueSource.java | 2 +- .../spatial/vector/DistanceValueSource.java | 4 +- .../spatial/vector/PointVectorStrategy.java | 16 +-- .../lucene/spatial/DistanceStrategyTest.java | 6 +- .../lucene/spatial/PortedSolr3Test.java | 8 +- .../spatial/QueryEqualsHashCodeTest.java | 4 +- .../lucene/spatial/SpatialArgsTest.java | 4 +- .../apache/lucene/spatial/SpatialExample.java | 8 +- .../lucene/spatial/SpatialTestCase.java | 8 +- .../lucene/spatial/SpatialTestData.java | 4 +- .../lucene/spatial/SpatialTestQuery.java | 2 +- .../lucene/spatial/StrategyTestCase.java | 4 +- .../lucene/spatial/TestTestFramework.java | 4 +- .../lucene/spatial/bbox/TestBBoxStrategy.java | 12 +- .../composite/CompositeStrategyTest.java | 12 +- .../spatial/prefix/DateNRStrategyTest.java | 2 +- .../prefix/HeatmapFacetCounterTest.java | 18 +-- .../lucene/spatial/prefix/JtsPolygonTest.java | 8 +- .../spatial/prefix/NumberRangeFacetsTest.java | 2 +- .../RandomSpatialOpFuzzyPrefixTreeTest.java | 24 ++-- .../RandomSpatialOpStrategyTestCase.java | 2 +- .../TestRecursivePrefixTreeStrategy.java | 8 +- .../TestTermQueryPrefixGridStrategy.java | 4 +- .../prefix/tree/DateRangePrefixTreeTest.java | 4 +- .../prefix/tree/SpatialPrefixTreeTest.java | 8 +- .../spatial/query/SpatialArgsParserTest.java | 4 +- .../serialized/SerializedStrategyTest.java | 2 +- .../spatial/spatial4j/Geo3dRptTest.java | 10 +- .../Geo3dShapeRectRelationTestCase.java | 18 +-- ...Geo3dShapeSphereModelRectRelationTest.java | 2 +- .../spatial4j/RandomizedShapeTestCase.java | 22 +-- .../spatial/spatial4j/geo3d/GeoPointTest.java | 2 +- .../vector/TestPointVectorStrategy.java | 6 +- solr/core/ivy.xml | 2 +- .../component/SpatialHeatmapFacets.java | 4 +- .../solr/schema/AbstractSpatialFieldType.java | 12 +- .../AbstractSpatialPrefixTreeFieldType.java | 2 +- .../org/apache/solr/schema/BBoxField.java | 2 +- .../apache/solr/schema/DateRangeField.java | 2 +- .../org/apache/solr/schema/GeoHashField.java | 8 +- .../org/apache/solr/schema/LatLonType.java | 8 +- .../org/apache/solr/schema/PointType.java | 2 +- .../schema/RptWithGeometrySpatialField.java | 6 +- .../apache/solr/search/ValueSourceParser.java | 2 +- .../distance/GeoDistValueSourceParser.java | 8 +- .../function/distance/GeohashFunction.java | 2 +- .../distance/GeohashHaversineFunction.java | 10 +- .../distance/HaversineConstFunction.java | 4 +- .../function/distance/HaversineFunction.java | 2 +- .../org/apache/solr/util/DistanceUnits.java | 2 +- .../org/apache/solr/util/SpatialUtils.java | 18 +-- .../apache/solr/search/TestSolr4Spatial.java | 8 +- .../distance/DistanceFunctionTest.java | 4 +- .../apache/solr/util/DistanceUnitsTest.java | 2 +- solr/licenses/spatial4j-0.5.jar.sha1 | 1 - solr/licenses/spatial4j-0.6.jar.sha1 | 1 + solr/licenses/spatial4j-NOTICE.txt | 136 +++++++++++++++++- 110 files changed, 595 insertions(+), 339 deletions(-) delete mode 100644 lucene/licenses/spatial4j-0.5-tests.jar.sha1 delete mode 100644 lucene/licenses/spatial4j-0.5.jar.sha1 create mode 100644 lucene/licenses/spatial4j-0.6-tests.jar.sha1 create mode 100644 lucene/licenses/spatial4j-0.6.jar.sha1 delete mode 100644 solr/licenses/spatial4j-0.5.jar.sha1 create mode 100644 solr/licenses/spatial4j-0.6.jar.sha1 diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c0925e8ab640..dcba07f7fd6e 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -110,6 +110,9 @@ API Changes comparator in ArrayUtil by Java 8's Comparator#naturalOrder(). (Mike McCandless, Uwe Schindler, Robert Muir) +* LUCENE-7060: Update Spatial4j to 0.6. The package com.spatial4j.core + is now org.locationtech.spatial4j. (David Smiley) + Optimizations * LUCENE-6891: Use prefix coding when writing points in diff --git a/lucene/benchmark/conf/spatial.alg b/lucene/benchmark/conf/spatial.alg index 93f10efc7743..0ee637f510f9 100644 --- a/lucene/benchmark/conf/spatial.alg +++ b/lucene/benchmark/conf/spatial.alg @@ -23,7 +23,7 @@ ### Spatial Context, Grid, Strategy config doc.maker=org.apache.lucene.benchmark.byTask.feeds.SpatialDocMaker # SpatialContext: see SpatialContextFactory.makeSpatialContext -#spatial.spatialContextFactory=com.spatial4j.core.context.jts.JtsSpatialContextFactory +#spatial.spatialContextFactory=org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory #spatial.geo=true #spatial.distCalculator=haversine #spatial.worldBounds=... diff --git a/lucene/benchmark/ivy.xml b/lucene/benchmark/ivy.xml index ec02b6ab4ff3..23c208cb3ed2 100644 --- a/lucene/benchmark/ivy.xml +++ b/lucene/benchmark/ivy.xml @@ -26,7 +26,7 @@ - + diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java index b7f04d1da3b4..b466e964cc9d 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java @@ -23,10 +23,10 @@ import java.util.Random; import java.util.Set; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java index c99de3db071f..b6b8f50b10ef 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Properties; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; @@ -34,7 +34,7 @@ /** * Reads spatial data from the body field docs from an internally created {@link LineDocSource}. - * It's parsed by {@link com.spatial4j.core.context.SpatialContext#readShapeFromWkt(String)} (String)} and then + * It's parsed by {@link org.locationtech.spatial4j.context.SpatialContext#readShapeFromWkt(String)} (String)} and then * further manipulated via a configurable {@link SpatialDocMaker.ShapeConverter}. When using point * data, it's likely you'll want to configure the shape converter so that the query shapes actually * cover a region. The queries are all created and cached in advance. This query maker works in diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 2d630a2a6a64..d5ef256337d7 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -41,7 +41,6 @@ com.google.inject.guice.version = 3.0 /com.googlecode.mp4parser/isoparser = 1.0.2 /com.ibm.icu/icu4j = 56.1 /com.pff/java-libpst = 0.8.1 -/com.spatial4j/spatial4j = 0.5 com.sun.jersey.version = 1.9 /com.sun.jersey.contribs/jersey-guice = ${com.sun.jersey.version} @@ -268,6 +267,8 @@ org.kitesdk.kite-morphlines.version = 1.1.0 /org.kitesdk/kite-morphlines-tika-decompress = ${org.kitesdk.kite-morphlines.version} /org.kitesdk/kite-morphlines-twitter = ${org.kitesdk.kite-morphlines.version} +/org.locationtech.spatial4j/spatial4j = 0.6 + /org.mockito/mockito-core = 1.9.5 org.mortbay.jetty.version = 6.1.26 diff --git a/lucene/licenses/spatial4j-0.5-tests.jar.sha1 b/lucene/licenses/spatial4j-0.5-tests.jar.sha1 deleted file mode 100644 index 0c514f8835ef..000000000000 --- a/lucene/licenses/spatial4j-0.5-tests.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdcdf20a723516a233b5bcc0ca7d4decaa88b6ed diff --git a/lucene/licenses/spatial4j-0.5.jar.sha1 b/lucene/licenses/spatial4j-0.5.jar.sha1 deleted file mode 100644 index c81a76cbe91b..000000000000 --- a/lucene/licenses/spatial4j-0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e16edaf6b1ba76db7f08c2f3723fce3b358ecc3 diff --git a/lucene/licenses/spatial4j-0.6-tests.jar.sha1 b/lucene/licenses/spatial4j-0.6-tests.jar.sha1 new file mode 100644 index 000000000000..0fd8404f2384 --- /dev/null +++ b/lucene/licenses/spatial4j-0.6-tests.jar.sha1 @@ -0,0 +1 @@ +0624ae8b9e43265822e0d79b481e34917fec1eba diff --git a/lucene/licenses/spatial4j-0.6.jar.sha1 b/lucene/licenses/spatial4j-0.6.jar.sha1 new file mode 100644 index 000000000000..56c02ad3016c --- /dev/null +++ b/lucene/licenses/spatial4j-0.6.jar.sha1 @@ -0,0 +1 @@ +21b15310bddcfd8c72611c180f20cf23279809a3 diff --git a/lucene/licenses/spatial4j-NOTICE.txt b/lucene/licenses/spatial4j-NOTICE.txt index d7d48d185c69..779b8df7e911 100644 --- a/lucene/licenses/spatial4j-NOTICE.txt +++ b/lucene/licenses/spatial4j-NOTICE.txt @@ -1,5 +1,133 @@ -Spatial4j -Copyright 2012-2014 The Apache Software Foundation +# about.md file -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). +## About This Content + +May 22, 2015 + +### License + +The Eclipse Foundation makes available all content in this plug-in ("Content"). Unless otherwise indicated below, the +Content is provided to you under the terms and conditions of the Apache License, Version 2.0. A copy of the Apache +License, Version 2.0 is available at +[http://www.apache.org/licenses/LICENSE-2.0.txt](http://www.apache.org/licenses/LICENSE-2.0.txt) + +If you did not receive this Content directly from the Eclipse Foundation, the Content is being redistributed by another +party ("Redistributor") and different terms and conditions may apply to your use of any object code in the Content. +Check the Redistributor’s license that was provided with the Content. If no such license exists, contact the +Redistributor. Unless otherwise indicated below, the terms and conditions of the Apache License, Version 2.0 still apply +to any source code in the Content and such source code may be obtained at +[http://www.eclipse.org](http://www.eclipse.org). + +# notice.md file + +Note: the below Eclipse user agreement is standard. It says "Unless otherwise indicated, "... before referring to the +EPL. We indicate above that all content is licensed under the ASLv2 license. -- David Smiley + +## Eclipse Foundation Software User Agreement + +April 9, 2014 + +### Usage Of Content + +THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE +PROJECTS (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR +THE TERMS AND CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE +THAT YOUR USE OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE +AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT +AND THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY +NOT USE THE CONTENT. + +### Applicable Licenses + +Unless otherwise indicated, all Content made available by the Eclipse Foundation is provided to you under the terms and +conditions of the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is provided with this Content and is +also available at [http://www.eclipse.org/legal/epl-v10.html](http://www.eclipse.org/legal/epl-v10.html). For purposes +of the EPL, "Program" will mean the Content. + +Content includes, but is not limited to, source code, object code, documentation and other files maintained in the +Eclipse Foundation source code repository ("Repository") in software modules ("Modules") and made available as +downloadable archives ("Downloads"). + +* Content may be structured and packaged into modules to facilitate delivering, extending, and upgrading the Content. + Typical modules may include plug-ins ("Plug-ins"), plug-in fragments ("Fragments"), and features ("Features"). +* Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins". +* A Feature is a bundle of one or more Plug-ins and/or Fragments and associated material. Each Feature may be packaged + as a sub-directory in a directory named "features". Within a Feature, files named "feature.xml" may contain a list + of the names and version numbers of the Plug-ins and/or Fragments associated with that Feature. +* Features may also include other Features ("Included Features"). Within a Feature, files named "feature.xml" may + contain a list of the names and version numbers of Included Features. + +The terms and conditions governing Plug-ins and Fragments should be contained in files named "about.html" ("Abouts"). +The terms and conditions governing Features and Included Features should be contained in files named "license.html" +("Feature Licenses"). Abouts and Feature Licenses may be located in any directory of a Download or Module including, but +not limited to the following locations: + +* The top-level (root) directory +* Plug-in and Fragment directories +* Inside Plug-ins and Fragments packaged as JARs +* Sub-directories of the directory named "src" of certain Plug-ins +* Feature directories + +Note: if a Feature made available by the Eclipse Foundation is installed using the Provisioning Technology (as defined +below), you must agree to a license ("Feature Update License") during the installation process. If the Feature contains +Included Features, the Feature Update License should either provide you with the terms and conditions governing the +Included Features or inform you where you can locate them. Feature Update Licenses may be found in the "license" +property of files named "feature.properties" found within a Feature. Such Abouts, Feature Licenses, and Feature Update +Licenses contain the terms and conditions (or references to such terms and conditions) that govern your use of the +associated Content in that directory. + +THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR +TERMS AND CONDITIONS. SOME OF THESE OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO): + +* Eclipse Distribution License Version 1.0 (available at + [http://www.eclipse.org/licenses/edl-v1.0.html](http://www.eclipse.org/licenses/edl-v10.html)) +* Common Public License Version 1.0 (available at + [http://www.eclipse.org/legal/cpl-v10.html](http://www.eclipse.org/legal/cpl-v10.html)) +* Apache Software License 1.1 (available at + [http://www.apache.org/licenses/LICENSE](http://www.apache.org/licenses/LICENSE)) +* Apache Software License 2.0 (available at + [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)) +* Mozilla Public License Version 1.1 (available at + [http://www.mozilla.org/MPL/MPL-1.1.html](http://www.mozilla.org/MPL/MPL-1.1.html)) + +IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature +License, or Feature Update License is provided, please contact the Eclipse Foundation to determine what terms and +conditions govern that particular Content. + +### Use of Provisioning Technology + +The Eclipse Foundation makes available provisioning software, examples of which include, but are not limited to, p2 and +the Eclipse Update Manager ("Provisioning Technology") for the purpose of allowing users to install software, +documentation, information and/or other materials (collectively "Installable Software"). This capability is provided +with the intent of allowing such users to install, extend and update Eclipse-based products. Information about packaging +Installable Software is available at +[http://eclipse.org/equinox/p2/repository_packaging.html](http://eclipse.org/equinox/p2/repository_packaging.html) +("Specification"). + +You may use Provisioning Technology to allow other parties to install Installable Software. You shall be responsible for +enabling the applicable license agreements relating to the Installable Software to be presented to, and accepted by, the +users of the Provisioning Technology in accordance with the Specification. By using Provisioning Technology in such a +manner and making it available in accordance with the Specification, you further acknowledge your agreement to, and the +acquisition of all necessary rights to permit the following: + +1. A series of actions may occur ("Provisioning Process") in which a user may execute the Provisioning Technology on a + machine ("Target Machine") with the intent of installing, extending or updating the functionality of an + Eclipse-based product. +2. During the Provisioning Process, the Provisioning Technology may cause third party Installable Software or a portion + thereof to be accessed and copied to the Target Machine. +3. Pursuant to the Specification, you will provide to the user the terms and conditions that govern the use of the + Installable Software ("Installable Software Agreement") and such Installable Software Agreement shall be accessed + from the Target Machine in accordance with the Specification. Such Installable Software Agreement must inform the + user of the terms and conditions that govern the Installable Software and must solicit acceptance by the end user in + the manner prescribed in such Installable Software Agreement. Upon such indication of agreement by the user, the + provisioning Technology will complete installation of the Installable Software. + +### Cryptography + +Content may contain encryption software. The country in which you are currently may have restrictions on the import, +possession, and use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, possession, or use, and re-export of +encryption software, to see if this is permitted. + +Java and all Java-based trademarks are trademarks of Oracle Corporation in the United States, other countries, +or both. \ No newline at end of file diff --git a/lucene/spatial-extras/ivy.xml b/lucene/spatial-extras/ivy.xml index 4fef30ec538a..448776f0b3aa 100644 --- a/lucene/spatial-extras/ivy.xml +++ b/lucene/spatial-extras/ivy.xml @@ -23,9 +23,9 @@ - + - + diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java index f433c111e0c3..d980ba9b302c 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java @@ -16,10 +16,10 @@ */ package org.apache.lucene.spatial; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Field; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction; @@ -100,7 +100,7 @@ public String getFieldName() { public abstract Field[] createIndexableFields(Shape shape); /** - * See {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point, double)} called with + * See {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point, double)} called with * a multiplier of 1.0 (i.e. units of degrees). */ public ValueSource makeDistanceValueSource(Point queryPoint) { @@ -127,7 +127,7 @@ public ValueSource makeDistanceValueSource(Point queryPoint) { /** * Returns a ValueSource with values ranging from 1 to 0, depending inversely - * on the distance from {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point,double)}. + * on the distance from {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point,double)}. * The formula is {@code c/(d + c)} where 'd' is the distance and 'c' is * one tenth the distance to the farthest edge from the center. Thus the * scores will be 1 for indexed points at the center of the query shape and as diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java index 9d0afe13dec5..101f373c3507 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java @@ -21,7 +21,7 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.Explanation; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.shape.Rectangle; /** * The algorithm is implemented as envelope on envelope (rect on rect) overlays rather than diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java index 15cd646173d6..1d8b4b0c6f96 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java @@ -27,12 +27,12 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.shape.Rectangle; /** * A base class for calculating a spatial relevance rank per document from a provided * {@link ValueSource} in which {@link FunctionValues#objectVal(int)} returns a {@link - * com.spatial4j.core.shape.Rectangle}. + * org.locationtech.spatial4j.shape.Rectangle}. *

* Implementers: remember to implement equals and hashCode if you have * fields! diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java index 9565cbf21e40..5cc6788a63b5 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java @@ -16,10 +16,10 @@ */ package org.apache.lucene.spatial.bbox; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -64,7 +64,7 @@ * and a boolean to mark a dateline cross. Depending on the particular {@link * SpatialOperation}s, there are a variety of {@link org.apache.lucene.search.LegacyNumericRangeQuery}s to be * done. - * The {@link #makeOverlapRatioValueSource(com.spatial4j.core.shape.Rectangle, double)} + * The {@link #makeOverlapRatioValueSource(org.locationtech.spatial4j.shape.Rectangle, double)} * works by calculating the query bbox overlap percentage against the indexed * shape overlap percentage. The indexed shape's coordinates are retrieved from * {@link org.apache.lucene.index.LeafReader#getNumericDocValues}. diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java index 5d954074d325..2bfbfd933a32 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java @@ -16,7 +16,7 @@ */ package org.apache.lucene.spatial.bbox; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java index 7dc2dfaa05a6..de5bb612d415 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java @@ -20,8 +20,8 @@ import java.util.Collections; import java.util.List; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Field; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.Query; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java index a963b6eac6b1..f60bfeeaa131 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.util.Map; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java index 127e68915884..bcf486777ed4 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractPrefixTreeQuery.java @@ -18,7 +18,7 @@ import java.io.IOException; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeQuery.java index 2237ca9dbf0e..8ccee99c40c1 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeQuery.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.util.Iterator; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSet; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeQuery.java index 00463784ebc0..b0864f67cd35 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeQuery.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.util.Arrays; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/HeatmapFacetCounter.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/HeatmapFacetCounter.java index c6700cdef596..adee2be9cecc 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/HeatmapFacetCounter.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/HeatmapFacetCounter.java @@ -20,11 +20,11 @@ import java.util.HashMap; import java.util.Map; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.spatial.prefix.tree.Cell; import org.apache.lucene.spatial.prefix.tree.CellIterator; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeQuery.java index ccb0f89ba177..89129a1c3fdc 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeQuery.java @@ -18,8 +18,8 @@ import java.io.IOException; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.spatial.prefix.tree.Cell; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java index 8001c82afc2d..c727c0da0753 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java @@ -23,8 +23,8 @@ import java.util.SortedMap; import java.util.TreeMap; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.spatial.prefix.tree.Cell; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java index 165c41885443..f44ca4474091 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PointPrefixTreeFieldCacheProvider.java @@ -16,7 +16,7 @@ */ package org.apache.lucene.spatial.prefix; -import com.spatial4j.core.shape.Point; +import org.locationtech.spatial4j.shape.Point; import org.apache.lucene.spatial.prefix.tree.Cell; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.apache.lucene.spatial.util.ShapeFieldCacheProvider; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeFacetCounter.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeFacetCounter.java index 173c30eb8530..b3b82db35c68 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeFacetCounter.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeFacetCounter.java @@ -18,7 +18,7 @@ import java.io.IOException; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; @@ -37,9 +37,9 @@ * of double-counting the document in the facet results. Since each shape is independently turned into grid cells at * a resolution chosen by the shape's size, it's possible they will be indexed at different resolutions. This means * the document could be present in BOTH the postings for a cell in both its prefix and leaf variants. To avoid this, - * use a single valued field with a {@link com.spatial4j.core.shape.ShapeCollection} (or WKT equivalent). Or + * use a single valued field with a {@link org.locationtech.spatial4j.shape.ShapeCollection} (or WKT equivalent). Or * calculate a suitable level/distErr to index both and call - * {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy#createIndexableFields(com.spatial4j.core.shape.Shape, int)} + * {@link org.apache.lucene.spatial.prefix.PrefixTreeStrategy#createIndexableFields(org.locationtech.spatial4j.shape.Shape, int)} * with the same value for all shapes for a given document/field. * * @lucene.experimental diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java index 608879be4ee7..e9f43fd43bd8 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java @@ -21,8 +21,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -47,7 +47,7 @@ *

  • Can index any shape; however only {@link RecursivePrefixTreeStrategy} * can effectively search non-point shapes.
    • *
  • Can index a variable number of shapes per field value. This strategy - * can do it via multiple calls to {@link #createIndexableFields(com.spatial4j.core.shape.Shape)} + * can do it via multiple calls to {@link #createIndexableFields(org.locationtech.spatial4j.shape.Shape)} * for a document or by giving it some sort of Shape aggregate (e.g. JTS * WKT MultiPoint). The shape's boundary is approximated to a grid precision. *
    • @@ -56,7 +56,7 @@ *
  • Only {@link org.apache.lucene.spatial.query.SpatialOperation#Intersects} * is supported. If only points are indexed then this is effectively equivalent * to IsWithin.
    • - *
  • The strategy supports {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point,double)} + *
  • The strategy supports {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point,double)} * even for multi-valued data, so long as the indexed data is all points; the * behavior is undefined otherwise. However, it will likely be removed in * the future in lieu of using another strategy with a more scalable @@ -93,7 +93,7 @@ public SpatialPrefixTree getGrid() { } /** - * A memory hint used by {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)} + * A memory hint used by {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point)} * for how big the initial size of each Document's array should be. The * default is 2. Set this to slightly more than the default expected number * of points per document. diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java index 68b04499d9cf..d3d16263b9c8 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java @@ -20,8 +20,8 @@ import java.util.Iterator; import java.util.List; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.tree.Cell; import org.apache.lucene.spatial.prefix.tree.CellIterator; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java index a74786bc7e5b..0273466aa2b6 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java @@ -19,8 +19,8 @@ import java.util.ArrayList; import java.util.List; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.tree.Cell; @@ -35,7 +35,7 @@ /** * A basic implementation of {@link PrefixTreeStrategy} using a large * {@link TermsQuery} of all the cells from - * {@link SpatialPrefixTree#getTreeCellIterator(com.spatial4j.core.shape.Shape, int)}. + * {@link SpatialPrefixTree#getTreeCellIterator(org.locationtech.spatial4j.shape.Shape, int)}. * It only supports the search of indexed Point shapes. *

    * The precision of query shapes (distErrPct) is an important factor in using diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeQuery.java index cf0d11bd0549..f595f0e4639b 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeQuery.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeQuery.java @@ -18,13 +18,13 @@ import java.io.IOException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Circle; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.spatial.prefix.tree.Cell; @@ -58,7 +58,7 @@ public class WithinPrefixTreeQuery extends AbstractVisitingPrefixTreeQuery { private final Shape bufferedQueryShape;//if null then the whole world /** - * See {@link AbstractVisitingPrefixTreeQuery#AbstractVisitingPrefixTreeQuery(com.spatial4j.core.shape.Shape, String, org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree, int, int)}. + * See {@link AbstractVisitingPrefixTreeQuery#AbstractVisitingPrefixTreeQuery(org.locationtech.spatial4j.shape.Shape, String, org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree, int, int)}. * {@code queryBuffer} is the (minimum) distance beyond the query shape edge * where non-matching documents are looked for so they can be excluded. If * -1 is used then the whole world is examined (a good default for correctness). diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/Cell.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/Cell.java index fe3846d1605f..f4bc45856e71 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/Cell.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/Cell.java @@ -16,14 +16,14 @@ */ package org.apache.lucene.spatial.prefix.tree; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.util.BytesRef; /** * Represents a grid cell. Cell instances are generally very transient and may be re-used * internally. To get an instance, you could start with {@link SpatialPrefixTree#getWorldCell()}. - * And from there you could either traverse down the tree with {@link #getNextLevelCells(com.spatial4j.core.shape.Shape)}, + * And from there you could either traverse down the tree with {@link #getNextLevelCells(org.locationtech.spatial4j.shape.Shape)}, * or you could read an indexed term via {@link SpatialPrefixTree#readCell(org.apache.lucene.util.BytesRef,Cell)}. * When a cell is read from a term, it is comprised of just the base bytes plus optionally a leaf flag. * diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java index 13281f3134fc..afdde71c0a7c 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTree.java @@ -24,7 +24,7 @@ import java.util.Locale; import java.util.TimeZone; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; /** * A PrefixTree for date ranges in which the levels of the tree occur at natural periods of time (e.g. years, diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/FilterCellIterator.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/FilterCellIterator.java index e4f50e05d67d..ef170071876b 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/FilterCellIterator.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/FilterCellIterator.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial.prefix.tree; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import java.util.Iterator; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java index fa4e98745ec7..237d26a5f9fa 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java @@ -20,11 +20,11 @@ import java.util.Collection; import java.util.List; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.io.GeohashUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.io.GeohashUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.util.BytesRef; /** diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyCell.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyCell.java index 27c56a763954..d978d3c84811 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyCell.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyCell.java @@ -18,9 +18,9 @@ import java.util.Collection; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyPrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyPrefixTree.java index 672c2fe04c73..1a3afcc8740e 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyPrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/LegacyPrefixTree.java @@ -18,10 +18,10 @@ import java.util.Arrays; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.util.BytesRef; /** The base for the original two SPT's: Geohash and Quad. Don't subclass this for new SPTs. diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java index 40e80bcc7d72..72b689bc0187 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java @@ -18,13 +18,13 @@ import java.text.ParseException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; @@ -47,7 +47,7 @@ * * * Unlike "normal" spatial components in this module, this special-purpose one only works with {@link Shape}s - * created by the methods on this class, not from any {@link com.spatial4j.core.context.SpatialContext}. + * created by the methods on this class, not from any {@link org.locationtech.spatial4j.context.SpatialContext}. * * @see org.apache.lucene.spatial.prefix.NumberRangePrefixTreeStrategy * @see LUCENE-5648 diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/PackedQuadPrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/PackedQuadPrefixTree.java index 6fe2bffea6f8..b86a6d116ff2 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/PackedQuadPrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/PackedQuadPrefixTree.java @@ -21,12 +21,12 @@ import java.util.List; import java.util.NoSuchElementException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.util.BytesRef; /** diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java index 48dac8714233..3242e7e623f4 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java @@ -24,11 +24,11 @@ import java.util.List; import java.util.Locale; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.util.BytesRef; /** diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java index 8ead954d66ca..ae2fe83d20e7 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial.prefix.tree; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.util.BytesRef; /** diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java index b74dc93bded6..67480d52ff8f 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeFactory.java @@ -18,8 +18,8 @@ import java.util.Map; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; /** * Abstract Factory for creating {@link SpatialPrefixTree} instances with useful diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/TreeCellIterator.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/TreeCellIterator.java index 3ec56aca3b80..39c8068d8f9f 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/TreeCellIterator.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/tree/TreeCellIterator.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial.prefix.tree; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; /** * Navigates a {@link org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree} from a given cell (typically the world diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgs.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgs.java index 0503072cd282..37a5503e16af 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgs.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgs.java @@ -16,10 +16,10 @@ */ package org.apache.lucene.spatial.query; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; /** * Principally holds the query {@link Shape} and the {@link SpatialOperation}. @@ -120,8 +120,8 @@ public void setShape(Shape shape) { * inflates the size of the shape but should not shrink it. * * @return 0 to 0.5 - * @see #calcDistanceFromErrPct(com.spatial4j.core.shape.Shape, double, - * com.spatial4j.core.context.SpatialContext) + * @see #calcDistanceFromErrPct(org.locationtech.spatial4j.shape.Shape, double, + * org.locationtech.spatial4j.context.SpatialContext) */ public Double getDistErrPct() { return distErrPct; diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java index 81612fff5858..79ad7a94803e 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialArgsParser.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.spatial.query; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.exception.InvalidShapeException; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.exception.InvalidShapeException; +import org.locationtech.spatial4j.shape.Shape; import java.text.ParseException; import java.util.HashMap; @@ -29,8 +29,8 @@ /** * Parses a string that usually looks like "OPERATION(SHAPE)" into a {@link SpatialArgs} * object. The set of operations supported are defined in {@link SpatialOperation}, such - * as "Intersects" being a common one. The shape portion is defined by WKT {@link com.spatial4j.core.io.WktShapeParser}, - * but it can be overridden/customized via {@link #parseShape(String, com.spatial4j.core.context.SpatialContext)}. + * as "Intersects" being a common one. The shape portion is defined by WKT {@link org.locationtech.spatial4j.io.WktShapeParser}, + * but it can be overridden/customized via {@link #parseShape(String, org.locationtech.spatial4j.context.SpatialContext)}. * There are some optional name-value pair parameters that follow the closing parenthesis. Example: * 

      *   Intersects(ENVELOPE(-10,-8,22,20)) distErrPct=0.025
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialOperation.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialOperation.java
index 7d750ac6ff96..1eeb4bc08d35 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialOperation.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/query/SpatialOperation.java
@@ -16,9 +16,9 @@
  */
 package org.apache.lucene.spatial.query;
 
-import com.spatial4j.core.shape.Rectangle;
-import com.spatial4j.core.shape.Shape;
-import com.spatial4j.core.shape.SpatialRelation;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.SpatialRelation;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -30,7 +30,7 @@
 /**
  * A predicate that compares a stored geometry to a supplied geometry. It's enum-like. For more
  * explanation of each predicate, consider looking at the source implementation
- * of {@link #evaluate(com.spatial4j.core.shape.Shape, com.spatial4j.core.shape.Shape)}. It's important
+ * of {@link #evaluate(org.locationtech.spatial4j.shape.Shape, org.locationtech.spatial4j.shape.Shape)}. It's important
  * to be aware that Lucene-spatial makes no distinction of shape boundaries, unlike many standardized
  * definitions. Nor does it make dimensional distinctions (e.g. line vs polygon).
  * You can lookup a predicate by "Covers" or "Contains", for example, and you will get the
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
index a6c575bc3f58..cf2c329205ef 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
@@ -24,10 +24,10 @@
 import java.io.IOException;
 import java.util.Map;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.io.BinaryCodec;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.io.BinaryCodec;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.BinaryDocValues;
@@ -55,7 +55,7 @@
  * SpatialStrategy that is approximated (like {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy})
  * to add precision or eventually make more specific / advanced calculations on the per-document
  * geometry.
- * The serialization uses Spatial4j's {@link com.spatial4j.core.io.BinaryCodec}.
+ * The serialization uses Spatial4j's {@link org.locationtech.spatial4j.io.BinaryCodec}.
  *
  * @lucene.experimental
  */
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java
index 7a3078a0bef1..518fb32b4f17 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java
@@ -16,13 +16,13 @@
  */
 package org.apache.lucene.spatial.spatial4j;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Rectangle;
-import com.spatial4j.core.shape.Shape;
-import com.spatial4j.core.shape.SpatialRelation;
-import com.spatial4j.core.shape.impl.RectangleImpl;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.SpatialRelation;
+import org.locationtech.spatial4j.shape.impl.RectangleImpl;
 import org.apache.lucene.geo3d.LatLonBounds;
 import org.apache.lucene.geo3d.GeoArea;
 import org.apache.lucene.geo3d.GeoAreaFactory;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
index 57cad87f97b9..7be24337d865 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
@@ -29,15 +29,15 @@
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceCalculator;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceCalculator;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
 
 /**
  * The distance from a provided Point to a Point retrieved from a ValueSource via
  * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)}. The distance
- * is calculated via a {@link com.spatial4j.core.distance.DistanceCalculator}.
+ * is calculated via a {@link org.locationtech.spatial4j.distance.DistanceCalculator}.
  *
  * @lucene.experimental
  */
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
index dd391d1fc963..257dc6778d23 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
@@ -22,8 +22,8 @@
 import java.util.List;
 import java.util.Map;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.shape.Shape;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
@@ -36,7 +36,7 @@
  * The area of a Shape retrieved from a ValueSource via
  * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)}.
  *
- * @see Shape#getArea(com.spatial4j.core.context.SpatialContext)
+ * @see Shape#getArea(org.locationtech.spatial4j.context.SpatialContext)
  *
  * @lucene.experimental
  */
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java
index 480369bcd82f..e24cd6832107 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCache.java
@@ -16,7 +16,7 @@
  */
 package org.apache.lucene.spatial.util;
 
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.shape.Shape;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
index e4cb1463672c..1ac84e8785c8 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
@@ -16,9 +16,9 @@
  */
 package org.apache.lucene.spatial.util;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceCalculator;
-import com.spatial4j.core.shape.Point;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceCalculator;
+import org.locationtech.spatial4j.shape.Point;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java
index 04c52f79a67f..bca73ccf4dc1 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheProvider.java
@@ -16,7 +16,7 @@
  */
 package org.apache.lucene.spatial.util;
 
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.shape.Shape;
 import org.apache.lucene.index.*;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.BytesRef;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
index b1dfaaaf5698..08c1e43fbcc5 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
@@ -16,7 +16,7 @@
  */
 package org.apache.lucene.spatial.util;
 
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.shape.Shape;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
index d31fd594466e..7cab3fe0a075 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
@@ -16,8 +16,8 @@
  */
 package org.apache.lucene.spatial.vector;
 
-import com.spatial4j.core.distance.DistanceCalculator;
-import com.spatial4j.core.shape.Point;
+import org.locationtech.spatial4j.distance.DistanceCalculator;
+import org.locationtech.spatial4j.shape.Point;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.DocValues;
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
index f572f82f3ca2..f5f5f34121f7 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
@@ -16,11 +16,11 @@
  */
 package org.apache.lucene.spatial.vector;
 
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.shape.Circle;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Rectangle;
-import com.spatial4j.core.shape.Shape;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.shape.Circle;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
 import org.apache.lucene.document.LegacyDoubleField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
@@ -50,7 +50,7 @@
  * org.apache.lucene.spatial.query.SpatialOperation#Intersects} and {@link
  * SpatialOperation#IsWithin} is supported.
    • *
  • Uses the FieldCache for - * {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)} and for + * {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point)} and for * searching with a Circle.
    • * * @@ -60,7 +60,7 @@ * This is a simple Strategy. Search works with {@link org.apache.lucene.search.LegacyNumericRangeQuery}s on * an x and y pair of fields. A Circle query does the same bbox query but adds a * ValueSource filter on - * {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}. + * {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point)}. *

    * One performance shortcoming with this strategy is that a scenario involving * both a search using a Circle and sort will result in calculations for the @@ -106,7 +106,7 @@ public Field[] createIndexableFields(Shape shape) { throw new UnsupportedOperationException("Can only index Point, not " + shape); } - /** @see #createIndexableFields(com.spatial4j.core.shape.Shape) */ + /** @see #createIndexableFields(org.locationtech.spatial4j.shape.Shape) */ public Field[] createIndexableFields(Point point) { FieldType doubleFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); doubleFieldType.setNumericPrecisionStep(precisionStep); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java index 9a29677c6e73..160267905436 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java @@ -22,9 +22,9 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.spatial.bbox.BBoxStrategy; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java index 8506c868ddee..a081497c387a 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java @@ -22,10 +22,10 @@ import java.util.Set; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java index b1a5e542fc50..5dbb8f837e85 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java @@ -19,8 +19,8 @@ import java.util.ArrayList; import java.util.Collection; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.bbox.BBoxStrategy; import org.apache.lucene.spatial.composite.CompositeSpatialStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialArgsTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialArgsTest.java index 09b5d4666266..094953a95db1 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialArgsTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialArgsTest.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.query.SpatialArgs; import org.junit.Test; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java index 1bd715969c05..76e0200f26b3 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java @@ -18,10 +18,10 @@ import java.io.IOException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java index 94e5a8e68a72..529e98bc5587 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -24,10 +24,10 @@ import java.util.Random; import java.util.logging.Logger; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestData.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestData.java index 27d47b333646..06a68ce72e97 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestData.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestData.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import java.io.BufferedReader; import java.io.IOException; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestQuery.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestQuery.java index bac90cf45ca1..47e9c120eecb 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestQuery.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestQuery.java @@ -16,7 +16,7 @@ */ package org.apache.lucene.spatial; -import com.spatial4j.core.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContext; import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialArgsParser; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java index 00e437b857d6..df37d18d0e00 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java @@ -29,8 +29,8 @@ import java.util.Set; import java.util.logging.Logger; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/TestTestFramework.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/TestTestFramework.java index d31fdf635bfb..6af7467c2caf 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/TestTestFramework.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/TestTestFramework.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.spatial.query.SpatialArgsParser; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java index 6140996637e9..20a7202ba331 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java @@ -19,12 +19,12 @@ import java.io.IOException; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java index 8e1bb513c46f..7d49e8b2e4cb 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java @@ -19,12 +19,12 @@ import java.io.IOException; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java index 7cd4723d5b6c..33c8a330af95 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java @@ -20,7 +20,7 @@ import java.util.Calendar; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.prefix.tree.DateRangePrefixTree; import org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree.UnitNRShape; import org.apache.lucene.spatial.query.SpatialOperation; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/HeatmapFacetCounterTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/HeatmapFacetCounterTest.java index 124af79041b3..2de18cc32256 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/HeatmapFacetCounterTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/HeatmapFacetCounterTest.java @@ -21,15 +21,15 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Circle; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.search.Query; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.spatial.StrategyTestCase; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java index 09fb3a9fd45d..3f1684f58b5a 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/JtsPolygonTest.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.spatial.prefix; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; @@ -45,7 +45,7 @@ public JtsPolygonTest() { try { HashMap args = new HashMap<>(); args.put("spatialContextFactory", - "com.spatial4j.core.context.jts.JtsSpatialContextFactory"); + "org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory"); ctx = SpatialContextFactory.makeSpatialContext(args, getClass().getClassLoader()); } catch (NoClassDefFoundError e) { assumeTrue("This test requires JTS jar: "+e, false); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/NumberRangeFacetsTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/NumberRangeFacetsTest.java index 11e1d18701cf..514c18e078be 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/NumberRangeFacetsTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/NumberRangeFacetsTest.java @@ -23,7 +23,7 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.queries.TermsQuery; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java index 8db131c84a6f..c7e107cfba17 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java @@ -29,14 +29,14 @@ import java.util.Set; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.ShapeCollection; -import com.spatial4j.core.shape.SpatialRelation; -import com.spatial4j.core.shape.impl.RectangleImpl; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.ShapeCollection; +import org.locationtech.spatial4j.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; @@ -56,10 +56,10 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; -import static com.spatial4j.core.shape.SpatialRelation.CONTAINS; -import static com.spatial4j.core.shape.SpatialRelation.DISJOINT; -import static com.spatial4j.core.shape.SpatialRelation.INTERSECTS; -import static com.spatial4j.core.shape.SpatialRelation.WITHIN; +import static org.locationtech.spatial4j.shape.SpatialRelation.CONTAINS; +import static org.locationtech.spatial4j.shape.SpatialRelation.DISJOINT; +import static org.locationtech.spatial4j.shape.SpatialRelation.INTERSECTS; +import static org.locationtech.spatial4j.shape.SpatialRelation.WITHIN; /** Randomized PrefixTree test that considers the fuzziness of the * results introduced by grid approximation. */ diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpStrategyTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpStrategyTestCase.java index 87f1071503c8..22c58393f5f8 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpStrategyTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpStrategyTestCase.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Set; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.StrategyTestCase; import org.apache.lucene.spatial.query.SpatialArgs; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java index a53d52dee7ad..f852464ca198 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java @@ -16,10 +16,10 @@ */ package org.apache.lucene.spatial.prefix; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.StrategyTestCase; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java index 1a912c046203..fc131c5d7884 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial.prefix; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java index 74a989efb249..12e9744064b1 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java @@ -21,8 +21,8 @@ import java.util.Calendar; import java.util.GregorianCalendar; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; import org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree.UnitNRShape; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java index 403b8d196374..8a3d79c3b291 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTreeTest.java @@ -16,10 +16,10 @@ */ package org.apache.lucene.spatial.prefix.tree; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java index 93b95f3d5244..9724c6519b22 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/query/SpatialArgsParserTest.java @@ -16,8 +16,8 @@ */ package org.apache.lucene.spatial.query; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java index bed833939400..6a73d23d2f28 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java @@ -18,7 +18,7 @@ import java.io.IOException; -import com.spatial4j.core.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContext; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.StrategyTestCase; import org.junit.Before; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java index 8040a351ce21..d26bb29a5979 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java @@ -21,10 +21,10 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.composite.CompositeSpatialStrategy; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; @@ -41,7 +41,7 @@ import org.apache.lucene.geo3d.PlanetModel; import org.junit.Test; -import static com.spatial4j.core.distance.DistanceUtils.DEGREES_TO_RADIANS; +import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; public class Geo3dRptTest extends RandomSpatialOpStrategyTestCase { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java index 58b520db81cf..134b8c753666 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java @@ -19,12 +19,12 @@ import java.util.ArrayList; import java.util.List; -import com.spatial4j.core.TestLog; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Circle; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.RectIntersectionTestHelper; +import org.locationtech.spatial4j.TestLog; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.RectIntersectionTestHelper; import org.apache.lucene.geo3d.LatLonBounds; import org.apache.lucene.geo3d.GeoBBox; import org.apache.lucene.geo3d.GeoBBoxFactory; @@ -37,7 +37,7 @@ import org.junit.Rule; import org.junit.Test; -import static com.spatial4j.core.distance.DistanceUtils.DEGREES_TO_RADIANS; +import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; public abstract class Geo3dShapeRectRelationTestCase extends RandomizedShapeTestCase { protected final static double RADIANS_PER_DEGREE = Math.PI/180.0; @@ -110,7 +110,6 @@ protected int getBoundingMinimum(int laps) { } } - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6867") @Test public void testGeoCircleRect() { new Geo3dRectIntersectionTestHelper(ctx) { @@ -133,7 +132,6 @@ protected Point randomPointInEmptyShape(Geo3dShape shape) { }.testRelateWithRectangle(); } - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6867") @Test public void testGeoBBoxRect() { new Geo3dRectIntersectionTestHelper(ctx) { @@ -168,7 +166,6 @@ protected Point randomPointInEmptyShape(Geo3dShape shape) { }.testRelateWithRectangle(); } - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6867") @Test public void testGeoPolygonRect() { new Geo3dRectIntersectionTestHelper(ctx) { @@ -212,7 +209,6 @@ protected int getWithinMinimum(int laps) { }.testRelateWithRectangle(); } - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6867") @Test public void testGeoPathRect() { new Geo3dRectIntersectionTestHelper(ctx) { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java index aac0a0a04148..2d958231e6d2 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java @@ -19,7 +19,7 @@ import java.util.ArrayList; import java.util.List; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.geo3d.GeoArea; import org.apache.lucene.geo3d.GeoBBox; import org.apache.lucene.geo3d.GeoBBoxFactory; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/RandomizedShapeTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/RandomizedShapeTestCase.java index 40d1b248a702..0c18f5d7ddea 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/RandomizedShapeTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/RandomizedShapeTestCase.java @@ -16,17 +16,17 @@ */ package org.apache.lucene.spatial.spatial4j; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Circle; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.SpatialRelation; -import com.spatial4j.core.shape.impl.Range; - -import static com.spatial4j.core.shape.SpatialRelation.CONTAINS; -import static com.spatial4j.core.shape.SpatialRelation.WITHIN; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.SpatialRelation; +import org.locationtech.spatial4j.shape.impl.Range; + +import static org.locationtech.spatial4j.shape.SpatialRelation.CONTAINS; +import static org.locationtech.spatial4j.shape.SpatialRelation.WITHIN; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java index 1d559da5835a..444647404e5c 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java @@ -21,7 +21,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java index d62a0a8231fb..69f8c4da7abf 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.spatial.vector; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Circle; -import com.spatial4j.core.shape.Point; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.StrategyTestCase; diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 2936c5bb339e..5f8706f0e43f 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -36,7 +36,7 @@ - + diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java b/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java index dc1b9afd54d6..4ad882c8e99f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java @@ -32,8 +32,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.prefix.HeatmapFacetCounter; import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; import org.apache.lucene.spatial.query.SpatialArgs; diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index 83fd44775f41..222f0b83eb69 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -57,12 +57,12 @@ import com.google.common.base.Throwables; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.context.SpatialContextFactory; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; /** * Abstract base class for Solr FieldTypes based on a Lucene 4 {@link SpatialStrategy}. diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java index 164398b48dd1..abf55f1bb625 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java @@ -31,7 +31,7 @@ import org.apache.lucene.spatial.query.SpatialArgsParser; import org.apache.solr.util.MapListener; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index 2f282c85549e..f655e68ce068 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource; diff --git a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java index 38b68005ce05..95b441a9ed1d 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; diff --git a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java index 7deae5f9ddeb..2baf72e192b6 100644 --- a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java +++ b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java @@ -18,10 +18,10 @@ import java.io.IOException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.io.GeohashUtils; -import com.spatial4j.core.shape.Point; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.io.GeohashUtils; +import org.locationtech.spatial4j.shape.Point; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.LiteralValueSource; diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 8592c5aa0583..c30729adb48f 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -47,10 +47,10 @@ import org.apache.solr.search.SpatialOptions; import org.apache.solr.util.SpatialUtils; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; /** diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.java index 934523189014..b2f15ab533e4 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointType.java +++ b/solr/core/src/java/org/apache/solr/schema/PointType.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Map; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java index fe4cedc04c58..b633174f9100 100644 --- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java +++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java @@ -20,9 +20,9 @@ import java.lang.ref.WeakReference; import java.util.Map; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.shape.Shape; -import com.spatial4j.core.shape.jts.JtsGeometry; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.jts.JtsGeometry; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java index 2c05f33c3e8f..0830267652aa 100644 --- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java @@ -16,7 +16,7 @@ */ package org.apache.solr.search; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java index 1f4ea34ff955..956550c3ac3d 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java @@ -20,9 +20,9 @@ import java.util.Collections; import java.util.List; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.ConstNumberSource; import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; @@ -41,7 +41,7 @@ /** * Parses "geodist" creating {@link HaversineConstFunction} or {@link HaversineFunction} - * or calling {@link SpatialStrategy#makeDistanceValueSource(com.spatial4j.core.shape.Point,double)}. + * or calling {@link SpatialStrategy#makeDistanceValueSource(org.locationtech.spatial4j.shape.Point,double)}. */ public class GeoDistValueSourceParser extends ValueSourceParser { diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java index c4a7bd55705b..b00f2fd861a0 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java @@ -18,7 +18,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import com.spatial4j.core.io.GeohashUtils; +import org.locationtech.spatial4j.io.GeohashUtils; import java.util.Map; import java.io.IOException; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java index 4dd49e481219..915db3336786 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java @@ -15,11 +15,11 @@ * limitations under the License. */ package org.apache.solr.search.function.distance; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.distance.GeodesicSphereDistCalc; -import com.spatial4j.core.io.GeohashUtils; -import com.spatial4j.core.shape.Point; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.GeodesicSphereDistCalc; +import org.locationtech.spatial4j.io.GeohashUtils; +import org.locationtech.spatial4j.shape.Point; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java index f9ac062dda3b..4b68f5cec706 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java @@ -15,7 +15,7 @@ * limitations under the License. */ package org.apache.solr.search.function.distance; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; @@ -26,7 +26,7 @@ import java.io.IOException; import java.util.Map; -import static com.spatial4j.core.distance.DistanceUtils.DEGREES_TO_RADIANS; +import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; /** * Haversine function with one point constant diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java index c0eb0442ed24..20ed5a733bbd 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java @@ -21,7 +21,7 @@ import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.apache.lucene.queries.function.valuesource.MultiValueSource; import org.apache.lucene.search.IndexSearcher; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.solr.common.SolrException; import java.io.IOException; diff --git a/solr/core/src/java/org/apache/solr/util/DistanceUnits.java b/solr/core/src/java/org/apache/solr/util/DistanceUnits.java index 63f43e4652fb..a2163ac7430d 100644 --- a/solr/core/src/java/org/apache/solr/util/DistanceUnits.java +++ b/solr/core/src/java/org/apache/solr/util/DistanceUnits.java @@ -21,7 +21,7 @@ import java.util.Set; import com.google.common.collect.ImmutableMap; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.solr.schema.AbstractSpatialFieldType; /** diff --git a/solr/core/src/java/org/apache/solr/util/SpatialUtils.java b/solr/core/src/java/org/apache/solr/util/SpatialUtils.java index 1b05d01f5e91..9f6019cff338 100644 --- a/solr/core/src/java/org/apache/solr/util/SpatialUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SpatialUtils.java @@ -18,11 +18,11 @@ import java.text.ParseException; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.exception.InvalidShapeException; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; -import com.spatial4j.core.shape.Shape; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.exception.InvalidShapeException; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import org.apache.solr.common.SolrException; /** Utility methods pertaining to spatial. */ @@ -32,7 +32,7 @@ private SpatialUtils() {} /** * Parses a 'geom' parameter (might also be used to parse shapes for indexing). {@code geomStr} can either be WKT or - * a rectangle-range syntax (see {@link #parseRectangle(String, com.spatial4j.core.context.SpatialContext)}. + * a rectangle-range syntax (see {@link #parseRectangle(String, org.locationtech.spatial4j.context.SpatialContext)}. */ public static Shape parseGeomSolrException(String geomStr, SpatialContext ctx) { if (geomStr.length() == 0) { @@ -100,7 +100,7 @@ private static int findIndexNotSpace(String str, int startIdx, int inc) { return idx; } - /** Calls {@link #parsePoint(String, com.spatial4j.core.context.SpatialContext)} and wraps + /** Calls {@link #parsePoint(String, org.locationtech.spatial4j.context.SpatialContext)} and wraps * the exception with {@link org.apache.solr.common.SolrException} with a helpful message. */ public static Point parsePointSolrException(String externalVal, SpatialContext ctx) throws SolrException { try { @@ -116,7 +116,7 @@ public static Point parsePointSolrException(String externalVal, SpatialContext c /** * Parses {@code str} in the format of '[minPoint TO maxPoint]' where {@code minPoint} is the lower left corner * and maxPoint is the upper-right corner of the bounding box. Both corners may optionally be wrapped with a quote - * and then it's parsed via {@link #parsePoint(String, com.spatial4j.core.context.SpatialContext)}. + * and then it's parsed via {@link #parsePoint(String, org.locationtech.spatial4j.context.SpatialContext)}. * @param str Non-null; may *not* have leading or trailing spaces * @param ctx Non-null * @return the Rectangle @@ -140,7 +140,7 @@ public static Rectangle parseRectangle(String str, SpatialContext ctx) throws In } /** - * Calls {@link #parseRectangle(String, com.spatial4j.core.context.SpatialContext)} and wraps the exception with + * Calls {@link #parseRectangle(String, org.locationtech.spatial4j.context.SpatialContext)} and wraps the exception with * {@link org.apache.solr.common.SolrException} with a helpful message. */ public static Rectangle parseRectangeSolrException(String externalVal, SpatialContext ctx) throws SolrException { diff --git a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java index 20555c6fa506..2fe37409520f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java @@ -21,10 +21,10 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Point; -import com.spatial4j.core.shape.Rectangle; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.spatial.bbox.BBoxStrategy; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; diff --git a/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java index f33bc2a686fc..b3ad183a596d 100644 --- a/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java +++ b/solr/core/src/test/org/apache/solr/search/function/distance/DistanceFunctionTest.java @@ -15,8 +15,8 @@ * limitations under the License. */ package org.apache.solr.search.function.distance; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.io.GeohashUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.io.GeohashUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.junit.BeforeClass; diff --git a/solr/core/src/test/org/apache/solr/util/DistanceUnitsTest.java b/solr/core/src/test/org/apache/solr/util/DistanceUnitsTest.java index f16292cc3c50..395637166a09 100644 --- a/solr/core/src/test/org/apache/solr/util/DistanceUnitsTest.java +++ b/solr/core/src/test/org/apache/solr/util/DistanceUnitsTest.java @@ -16,7 +16,7 @@ */ package org.apache.solr.util; -import com.spatial4j.core.distance.DistanceUtils; +import org.locationtech.spatial4j.distance.DistanceUtils; import org.apache.lucene.util.LuceneTestCase; public class DistanceUnitsTest extends LuceneTestCase { diff --git a/solr/licenses/spatial4j-0.5.jar.sha1 b/solr/licenses/spatial4j-0.5.jar.sha1 deleted file mode 100644 index c81a76cbe91b..000000000000 --- a/solr/licenses/spatial4j-0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e16edaf6b1ba76db7f08c2f3723fce3b358ecc3 diff --git a/solr/licenses/spatial4j-0.6.jar.sha1 b/solr/licenses/spatial4j-0.6.jar.sha1 new file mode 100644 index 000000000000..56c02ad3016c --- /dev/null +++ b/solr/licenses/spatial4j-0.6.jar.sha1 @@ -0,0 +1 @@ +21b15310bddcfd8c72611c180f20cf23279809a3 diff --git a/solr/licenses/spatial4j-NOTICE.txt b/solr/licenses/spatial4j-NOTICE.txt index d7d48d185c69..779b8df7e911 100644 --- a/solr/licenses/spatial4j-NOTICE.txt +++ b/solr/licenses/spatial4j-NOTICE.txt @@ -1,5 +1,133 @@ -Spatial4j -Copyright 2012-2014 The Apache Software Foundation +# about.md file -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). +## About This Content + +May 22, 2015 + +### License + +The Eclipse Foundation makes available all content in this plug-in ("Content"). Unless otherwise indicated below, the +Content is provided to you under the terms and conditions of the Apache License, Version 2.0. A copy of the Apache +License, Version 2.0 is available at +[http://www.apache.org/licenses/LICENSE-2.0.txt](http://www.apache.org/licenses/LICENSE-2.0.txt) + +If you did not receive this Content directly from the Eclipse Foundation, the Content is being redistributed by another +party ("Redistributor") and different terms and conditions may apply to your use of any object code in the Content. +Check the Redistributor’s license that was provided with the Content. If no such license exists, contact the +Redistributor. Unless otherwise indicated below, the terms and conditions of the Apache License, Version 2.0 still apply +to any source code in the Content and such source code may be obtained at +[http://www.eclipse.org](http://www.eclipse.org). + +# notice.md file + +Note: the below Eclipse user agreement is standard. It says "Unless otherwise indicated, "... before referring to the +EPL. We indicate above that all content is licensed under the ASLv2 license. -- David Smiley + +## Eclipse Foundation Software User Agreement + +April 9, 2014 + +### Usage Of Content + +THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE +PROJECTS (COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR +THE TERMS AND CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE +THAT YOUR USE OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE +AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT +AND THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY +NOT USE THE CONTENT. + +### Applicable Licenses + +Unless otherwise indicated, all Content made available by the Eclipse Foundation is provided to you under the terms and +conditions of the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is provided with this Content and is +also available at [http://www.eclipse.org/legal/epl-v10.html](http://www.eclipse.org/legal/epl-v10.html). For purposes +of the EPL, "Program" will mean the Content. + +Content includes, but is not limited to, source code, object code, documentation and other files maintained in the +Eclipse Foundation source code repository ("Repository") in software modules ("Modules") and made available as +downloadable archives ("Downloads"). + +* Content may be structured and packaged into modules to facilitate delivering, extending, and upgrading the Content. + Typical modules may include plug-ins ("Plug-ins"), plug-in fragments ("Fragments"), and features ("Features"). +* Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java™ ARchive) in a directory named "plugins". +* A Feature is a bundle of one or more Plug-ins and/or Fragments and associated material. Each Feature may be packaged + as a sub-directory in a directory named "features". Within a Feature, files named "feature.xml" may contain a list + of the names and version numbers of the Plug-ins and/or Fragments associated with that Feature. +* Features may also include other Features ("Included Features"). Within a Feature, files named "feature.xml" may + contain a list of the names and version numbers of Included Features. + +The terms and conditions governing Plug-ins and Fragments should be contained in files named "about.html" ("Abouts"). +The terms and conditions governing Features and Included Features should be contained in files named "license.html" +("Feature Licenses"). Abouts and Feature Licenses may be located in any directory of a Download or Module including, but +not limited to the following locations: + +* The top-level (root) directory +* Plug-in and Fragment directories +* Inside Plug-ins and Fragments packaged as JARs +* Sub-directories of the directory named "src" of certain Plug-ins +* Feature directories + +Note: if a Feature made available by the Eclipse Foundation is installed using the Provisioning Technology (as defined +below), you must agree to a license ("Feature Update License") during the installation process. If the Feature contains +Included Features, the Feature Update License should either provide you with the terms and conditions governing the +Included Features or inform you where you can locate them. Feature Update Licenses may be found in the "license" +property of files named "feature.properties" found within a Feature. Such Abouts, Feature Licenses, and Feature Update +Licenses contain the terms and conditions (or references to such terms and conditions) that govern your use of the +associated Content in that directory. + +THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR +TERMS AND CONDITIONS. SOME OF THESE OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO): + +* Eclipse Distribution License Version 1.0 (available at + [http://www.eclipse.org/licenses/edl-v1.0.html](http://www.eclipse.org/licenses/edl-v10.html)) +* Common Public License Version 1.0 (available at + [http://www.eclipse.org/legal/cpl-v10.html](http://www.eclipse.org/legal/cpl-v10.html)) +* Apache Software License 1.1 (available at + [http://www.apache.org/licenses/LICENSE](http://www.apache.org/licenses/LICENSE)) +* Apache Software License 2.0 (available at + [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)) +* Mozilla Public License Version 1.1 (available at + [http://www.mozilla.org/MPL/MPL-1.1.html](http://www.mozilla.org/MPL/MPL-1.1.html)) + +IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature +License, or Feature Update License is provided, please contact the Eclipse Foundation to determine what terms and +conditions govern that particular Content. + +### Use of Provisioning Technology + +The Eclipse Foundation makes available provisioning software, examples of which include, but are not limited to, p2 and +the Eclipse Update Manager ("Provisioning Technology") for the purpose of allowing users to install software, +documentation, information and/or other materials (collectively "Installable Software"). This capability is provided +with the intent of allowing such users to install, extend and update Eclipse-based products. Information about packaging +Installable Software is available at +[http://eclipse.org/equinox/p2/repository_packaging.html](http://eclipse.org/equinox/p2/repository_packaging.html) +("Specification"). + +You may use Provisioning Technology to allow other parties to install Installable Software. You shall be responsible for +enabling the applicable license agreements relating to the Installable Software to be presented to, and accepted by, the +users of the Provisioning Technology in accordance with the Specification. By using Provisioning Technology in such a +manner and making it available in accordance with the Specification, you further acknowledge your agreement to, and the +acquisition of all necessary rights to permit the following: + +1. A series of actions may occur ("Provisioning Process") in which a user may execute the Provisioning Technology on a + machine ("Target Machine") with the intent of installing, extending or updating the functionality of an + Eclipse-based product. +2. During the Provisioning Process, the Provisioning Technology may cause third party Installable Software or a portion + thereof to be accessed and copied to the Target Machine. +3. Pursuant to the Specification, you will provide to the user the terms and conditions that govern the use of the + Installable Software ("Installable Software Agreement") and such Installable Software Agreement shall be accessed + from the Target Machine in accordance with the Specification. Such Installable Software Agreement must inform the + user of the terms and conditions that govern the Installable Software and must solicit acceptance by the end user in + the manner prescribed in such Installable Software Agreement. Upon such indication of agreement by the user, the + provisioning Technology will complete installation of the Installable Software. + +### Cryptography + +Content may contain encryption software. The country in which you are currently may have restrictions on the import, +possession, and use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, possession, or use, and re-export of +encryption software, to see if this is permitted. + +Java and all Java-based trademarks are trademarks of Oracle Corporation in the United States, other countries, +or both. \ No newline at end of file From 6dcb01ccb4ec77a1cf2fa89155335ead689535c1 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 2 Mar 2016 23:03:44 -0500 Subject: [PATCH 0008/1113] SOLR-8764: test schema-latest.xml spatial dist units should be kilometers (no test uses yet?) (cherry picked from commit deb6a49) --- .../core/src/test-files/solr/collection1/conf/schema_latest.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index 803d45eed770..ef82aa189b70 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -718,7 +718,7 @@ http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4 --> + geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" /> - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - 6.0.0 + 6.1.0 - diff --git a/solr/build.xml b/solr/build.xml index 1ab1a9279862..218bf8ce6d39 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -444,7 +444,7 @@ - From 267e326dbf137de5357e0aca7418f648752cb22a Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 7 Mar 2016 05:22:40 -0500 Subject: [PATCH 0047/1113] LUCENE-7072: always use WGS84 planet model in Geo3DPoint --- lucene/CHANGES.txt | 3 ++ .../apache/lucene/geo3d/BasePlanetObject.java | 5 +++ .../org/apache/lucene/geo3d/Geo3DPoint.java | 43 ++++++++----------- .../lucene/geo3d/PointInGeo3DShapeQuery.java | 30 ++++++------- .../apache/lucene/geo3d/TestGeo3DPoint.java | 25 +++++------ 5 files changed, 52 insertions(+), 54 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c23a3fe08a29..3647d5d191cb 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -118,6 +118,9 @@ API Changes * LUCENE-7064: MultiPhraseQuery is now immutable and should be constructed with MultiPhraseQuery.Builder. (Luc Vanlerberghe via Adrien Grand) +* LUCENE-7072: Geo3DPoint always uses WGS84 planet model. + (Robert Muir, Mike McCandless) + Optimizations * LUCENE-6891: Use prefix coding when writing points in diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java index b5e3d286adbe..c64b974fd1fb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java @@ -34,6 +34,11 @@ public abstract class BasePlanetObject { public BasePlanetObject(final PlanetModel planetModel) { this.planetModel = planetModel; } + + /** Returns the {@link PlanetModel} provided when this shape was created. */ + public PlanetModel getPlanetModel() { + return planetModel; + } @Override public int hashCode() { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java index fbdb00d7a3c1..cde87f3c77ba 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java @@ -36,8 +36,6 @@ * @lucene.experimental */ public final class Geo3DPoint extends Field { - private final PlanetModel planetModel; - /** Indexing {@link FieldType}. */ public static final FieldType TYPE = new FieldType(); static { @@ -46,16 +44,15 @@ public final class Geo3DPoint extends Field { } /** - * Creates a new Geo3DPoint field with the specified lat, lon (in radians), given a planet model. + * Creates a new Geo3DPoint field with the specified lat, lon (in radians). * * @throws IllegalArgumentException if the field name is null or lat or lon are out of bounds */ - public Geo3DPoint(String name, PlanetModel planetModel, double lat, double lon) { + public Geo3DPoint(String name, double lat, double lon) { super(name, TYPE); - this.planetModel = planetModel; // Translate lat/lon to x,y,z: - final GeoPoint point = new GeoPoint(planetModel, lat, lon); - fillFieldsData(planetModel, point.x, point.y, point.z); + final GeoPoint point = new GeoPoint(PlanetModel.WGS84, lat, lon); + fillFieldsData(point.x, point.y, point.z); } /** @@ -63,40 +60,38 @@ public Geo3DPoint(String name, PlanetModel planetModel, double lat, double lon) * * @throws IllegalArgumentException if the field name is null or lat or lon are out of bounds */ - public Geo3DPoint(String name, PlanetModel planetModel, double x, double y, double z) { + public Geo3DPoint(String name, double x, double y, double z) { super(name, TYPE); - this.planetModel = planetModel; - fillFieldsData(planetModel, x, y, z); + fillFieldsData(x, y, z); } - private void fillFieldsData(PlanetModel planetModel, double x, double y, double z) { + private void fillFieldsData(double x, double y, double z) { byte[] bytes = new byte[12]; - encodeDimension(planetModel, x, bytes, 0); - encodeDimension(planetModel, y, bytes, Integer.BYTES); - encodeDimension(planetModel, z, bytes, 2*Integer.BYTES); + encodeDimension(x, bytes, 0); + encodeDimension(y, bytes, Integer.BYTES); + encodeDimension(z, bytes, 2*Integer.BYTES); fieldsData = new BytesRef(bytes); } // public helper methods (e.g. for queries) /** Encode single dimension */ - public static void encodeDimension(PlanetModel planetModel, double value, byte bytes[], int offset) { - NumericUtils.intToSortableBytes(Geo3DUtil.encodeValue(planetModel.getMaximumMagnitude(), value), bytes, offset); + public static void encodeDimension(double value, byte bytes[], int offset) { + NumericUtils.intToSortableBytes(Geo3DUtil.encodeValue(PlanetModel.WGS84.getMaximumMagnitude(), value), bytes, offset); } /** Decode single dimension */ - public static double decodeDimension(PlanetModel planetModel, byte value[], int offset) { - return Geo3DUtil.decodeValueCenter(planetModel.getMaximumMagnitude(), NumericUtils.sortableBytesToInt(value, offset)); + public static double decodeDimension(byte value[], int offset) { + return Geo3DUtil.decodeValueCenter(PlanetModel.WGS84.getMaximumMagnitude(), NumericUtils.sortableBytesToInt(value, offset)); } /** Returns a query matching all points inside the provided shape. * - * @param planetModel The {@link PlanetModel} to use, which must match what was used during indexing * @param field field name. must not be {@code null}. * @param shape Which {@link GeoShape} to match */ - public static Query newShapeQuery(PlanetModel planetModel, String field, GeoShape shape) { - return new PointInGeo3DShapeQuery(planetModel, field, shape); + public static Query newShapeQuery(String field, GeoShape shape) { + return new PointInGeo3DShapeQuery(field, shape); } @Override @@ -108,9 +103,9 @@ public String toString() { result.append(':'); BytesRef bytes = (BytesRef) fieldsData; - result.append(" x=" + decodeDimension(planetModel, bytes.bytes, bytes.offset)); - result.append(" y=" + decodeDimension(planetModel, bytes.bytes, bytes.offset + Integer.BYTES)); - result.append(" z=" + decodeDimension(planetModel, bytes.bytes, bytes.offset + 2*Integer.BYTES)); + result.append(" x=" + decodeDimension(bytes.bytes, bytes.offset)); + result.append(" y=" + decodeDimension(bytes.bytes, bytes.offset + Integer.BYTES)); + result.append(" z=" + decodeDimension(bytes.bytes, bytes.offset + 2*Integer.BYTES)); result.append('>'); return result.toString(); } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java index 4d816963080b..9e2132d680db 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java @@ -40,14 +40,19 @@ class PointInGeo3DShapeQuery extends Query { final String field; - final PlanetModel planetModel; final GeoShape shape; /** The lats/lons must be clockwise or counter-clockwise. */ - public PointInGeo3DShapeQuery(PlanetModel planetModel, String field, GeoShape shape) { + public PointInGeo3DShapeQuery(String field, GeoShape shape) { this.field = field; - this.planetModel = planetModel; this.shape = shape; + + if (shape instanceof BasePlanetObject) { + BasePlanetObject planetObject = (BasePlanetObject) shape; + if (planetObject.getPlanetModel().equals(PlanetModel.WGS84) == false) { + throw new IllegalArgumentException("this qurey requires PlanetModel.WGS84, but got: " + planetObject.getPlanetModel()); + } + } } @Override @@ -88,7 +93,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { assert xyzSolid.getRelationship(shape) == GeoArea.WITHIN || xyzSolid.getRelationship(shape) == GeoArea.OVERLAPS: "expected WITHIN (1) or OVERLAPS (2) but got " + xyzSolid.getRelationship(shape) + "; shape="+shape+"; XYZSolid="+xyzSolid; */ - double planetMax = planetModel.getMaximumMagnitude(); + double planetMax = PlanetModel.WGS84.getMaximumMagnitude(); DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc()); @@ -103,9 +108,9 @@ public void visit(int docID) { @Override public void visit(int docID, byte[] packedValue) { assert packedValue.length == 12; - double x = Geo3DPoint.decodeDimension(planetModel, packedValue, 0); - double y = Geo3DPoint.decodeDimension(planetModel, packedValue, Integer.BYTES); - double z = Geo3DPoint.decodeDimension(planetModel, packedValue, 2 * Integer.BYTES); + double x = Geo3DPoint.decodeDimension(packedValue, 0); + double y = Geo3DPoint.decodeDimension(packedValue, Integer.BYTES); + double z = Geo3DPoint.decodeDimension(packedValue, 2 * Integer.BYTES); if (shape.isWithin(x, y, z)) { result.add(docID); } @@ -129,7 +134,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { assert yMin <= yMax; assert zMin <= zMax; - GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(planetModel, xMin, xMax, yMin, yMax, zMin, zMax); + GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, xMin, xMax, yMin, yMax, zMin, zMax); switch(xyzSolid.getRelationship(shape)) { case GeoArea.CONTAINS: @@ -165,10 +170,6 @@ public String getField() { return field; } - public PlanetModel getPlanetModel() { - return planetModel; - } - public GeoShape getShape() { return shape; } @@ -182,13 +183,12 @@ public boolean equals(Object o) { PointInGeo3DShapeQuery that = (PointInGeo3DShapeQuery) o; - return planetModel.equals(that.planetModel) && shape.equals(that.shape); + return shape.equals(that.shape); } @Override public final int hashCode() { int result = super.hashCode(); - result = 31 * result + planetModel.hashCode(); result = 31 * result + shape.hashCode(); return result; } @@ -203,8 +203,6 @@ public String toString(String field) { sb.append(this.field); sb.append(':'); } - sb.append(" PlanetModel: "); - sb.append(planetModel); sb.append(" Shape: "); sb.append(shape); return sb.toString(); diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java index 9d00d3e6ccb4..17a40755d2fc 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java @@ -106,13 +106,12 @@ public void testBasic() throws Exception { iwc.setCodec(getCodec()); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); - doc.add(new Geo3DPoint("field", PlanetModel.WGS84, toRadians(50.7345267), toRadians(-97.5303555))); + doc.add(new Geo3DPoint("field", toRadians(50.7345267), toRadians(-97.5303555))); w.addDocument(doc); IndexReader r = DirectoryReader.open(w); // We can't wrap with "exotic" readers because the query must see the BKD3DDVFormat: IndexSearcher s = newSearcher(r, false); - assertEquals(1, s.search(Geo3DPoint.newShapeQuery(PlanetModel.WGS84, - "field", + assertEquals(1, s.search(Geo3DPoint.newShapeQuery("field", GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, toRadians(50), toRadians(-97), Math.PI/180.)), 1).totalHits); w.close(); r.close(); @@ -640,8 +639,6 @@ private static GeoShape randomShape(PlanetModel planetModel) { private static void verify(double[] lats, double[] lons) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); - PlanetModel planetModel = getPlanetModel(); - // Else we can get O(N^2) merging: int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < lats.length/100) { @@ -662,7 +659,7 @@ private static void verify(double[] lats, double[] lons) throws Exception { doc.add(newStringField("id", ""+id, Field.Store.NO)); doc.add(new NumericDocValuesField("id", id)); if (Double.isNaN(lats[id]) == false) { - doc.add(new Geo3DPoint("point", planetModel, lats[id], lons[id])); + doc.add(new Geo3DPoint("point", lats[id], lons[id])); } w.addDocument(doc); if (id > 0 && random().nextInt(100) == 42) { @@ -710,13 +707,13 @@ private void _run() throws Exception { for (int iter=0;iter", point.toString()); + Geo3DPoint point = new Geo3DPoint("point", toRadians(44.244272), toRadians(7.769736)); + assertEquals("Geo3DPoint ", point.toString()); } public void testShapeQueryToString() { - assertEquals("PointInGeo3DShapeQuery: field=point: PlanetModel: PlanetModel.SPHERE Shape: GeoStandardCircle: {planetmodel=PlanetModel.SPHERE, center=[lat=0.3861041107739683, lon=0.06780373760536706], radius=0.1(5.729577951308232)}", - Geo3DPoint.newShapeQuery(PlanetModel.SPHERE, "point", GeoCircleFactory.makeGeoCircle(PlanetModel.SPHERE, toRadians(44.244272), toRadians(7.769736), 0.1)).toString()); + assertEquals("PointInGeo3DShapeQuery: field=point: Shape: GeoStandardCircle: {planetmodel=PlanetModel.WGS84, center=[lat=0.3861041107739683, lon=0.06780373760536706], radius=0.1(5.729577951308232)}", + Geo3DPoint.newShapeQuery("point", GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, toRadians(44.244272), toRadians(7.769736), 0.1)).toString()); } private static Directory getDirectory() { From dd04b6173955d55348b3abaec4c2a3e875e12487 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Mon, 7 Mar 2016 15:03:03 +0530 Subject: [PATCH 0048/1113] SOLR-8745: Deprecate costly ZkStateReader.updateClusterState(), replace with a narrow forceUpdateCollection(collection) (cherry picked from commit 093a8ce) --- solr/CHANGES.txt | 3 ++ .../hadoop/MorphlineGoLiveMiniMRTest.java | 1 - .../apache/solr/cloud/ElectionContext.java | 2 +- .../cloud/LeaderInitiatedRecoveryThread.java | 6 --- .../OverseerCollectionMessageHandler.java | 1 - .../org/apache/solr/cloud/ZkController.java | 2 +- .../solr/handler/CdcrRequestHandler.java | 2 +- .../solr/handler/admin/ClusterStatus.java | 3 -- .../handler/admin/CollectionsHandler.java | 2 - .../handler/admin/CoreAdminOperation.java | 6 +-- .../solr/handler/admin/RebalanceLeaders.java | 2 +- .../solr/cloud/BaseCdcrDistributedZkTest.java | 1 - .../solr/cloud/BasicDistributedZkTest.java | 4 +- .../cloud/ChaosMonkeyNothingIsSafeTest.java | 2 +- .../solr/cloud/ChaosMonkeyShardSplitTest.java | 2 +- .../solr/cloud/CollectionReloadTest.java | 2 +- .../cloud/CollectionTooManyReplicasTest.java | 6 +-- .../CollectionsAPIDistributedZkTest.java | 9 ++-- .../solr/cloud/CustomCollectionTest.java | 1 - .../apache/solr/cloud/DeleteShardTest.java | 2 - .../apache/solr/cloud/ForceLeaderTest.java | 11 ++--- .../apache/solr/cloud/HttpPartitionTest.java | 9 +--- .../LeaderFailoverAfterPartitionTest.java | 2 - .../LeaderInitiatedRecoveryOnCommitTest.java | 4 +- .../solr/cloud/MigrateRouteKeyTest.java | 4 +- .../org/apache/solr/cloud/OverseerTest.java | 12 ++--- .../solr/cloud/ReplicaPropertiesBase.java | 3 -- .../org/apache/solr/cloud/ShardSplitTest.java | 1 - .../org/apache/solr/cloud/SyncSliceTest.java | 1 - .../solr/cloud/TestCloudDeleteByQuery.java | 1 - .../apache/solr/cloud/TestCollectionAPI.java | 4 +- .../TestLeaderInitiatedRecoveryThread.java | 1 - .../solr/cloud/TestMiniSolrCloudCluster.java | 47 +++++++++---------- .../cloud/TestMiniSolrCloudClusterBase.java | 3 +- .../cloud/TestRandomRequestDistribution.java | 4 +- .../solr/cloud/TestRebalanceLeaders.java | 1 - .../solr/cloud/TestReplicaProperties.java | 1 - .../cloud/TestSolrCloudWithKerberosAlt.java | 1 + .../solr/cloud/UnloadDistributedZkTest.java | 4 +- .../apache/solr/cloud/ZkControllerTest.java | 2 +- .../solr/cloud/hdfs/StressHdfsTest.java | 3 +- .../cloud/overseer/ZkStateReaderTest.java | 6 +-- .../cloud/overseer/ZkStateWriterTest.java | 10 ++-- .../solr/common/cloud/ZkStateReader.java | 46 ++++++++++++++++++ .../solr/cloud/AbstractDistribZkTestBase.java | 4 +- .../cloud/AbstractFullDistribZkTestBase.java | 10 ++-- .../org/apache/solr/cloud/ChaosMonkey.java | 14 +----- 47 files changed, 129 insertions(+), 139 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 406776fe936f..d7ae2269e242 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -296,6 +296,9 @@ Optimizations * SOLR-8720: ZkController#publishAndWaitForDownStates should use #publishNodeAsDown. (Mark Miller) +* SOLR-8745: Deprecate costly ZkStateReader.updateClusterState(), replace with a narrow + forceUpdateCollection(collection) (Scott Blum via shalin) + Other Changes ---------------------- diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java index 1cc1723db9bd..95ed9b2b17d1 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java @@ -646,7 +646,6 @@ public void test() throws Exception { } Thread.sleep(200); - cloudClient.getZkStateReader().updateClusterState(); } if (TEST_NIGHTLY) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java index 210787757bbc..38f6083bcb62 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java @@ -462,7 +462,7 @@ void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStart) throws Kee public void publishActiveIfRegisteredAndNotActive(SolrCore core) throws KeeperException, InterruptedException { if (core.getCoreDescriptor().getCloudDescriptor().hasRegistered()) { ZkStateReader zkStateReader = zkController.getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collection); ClusterState clusterState = zkStateReader.getClusterState(); Replica rep = (clusterState == null) ? null : clusterState.getReplica(collection, leaderProps.getStr(ZkStateReader.CORE_NODE_NAME_PROP)); diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java index 7a72a6782ee8..589ed83e8330 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java +++ b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java @@ -244,12 +244,6 @@ protected void sendRecoveryCommandWithRetry() throws Exception { // see if the replica's node is still live, if not, no need to keep doing this loop ZkStateReader zkStateReader = zkController.getZkStateReader(); - try { - zkStateReader.updateClusterState(); - } catch (Exception exc) { - log.warn("Error when updating cluster state: "+exc); - } - if (!zkStateReader.getClusterState().liveNodesContain(replicaNodeName)) { log.warn("Node "+replicaNodeName+" hosting core "+coreNeedingRecovery+ " is no longer live. No need to keep trying to tell it to recover!"); diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index 6b7f6067d8b1..d7d894bc69b7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -1371,7 +1371,6 @@ private void waitForNewShard(String collectionName, String sliceName) throws Kee return; } Thread.sleep(1000); - zkStateReader.updateClusterState(); } throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find new slice " + sliceName + " in collection " + collectionName diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 7d2752a75aee..81897b717e92 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -883,7 +883,7 @@ public String register(String coreName, final CoreDescriptor desc, boolean recov } // make sure we have an update cluster state right away - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collection); return shardId; } finally { MDCLoggingContext.clear(); diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java index 585c8396d239..23e4abac304b 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java @@ -361,7 +361,7 @@ private void handleCollectionCheckpointAction(SolrQueryRequest req, SolrQueryRes throws IOException, SolrServerException { ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController(); try { - zkController.getZkStateReader().updateClusterState(); + zkController.getZkStateReader().forceUpdateCollection(collection); } catch (Exception e) { log.warn("Error when updating cluster state", e); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java index 667d9fa11f53..ff60adc465b3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java @@ -57,9 +57,6 @@ public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) { @SuppressWarnings("unchecked") public void getClusterStatus(NamedList results) throws KeeperException, InterruptedException { - zkStateReader.updateClusterState(); - - // read aliases Aliases aliases = zkStateReader.getAliases(); Map> collectionVsAliases = new HashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index de2104f4d074..593dac81bcc6 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -920,8 +920,6 @@ private static void waitForActiveCollection(String collectionName, ZkNodeProps m + (checkLeaderOnly ? "leaders" : "replicas")); ZkStateReader zkStateReader = cc.getZkController().getZkStateReader(); for (int i = 0; i < numRetries; i++) { - - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); Collection shards = clusterState.getSlices(collectionName); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 8240189cf040..e755b82ff49f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -461,6 +461,7 @@ public void call(CallInfo callInfo) throws InterruptedException, IOException, Ke // to accept updates CloudDescriptor cloudDescriptor = core.getCoreDescriptor() .getCloudDescriptor(); + String collection = cloudDescriptor.getCollectionName(); if (retry % 15 == 0) { if (retry > 0 && log.isInfoEnabled()) @@ -470,7 +471,7 @@ public void call(CallInfo callInfo) throws InterruptedException, IOException, Ke waitForState + "; forcing ClusterState update from ZooKeeper"); // force a cluster state update - coreContainer.getZkController().getZkStateReader().updateClusterState(); + coreContainer.getZkController().getZkStateReader().forceUpdateCollection(collection); } if (maxTries == 0) { @@ -483,7 +484,6 @@ public void call(CallInfo callInfo) throws InterruptedException, IOException, Ke } ClusterState clusterState = coreContainer.getZkController().getClusterState(); - String collection = cloudDescriptor.getCollectionName(); Slice slice = clusterState.getSlice(collection, cloudDescriptor.getShardId()); if (slice != null) { final Replica replica = slice.getReplicasMap().get(coreNodeName); @@ -937,4 +937,4 @@ long getIndexSize(SolrCore core) { return size; } -} \ No newline at end of file +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 4626fc929674..98e796da73da 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -79,7 +79,7 @@ void execute() throws KeeperException, InterruptedException { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format(Locale.ROOT, "The " + COLLECTION_PROP + " is required for the Rebalance Leaders command.")); } - coreContainer.getZkController().getZkStateReader().updateClusterState(); + coreContainer.getZkController().getZkStateReader().forceUpdateCollection(collectionName); ClusterState clusterState = coreContainer.getZkController().getClusterState(); DocCollection dc = clusterState.getCollection(collectionName); if (dc == null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java index f1f3e9167b13..fe94309bba26 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java @@ -635,7 +635,6 @@ protected void updateMappingsFromZk(String collection) throws Exception { try { cloudClient.connect(); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollection(collection); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index d25ce6648097..8222e91677f4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -552,7 +552,7 @@ private void testStopAndStartCoresInOneInstance() throws Exception { Thread.sleep(5000); ChaosMonkey.start(cloudJettys.get(0).jetty); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection("multiunload2"); try { cloudClient.getZkStateReader().getLeaderRetry("multiunload2", "shard1", 30000); } catch (SolrException e) { @@ -830,7 +830,7 @@ private void testANewCollectionInOneInstanceWithManualShardAssignement() throws // we added a role of none on these creates - check for it ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(oneInstanceCollection2); Map slices = zkStateReader.getClusterState().getSlicesMap(oneInstanceCollection2); assertNotNull(slices); String roles = slices.get("slice1").getReplicasMap().values().iterator().next().getStr(ZkStateReader.ROLES_PROP); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 8cc80d9c8433..7dceada16687 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -205,7 +205,7 @@ public void test() throws Exception { // TODO: assert we didnt kill everyone - zkStateReader.updateClusterState(); + zkStateReader.updateLiveNodes(); assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java index 7a44561d38b8..190db573a504 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java @@ -206,7 +206,7 @@ private void waitTillRecovered() throws Exception { for (int i = 0; i < 30; i++) { Thread.sleep(3000); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection("collection1"); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection collection1 = clusterState.getCollection("collection1"); Slice slice = collection1.getSlice("shard1"); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java index b6eb5e2a494f..65ff78bf06a1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java @@ -103,7 +103,7 @@ public void testReloadedLeaderStateAfterZkSessionLoss() throws Exception { timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutSecs, TimeUnit.SECONDS); while (System.nanoTime() < timeout) { // state of leader should be active after session loss recovery - see SOLR-7338 - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); ClusterState cs = cloudClient.getZkStateReader().getClusterState(); Slice slice = cs.getSlice(testCollectionName, shardId); replicaState = slice.getReplica(leader.getName()).getStr(ZkStateReader.STATE_PROP); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionTooManyReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionTooManyReplicasTest.java index 92fea45e3353..afc7c483fb08 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionTooManyReplicasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionTooManyReplicasTest.java @@ -97,7 +97,7 @@ public void testAddTooManyReplicas() throws Exception { assertEquals(0, response.getStatus()); ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); Slice slice = zkStateReader.getClusterState().getSlicesMap(collectionName).get("shard1"); Replica rep = null; @@ -194,7 +194,7 @@ public void testAddShard() throws Exception { // And finally, insure that there are all the replcias we expect. We should have shards 1, 2 and 4 and each // should have exactly two replicas ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); Map slices = zkStateReader.getClusterState().getSlicesMap(collectionName); assertEquals("There should be exaclty four slices", slices.size(), 4); assertNotNull("shardstart should exist", slices.get("shardstart")); @@ -275,7 +275,7 @@ public void testDownedShards() throws Exception { private List getAllNodeNames(String collectionName) throws KeeperException, InterruptedException { ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); Slice slice = zkStateReader.getClusterState().getSlicesMap(collectionName).get("shard1"); List nodes = new ArrayList<>(); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java index 93f82acf20db..641dadfc236f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java @@ -368,7 +368,6 @@ private void deleteCollectionWithDownNodes() throws Exception { } Thread.sleep(200); - cloudClient.getZkStateReader().updateClusterState(); } assertFalse("Still found collection that should be gone", cloudClient.getZkStateReader().getClusterState().hasCollection("halfdeletedcollection2")); @@ -540,8 +539,6 @@ private void testErrorHandling() throws Exception { } private void testNoCollectionSpecified() throws Exception { - - cloudClient.getZkStateReader().updateClusterState(); assertFalse(cloudClient.getZkStateReader().getClusterState().hasCollection("corewithnocollection")); assertFalse(cloudClient.getZkStateReader().getClusterState().hasCollection("corewithnocollection2")); @@ -565,13 +562,13 @@ private void testNoCollectionSpecified() throws Exception { makeRequest(getBaseUrl((HttpSolrClient) clients.get(1)), createCmd); // in both cases, the collection should have default to the core name - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection("corewithnocollection"); + cloudClient.getZkStateReader().forceUpdateCollection("corewithnocollection2"); assertTrue(cloudClient.getZkStateReader().getClusterState().hasCollection("corewithnocollection")); assertTrue(cloudClient.getZkStateReader().getClusterState().hasCollection("corewithnocollection2")); } private void testNoConfigSetExist() throws Exception { - cloudClient.getZkStateReader().updateClusterState(); assertFalse(cloudClient.getZkStateReader().getClusterState().hasCollection("corewithnocollection3")); // try and create a SolrCore with no collection name @@ -592,7 +589,7 @@ private void testNoConfigSetExist() throws Exception { assertTrue(gotExp); TimeUnit.MILLISECONDS.sleep(200); // in both cases, the collection should have default to the core name - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection("corewithnocollection3"); Collection slices = cloudClient.getZkStateReader().getClusterState().getActiveSlices("corewithnocollection3"); int replicaCount = 0; diff --git a/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java index 081e96f4f081..0951b5d6fc02 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java @@ -409,7 +409,6 @@ private void testCreateShardRepFactor() throws Exception { int attempts = 0; while (true) { if (attempts > 30) fail("Not enough active replicas in the shard 'x'"); - zkStateReader.updateClusterState(); attempts++; replicaCount = zkStateReader.getClusterState().getSlice(collectionName, "x").getReplicas().size(); if (replicaCount >= 1) break; diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java index 101bfb98c20a..812fbe932187 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java @@ -96,7 +96,6 @@ protected void confirmShardDeletion(String shard) throws SolrServerException, Ke ClusterState clusterState = zkStateReader.getClusterState(); int counter = 10; while (counter-- > 0) { - zkStateReader.updateClusterState(); clusterState = zkStateReader.getClusterState(); if (clusterState.getSlice("collection1", shard) == null) { break; @@ -142,7 +141,6 @@ protected void setSliceState(String slice, State state) throws SolrServerExcepti boolean transition = false; for (int counter = 10; counter > 0; counter--) { - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); State sliceState = clusterState.getSlice("collection1", slice).getState(); if (sliceState == state) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java index c68fe9c3a265..a71c3e614131 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java @@ -89,7 +89,7 @@ public void testReplicasInLIRNoLeader() throws Exception { putNonLeadersIntoLIR(testCollectionName, SHARD1, zkController, leader, notLeaders); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); int numActiveReplicas = getNumberOfActiveReplicas(clusterState, testCollectionName, SHARD1); assertEquals("Expected only 0 active replica but found " + numActiveReplicas + @@ -114,7 +114,7 @@ public void testReplicasInLIRNoLeader() throws Exception { // By now we have an active leader. Wait for recoveries to begin waitForRecoveriesToFinish(testCollectionName, cloudClient.getZkStateReader(), true); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); clusterState = cloudClient.getZkStateReader().getClusterState(); log.info("After forcing leader: " + clusterState.getSlice(testCollectionName, SHARD1)); // we have a leader @@ -187,7 +187,7 @@ public void testLastPublishedStateIsActive() throws Exception { setReplicaState(testCollectionName, SHARD1, rep, State.DOWN); } - zkController.getZkStateReader().updateClusterState(); + zkController.getZkStateReader().forceUpdateCollection(testCollectionName); // Assert all replicas are down and that there is no leader assertEquals(0, getActiveOrRecoveringReplicas(testCollectionName, SHARD1).size()); @@ -224,7 +224,6 @@ protected void unsetLeader(String collection, String slice) throws Exception { ClusterState clusterState = null; boolean transition = false; for (int counter = 10; counter > 0; counter--) { - zkStateReader.updateClusterState(); clusterState = zkStateReader.getClusterState(); Replica newLeader = clusterState.getSlice(collection, slice).getLeader(); if (newLeader == null) { @@ -259,7 +258,6 @@ protected void setReplicaState(String collection, String slice, Replica replica, Replica.State replicaState = null; for (int counter = 10; counter > 0; counter--) { - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); replicaState = clusterState.getSlice(collection, slice).getReplica(replica.getName()).getState(); if (replicaState == state) { @@ -355,7 +353,6 @@ void putNonLeadersIntoLIR(String collectionName, String shard, ZkController zkCo for (int j = 0; j < notLeaders.size(); j++) lirStates[j] = zkController.getLeaderInitiatedRecoveryState(collectionName, shard, notLeaders.get(j).getName()); - zkController.getZkStateReader().updateClusterState(); ClusterState clusterState = zkController.getZkStateReader().getClusterState(); boolean allDown = true; for (State lirState : lirStates) @@ -391,7 +388,7 @@ protected void bringBackOldLeaderAndSendDoc(String collection, Replica leader, L JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(leader)); leaderJetty.start(); waitForRecoveriesToFinish(collection, cloudClient.getZkStateReader(), true); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(collection); ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); log.info("After bringing back leader: " + clusterState.getSlice(collection, SHARD1)); int numActiveReplicas = getNumberOfActiveReplicas(clusterState, collection, SHARD1); diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java index 8fecc84045ad..f1960aa952ea 100644 --- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java @@ -215,7 +215,7 @@ protected void testMinRf() throws Exception { // Verify that the partitioned replica is DOWN ZkStateReader zkr = cloudClient.getZkStateReader(); - zkr.updateClusterState(); // force the state to be fresh + zkr.forceUpdateCollection(testCollectionName);; // force the state to be fresh ClusterState cs = zkr.getClusterState(); Collection slices = cs.getActiveSlices(testCollectionName); Slice slice = slices.iterator().next(); @@ -645,18 +645,13 @@ protected void waitToSeeReplicasActive(String testCollectionName, String shardId final RTimer timer = new RTimer(); ZkStateReader zkr = cloudClient.getZkStateReader(); - zkr.updateClusterState(); // force the state to be fresh - + zkr.forceUpdateCollection(testCollectionName); ClusterState cs = zkr.getClusterState(); Collection slices = cs.getActiveSlices(testCollectionName); boolean allReplicasUp = false; long waitMs = 0L; long maxWaitMs = maxWaitSecs * 1000L; while (waitMs < maxWaitMs && !allReplicasUp) { - // refresh state every 2 secs - if (waitMs % 2000 == 0) - cloudClient.getZkStateReader().updateClusterState(); - cs = cloudClient.getZkStateReader().getClusterState(); assertNotNull(cs); Slice shard = cs.getSlice(testCollectionName, shardId); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderFailoverAfterPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderFailoverAfterPartitionTest.java index 6fd7c534809f..0436d5e874b3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderFailoverAfterPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderFailoverAfterPartitionTest.java @@ -159,8 +159,6 @@ protected void testRf3WithLeaderFailover() throws Exception { long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); while (System.nanoTime() < timeout) { - cloudClient.getZkStateReader().updateClusterState(); - List activeReps = getActiveOrRecoveringReplicas(testCollectionName, "shard1"); if (activeReps.size() >= 2) break; Thread.sleep(1000); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java index 8d2cc70c786a..7d6c633f4827 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java @@ -80,7 +80,7 @@ private void multiShardTest() throws Exception { Thread.sleep(sleepMsBeforeHealPartition); - cloudClient.getZkStateReader().updateClusterState(); // get the latest state + cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); // get the latest state leader = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1"); assertSame("Leader was not active", Replica.State.ACTIVE, leader.getState()); @@ -128,7 +128,7 @@ private void oneShardTest() throws Exception { sendCommitWithRetry(replica); Thread.sleep(sleepMsBeforeHealPartition); - cloudClient.getZkStateReader().updateClusterState(); // get the latest state + cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); // get the latest state leader = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1"); assertSame("Leader was not active", Replica.State.ACTIVE, leader.getState()); diff --git a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java index f9566e30f088..c09e0d1dc6a0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java @@ -72,7 +72,7 @@ private boolean waitForRuleToExpire(String splitKey, long finishTime) throws Kee boolean ruleRemoved = false; long expiryTime = finishTime + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); while (System.nanoTime() < expiryTime) { - getCommonCloudSolrClient().getZkStateReader().updateClusterState(); + getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); state = getCommonCloudSolrClient().getZkStateReader().getClusterState(); slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2); Map routingRules = slice.getRoutingRules(); @@ -186,7 +186,7 @@ protected void multipleShardMigrateTest() throws Exception { log.info("Response from target collection: " + response); assertEquals("DocCount on target collection does not match", splitKeyCount[0], response.getResults().getNumFound()); - getCommonCloudSolrClient().getZkStateReader().updateClusterState(); + getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); ClusterState state = getCommonCloudSolrClient().getZkStateReader().getClusterState(); Slice slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2); assertNotNull("Routing rule map is null", slice.getRoutingRules()); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index 66a214f7fcbf..85a88ec3ae9d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -439,7 +439,6 @@ public void testShardAssignmentBigger() throws Exception { int cloudStateSliceCount = 0; for (int i = 0; i < 40; i++) { cloudStateSliceCount = 0; - reader.updateClusterState(); ClusterState state = reader.getClusterState(); final Map slices = state.getSlicesMap(collection); if (slices != null) { @@ -524,7 +523,6 @@ public void testShardAssignmentBigger() throws Exception { private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException { int maxIterations = 100; while (0 < maxIterations--) { - stateReader.updateClusterState(); final ClusterState state = stateReader.getClusterState(); Set availableCollections = state.getCollections(); int availableCount = 0; @@ -605,7 +603,6 @@ public void testStateChange() throws Exception { private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException { int maxIterations = 200; while(maxIterations-->0) { - reader.updateClusterState(); // poll state ZkNodeProps props = reader.getClusterState().getLeader(collection, shard); if(props!=null) { if(expectedCore.equals(props.getStr(ZkStateReader.CORE_NAME_PROP))) { @@ -832,7 +829,8 @@ public void testShardLeaderChange() throws Exception { killerThread = new Thread(killer); killerThread.start(); - reader = new ZkStateReader(controllerClient); //no watches, we'll poll + reader = new ZkStateReader(controllerClient); + reader.createClusterStateWatchersAndUpdate(); for (int i = 0; i < atLeast(4); i++) { killCounter.incrementAndGet(); //for each round allow 1 kill @@ -905,9 +903,10 @@ public void testDoubleAssignment() throws Exception { mockController = new MockZKController(server.getZkAddress(), "node1"); mockController.publishState(collection, "core1", "core_node1", Replica.State.RECOVERING, 1); - while (version == getClusterStateVersion(controllerClient)); + while (version == reader.getClusterState().getZkClusterStateVersion()) { + Thread.sleep(100); + } - reader.updateClusterState(); ClusterState state = reader.getClusterState(); int numFound = 0; @@ -1048,7 +1047,6 @@ public void testPerformance() throws Exception { assertTrue(overseers.size() > 0); while (true) { - reader.updateClusterState(); ClusterState state = reader.getClusterState(); if (state.hasCollection("perf_sentinel")) { break; diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicaPropertiesBase.java b/solr/core/src/test/org/apache/solr/cloud/ReplicaPropertiesBase.java index 8347af093722..fe83a8431a3d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplicaPropertiesBase.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplicaPropertiesBase.java @@ -56,7 +56,6 @@ public static void verifyPropertyNotPresent(CloudSolrClient client, String colle ClusterState clusterState = null; Replica replica = null; for (int idx = 0; idx < 300; ++idx) { - client.getZkStateReader().updateClusterState(); clusterState = client.getZkStateReader().getClusterState(); replica = clusterState.getReplica(collectionName, replicaName); if (replica == null) { @@ -82,7 +81,6 @@ public static void verifyPropertyVal(CloudSolrClient client, String collectionNa ClusterState clusterState = null; for (int idx = 0; idx < 300; ++idx) { // Keep trying while Overseer writes the ZK state for up to 30 seconds. - client.getZkStateReader().updateClusterState(); clusterState = client.getZkStateReader().getClusterState(); replica = clusterState.getReplica(collectionName, replicaName); if (replica == null) { @@ -116,7 +114,6 @@ public static void verifyUnique(CloudSolrClient client, String collectionName, S DocCollection col = null; for (int idx = 0; idx < 300; ++idx) { - client.getZkStateReader().updateClusterState(); ClusterState clusterState = client.getZkStateReader().getClusterState(); col = clusterState.getCollection(collectionName); diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java index 22735abdb251..6d4b9cc3b8f9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java @@ -416,7 +416,6 @@ protected void checkDocCountsAndShardStates(int[] docCounts, int numReplicas) th int i = 0; for (i = 0; i < 10; i++) { ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - zkStateReader.updateClusterState(); clusterState = zkStateReader.getClusterState(); slice1_0 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, "shard1_0"); slice1_1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, "shard1_1"); diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java index e753be9f459c..362009e684bc 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java @@ -218,7 +218,6 @@ private void waitTillAllNodesActive() throws Exception { for (int i = 0; i < 60; i++) { Thread.sleep(3000); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection collection1 = clusterState.getCollection("collection1"); Slice slice = collection1.getSlice("shard1"); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java index a0bb42a3ee61..f4436eb9e65b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java @@ -119,7 +119,6 @@ private static void createMiniSolrCloudCluster() throws Exception { String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/","_"); urlMap.put(nodeKey, jettyURL.toString()); } - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); for (Slice slice : clusterState.getSlices(COLLECTION_NAME)) { String shardName = slice.getName(); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java index b203f02c8770..45b6f733becf 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java @@ -625,7 +625,7 @@ private void testClusterStateMigration() throws Exception { .setCollectionName("testClusterStateMigration") .process(client); - client.getZkStateReader().updateClusterState(); + client.getZkStateReader().forceUpdateCollection("testClusterStateMigration"); assertEquals(2, client.getZkStateReader().getClusterState().getCollection("testClusterStateMigration").getStateFormat()); @@ -735,7 +735,7 @@ private void testShardCreationNameValidation() throws Exception { private Map getProps(CloudSolrClient client, String collectionName, String replicaName, String... props) throws KeeperException, InterruptedException { - client.getZkStateReader().updateClusterState(); + client.getZkStateReader().forceUpdateCollection(collectionName); ClusterState clusterState = client.getZkStateReader().getClusterState(); Replica replica = clusterState.getReplica(collectionName, replicaName); if (replica == null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java b/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java index f2c58cf808a3..11858f828b7f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java @@ -175,7 +175,6 @@ protected void updateLIRState(String replicaCoreNodeName) { timeOut = new TimeOut(30, TimeUnit.SECONDS); while (!timeOut.hasTimedOut()) { - cloudClient.getZkStateReader().updateClusterState(); Replica r = cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, replica.getName()); if (r.getState() == Replica.State.DOWN) { break; diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java index 9be89190d424..880051b1f83f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java @@ -176,7 +176,7 @@ public void testCollectionCreateSearchDelete() throws Exception { assertEquals(1, rsp.getResults().getNumFound()); // remove a server not hosting any replicas - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); ClusterState clusterState = zkStateReader.getClusterState(); HashMap jettyMap = new HashMap(); for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) { @@ -321,7 +321,8 @@ public void testCollectionCreateWithoutCoresThenDelete() throws Exception { try (SolrZkClient zkClient = new SolrZkClient (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null); ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { - + zkStateReader.createClusterStateWatchersAndUpdate(); + // wait for collection to appear AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); @@ -368,6 +369,7 @@ public void testStopAllStartAll() throws Exception { try (SolrZkClient zkClient = new SolrZkClient (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null); ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { + zkStateReader.createClusterStateWatchersAndUpdate(); AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); // modify collection @@ -385,7 +387,7 @@ public void testStopAllStartAll() throws Exception { } // the test itself - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); final ClusterState clusterState = zkStateReader.getClusterState(); final HashSet leaderIndices = new HashSet(); @@ -444,7 +446,7 @@ public void testStopAllStartAll() throws Exception { } AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); // re-query collection { @@ -489,32 +491,29 @@ public void testSegmentTerminateEarly() throws Exception { } } - try (SolrZkClient zkClient = new SolrZkClient - (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null); - ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { - AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); + ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); + AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); - // add some documents, then optimize to get merged-sorted segments - tstes.addDocuments(cloudSolrClient, 10, 10, true); + // add some documents, then optimize to get merged-sorted segments + tstes.addDocuments(cloudSolrClient, 10, 10, true); - // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent - tstes.queryTimestampDescending(cloudSolrClient); + // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent + tstes.queryTimestampDescending(cloudSolrClient); - // add a few more documents, but don't optimize to have some not-merge-sorted segments - tstes.addDocuments(cloudSolrClient, 2, 10, false); + // add a few more documents, but don't optimize to have some not-merge-sorted segments + tstes.addDocuments(cloudSolrClient, 2, 10, false); - // CommonParams.SEGMENT_TERMINATE_EARLY parameter now present - tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient); - tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient); + // CommonParams.SEGMENT_TERMINATE_EARLY parameter now present + tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient); + tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient); - // CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used - tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient); - tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order + // CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used + tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient); + tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order - // delete the collection we created earlier - miniCluster.deleteCollection(collectionName); - AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, zkStateReader, true, true, 330); - } + // delete the collection we created earlier + miniCluster.deleteCollection(collectionName); + AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, zkStateReader, true, true, 330); } finally { miniCluster.shutdown(); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterBase.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterBase.java index 54b21dff5de8..18285617d9a9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterBase.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterBase.java @@ -146,6 +146,7 @@ protected void testCollectionCreateSearchDelete(String collectionName) throws Ex try (SolrZkClient zkClient = new SolrZkClient (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null); ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { + zkStateReader.createClusterStateWatchersAndUpdate(); AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); // modify/query collection @@ -160,7 +161,7 @@ protected void testCollectionCreateSearchDelete(String collectionName) throws Ex assertEquals(1, rsp.getResults().getNumFound()); // remove a server not hosting any replicas - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collectionName); ClusterState clusterState = zkStateReader.getClusterState(); HashMap jettyMap = new HashMap(); for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java index 25ffe8425695..256774d08c39 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java @@ -88,7 +88,7 @@ private void testRequestTracking() throws Exception { waitForRecoveriesToFinish("a1x2", true); waitForRecoveriesToFinish("b1x1", true); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection("b1x1"); ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); DocCollection b1x1 = clusterState.getCollection("b1x1"); @@ -137,7 +137,7 @@ private void testQueryAgainstDownReplica() throws Exception { waitForRecoveriesToFinish("football", true); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection("football"); Replica leader = null; Replica notLeader = null; diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java index 3c720bfeddfc..9208229976a3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java @@ -310,7 +310,6 @@ boolean waitForAllPreferreds() throws KeeperException, InterruptedException { TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS); while (! timeout.hasTimedOut()) { goAgain = false; - cloudClient.getZkStateReader().updateClusterState(); Map slices = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION_NAME).getSlicesMap(); for (Map.Entry ent : expected.entrySet()) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java b/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java index 5cc15e2ba366..fc2a7e25740e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestReplicaProperties.java @@ -192,7 +192,6 @@ private void verifyLeaderAssignment(CloudSolrClient client, String collectionNam String lastFailMsg = ""; for (int idx = 0; idx < 300; ++idx) { // Keep trying while Overseer writes the ZK state for up to 30 seconds. lastFailMsg = ""; - client.getZkStateReader().updateClusterState(); ClusterState clusterState = client.getZkStateReader().getClusterState(); for (Slice slice : clusterState.getSlices(collectionName)) { Boolean foundLeader = false; diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java index 4d3ee30ad696..f4dc97de95b0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java @@ -205,6 +205,7 @@ protected void testCollectionCreateSearchDelete() throws Exception { try (SolrZkClient zkClient = new SolrZkClient (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null); ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { + zkStateReader.createClusterStateWatchersAndUpdate(); AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); // modify/query collection diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index dd337fb8530e..7d53feebf73c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -187,7 +187,7 @@ private void testCoreUnloadAndLeaders() throws Exception { } ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection("unloadcollection"); int slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); assertEquals(1, slices); @@ -203,7 +203,7 @@ private void testCoreUnloadAndLeaders() throws Exception { createCmd.setDataDir(getDataDir(core2dataDir)); adminClient.request(createCmd); } - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection("unloadcollection"); slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); assertEquals(1, slices); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index cffbb543e493..7b293ca5ea6e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -296,7 +296,7 @@ public List getCurrentDescriptors() { byte[] bytes = Utils.toJSON(state); zkController.getZkClient().makePath(ZkStateReader.getCollectionPath("testPublishAndWaitForDownStates"), bytes, CreateMode.PERSISTENT, true); - zkController.getZkStateReader().updateClusterState(); + zkController.getZkStateReader().forceUpdateCollection("testPublishAndWaitForDownStates"); assertTrue(zkController.getZkStateReader().getClusterState().hasCollection("testPublishAndWaitForDownStates")); assertNotNull(zkController.getZkStateReader().getClusterState().getCollection("testPublishAndWaitForDownStates")); diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java index 445c4b8f615d..601f4fe723ab 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java @@ -154,7 +154,7 @@ private void createAndDeleteCollection() throws SolrServerException, waitForRecoveriesToFinish(DELETE_DATA_DIR_COLLECTION, false); cloudClient.setDefaultCollection(DELETE_DATA_DIR_COLLECTION); - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(DELETE_DATA_DIR_COLLECTION); for (int i = 1; i < nShards + 1; i++) { cloudClient.getZkStateReader().getLeaderRetry(DELETE_DATA_DIR_COLLECTION, "shard" + i, 30000); @@ -211,7 +211,6 @@ private void createAndDeleteCollection() throws SolrServerException, } Thread.sleep(200); - cloudClient.getZkStateReader().updateClusterState(); } // check that all dirs are gone diff --git a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java index 69626b0828b2..10cc46c51657 100644 --- a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java @@ -94,7 +94,7 @@ public void testStateFormatUpdate(boolean explicitRefresh, boolean isInteresting assertFalse(exists); if (explicitRefresh) { - reader.updateClusterState(); + reader.forceUpdateCollection("c1"); } else { for (int i = 0; i < 100; ++i) { if (reader.getClusterState().hasCollection("c1")) { @@ -122,7 +122,7 @@ public void testStateFormatUpdate(boolean explicitRefresh, boolean isInteresting assertTrue(exists); if (explicitRefresh) { - reader.updateClusterState(); + reader.forceUpdateCollection("c1"); } else { for (int i = 0; i < 100; ++i) { if (reader.getClusterState().getCollection("c1").getStateFormat() == 2) { @@ -167,7 +167,7 @@ public void testExternalCollectionWatchedNotWatched() throws Exception{ new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json")); writer.enqueueUpdate(reader.getClusterState(), c1, null); writer.writePendingUpdates(); - reader.updateClusterState(); + reader.forceUpdateCollection("c1"); assertTrue(reader.getClusterState().getCollectionRef("c1").isLazilyLoaded()); reader.addCollectionWatch("c1"); diff --git a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java index 8e7b0098121f..f5648bf148c7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java @@ -233,7 +233,8 @@ public void testExternalModificationToSharedClusterState() throws Exception { writer.enqueueUpdate(reader.getClusterState(), c1, null); writer.writePendingUpdates(); - reader.updateClusterState(); + reader.forceUpdateCollection("c1"); + reader.forceUpdateCollection("c2"); ClusterState clusterState = reader.getClusterState(); // keep a reference to the current cluster state object assertTrue(clusterState.hasCollection("c1")); assertFalse(clusterState.hasCollection("c2")); @@ -257,7 +258,6 @@ public void testExternalModificationToSharedClusterState() throws Exception { // expected } - reader.updateClusterState(); try { writer.enqueueUpdate(reader.getClusterState(), c2, null); fail("enqueueUpdate after BadVersionException should not have suceeded"); @@ -317,7 +317,7 @@ public void testExternalModificationToStateFormat2() throws Exception { zkClient.setData(ZkStateReader.getCollectionPath("c2"), data, true); // get the most up-to-date state - reader.updateClusterState(); + reader.forceUpdateCollection("c2"); state = reader.getClusterState(); assertTrue(state.hasCollection("c2")); assertEquals(sharedClusterStateVersion, (int) state.getZkClusterStateVersion()); @@ -328,7 +328,7 @@ public void testExternalModificationToStateFormat2() throws Exception { assertTrue(writer.hasPendingUpdates()); // get the most up-to-date state - reader.updateClusterState(); + reader.forceUpdateCollection("c2"); state = reader.getClusterState(); // enqueue a stateFormat=1 collection which should cause a flush @@ -336,7 +336,7 @@ public void testExternalModificationToStateFormat2() throws Exception { new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); try { - state = writer.enqueueUpdate(state, c1, null); + writer.enqueueUpdate(state, c1, null); fail("Enqueue should not have succeeded"); } catch (KeeperException.BadVersionException bve) { // expected diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 3dbc6d2876d0..308b3e000a54 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -226,7 +226,10 @@ public ZkConfigManager getConfigManager() { /** * Forcibly refresh cluster state from ZK. Do this only to avoid race conditions because it's expensive. + * + * @deprecated Don't call this, call {@link #forceUpdateCollection(String)} on a single collection if you must. */ + @Deprecated public void updateClusterState() throws KeeperException, InterruptedException { synchronized (getUpdateLock()) { if (clusterState == null) { @@ -248,6 +251,49 @@ public void updateClusterState() throws KeeperException, InterruptedException { } } + /** + * Forcibly refresh a collection's internal state from ZK. Try to avoid having to resort to this when + * a better design is possible. + */ + public void forceUpdateCollection(String collection) throws KeeperException, InterruptedException { + synchronized (getUpdateLock()) { + if (clusterState == null) { + return; + } + + ClusterState.CollectionRef ref = clusterState.getCollectionRef(collection); + if (ref == null) { + // We don't know anything about this collection, maybe it's new? + // First try to update the legacy cluster state. + refreshLegacyClusterState(null); + if (!legacyCollectionStates.containsKey(collection)) { + // No dice, see if a new collection just got created. + LazyCollectionRef tryLazyCollection = new LazyCollectionRef(collection); + if (tryLazyCollection.get() == null) { + // No dice, just give up. + return; + } + // What do you know, it exists! + lazyCollectionStates.putIfAbsent(collection, tryLazyCollection); + } + } else if (ref.isLazilyLoaded()) { + if (ref.get() != null) { + return; + } + // Edge case: if there's no external collection, try refreshing legacy cluster state in case it's there. + refreshLegacyClusterState(null); + } else if (legacyCollectionStates.containsKey(collection)) { + // Exists, and lives in legacy cluster state, force a refresh. + refreshLegacyClusterState(null); + } else if (watchedCollectionStates.containsKey(collection)) { + // Exists as a watched collection, force a refresh. + DocCollection newState = fetchCollectionState(collection, null); + updateWatchedCollection(collection, newState); + } + constructState(); + } + } + /** Refresh the set of live nodes. */ public void updateLiveNodes() throws KeeperException, InterruptedException { refreshLiveNodes(null); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java index ff4238260803..7b3617ba86c2 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java @@ -145,7 +145,6 @@ public static void waitForRecoveriesToFinish(String collection, while (cont) { if (verbose) System.out.println("-"); boolean sawLiveRecovering = false; - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); Map slices = clusterState.getSlicesMap(collection); assertNotNull("Could not find collection:" + collection, slices); @@ -195,7 +194,6 @@ public static void waitForCollectionToDisappear(String collection, while (cont) { if (verbose) System.out.println("-"); - zkStateReader.updateClusterState(); ClusterState clusterState = zkStateReader.getClusterState(); if (!clusterState.hasCollection(collection)) break; if (cnt == timeoutSeconds) { @@ -239,7 +237,7 @@ public static void verifyReplicaStatus(ZkStateReader reader, String collection, protected void assertAllActive(String collection,ZkStateReader zkStateReader) throws KeeperException, InterruptedException { - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collection); ClusterState clusterState = zkStateReader.getClusterState(); Map slices = clusterState.getSlicesMap(collection); if (slices == null) { diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index bf8f643656ba..a584dbd450bc 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -626,7 +626,7 @@ protected void updateMappingsFromZk(List jettys, List jettys, List clients, boolean allowOverSharding) throws Exception { ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION); cloudJettys.clear(); shardToJetty.clear(); @@ -1814,7 +1814,7 @@ protected List ensureAllReplicasAreActive(String testCollectionName, St Map notLeaders = new HashMap<>(); ZkStateReader zkr = cloudClient.getZkStateReader(); - zkr.updateClusterState(); // force the state to be fresh + zkr.forceUpdateCollection(testCollectionName); // force the state to be fresh ClusterState cs = zkr.getClusterState(); Collection slices = cs.getActiveSlices(testCollectionName); @@ -1824,10 +1824,6 @@ protected List ensureAllReplicasAreActive(String testCollectionName, St long maxWaitMs = maxWaitSecs * 1000L; Replica leader = null; while (waitMs < maxWaitMs && !allReplicasUp) { - // refresh state every 2 secs - if (waitMs % 2000 == 0) - cloudClient.getZkStateReader().updateClusterState(); - cs = cloudClient.getZkStateReader().getClusterState(); assertNotNull(cs); Slice shard = cs.getSlice(testCollectionName, shardId); @@ -1879,7 +1875,7 @@ protected String printClusterStateInfo() throws Exception { } protected String printClusterStateInfo(String collection) throws Exception { - cloudClient.getZkStateReader().updateClusterState(); + cloudClient.getZkStateReader().forceUpdateCollection(collection); String cs = null; ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); if (collection != null) { diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java index d13d62f06835..511fdf34b157 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java @@ -425,7 +425,7 @@ private int checkIfKillIsLegal(String slice, int numActive) throws KeeperExcepti for (CloudJettyRunner cloudJetty : shardToJetty.get(slice)) { // get latest cloud state - zkStateReader.updateClusterState(); + zkStateReader.forceUpdateCollection(collection); Slice theShards = zkStateReader.getClusterState().getSlicesMap(collection) .get(slice); @@ -447,18 +447,6 @@ private int checkIfKillIsLegal(String slice, int numActive) throws KeeperExcepti return numActive; } - public SolrClient getRandomClient(String slice) throws KeeperException, InterruptedException { - // get latest cloud state - zkStateReader.updateClusterState(); - - // get random shard - List clients = shardToClient.get(slice); - int index = LuceneTestCase.random().nextInt(clients.size() - 1); - SolrClient client = clients.get(index); - - return client; - } - // synchronously starts and stops shards randomly, unless there is only one // active shard up for a slice or if there is one active and others recovering public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) { From 5429356fc4ba021d2afe7766b832ca0dc93a0d57 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 7 Mar 2016 08:38:38 -0500 Subject: [PATCH 0049/1113] make test less evil --- .../src/test/org/apache/lucene/search/TestPointQueries.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index 5a3483bc30fc..500bb8fd25f4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -351,12 +351,12 @@ public void testRandomLongsMedium() throws Exception { @Nightly public void testRandomLongsBig() throws Exception { - doTestRandomLongs(200000); + doTestRandomLongs(100000); } private void doTestRandomLongs(int count) throws Exception { - int numValues = atLeast(count); + int numValues = TestUtil.nextInt(random(), count, count*2); if (VERBOSE) { System.out.println("TEST: numValues=" + numValues); From 5146e78a6445c1d1578154678518c8c904bf21e0 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Mon, 7 Mar 2016 22:44:36 +0530 Subject: [PATCH 0050/1113] SOLR-8736: schema GET operations on fields, dynamicFields, fieldTypes, copyField are reimplemented as a part of the bulk API with less details. The tests and write implementations are removed --- solr/CHANGES.txt | 5 + .../apache/solr/handler/SchemaHandler.java | 78 +++- .../apache/solr/rest/SolrSchemaRestApi.java | 56 +-- .../solr/rest/schema/BaseFieldResource.java | 146 ------- .../rest/schema/BaseFieldTypeResource.java | 98 ----- .../schema/CopyFieldCollectionResource.java | 198 ---------- .../DynamicFieldCollectionResource.java | 207 ---------- .../rest/schema/DynamicFieldResource.java | 197 ---------- .../rest/schema/FieldCollectionResource.java | 225 ----------- .../solr/rest/schema/FieldResource.java | 201 ---------- .../schema/FieldTypeCollectionResource.java | 197 ---------- .../solr/rest/schema/FieldTypeResource.java | 203 ---------- .../org/apache/solr/schema/IndexSchema.java | 11 +- .../org/apache/solr/servlet/HttpSolrCall.java | 3 + .../rest/schema/TestClassNameShortening.java | 3 +- .../TestCopyFieldCollectionResource.java | 96 +---- .../TestDynamicFieldCollectionResource.java | 29 -- .../rest/schema/TestDynamicFieldResource.java | 7 - .../schema/TestFieldCollectionResource.java | 45 --- .../solr/rest/schema/TestFieldResource.java | 23 +- .../TestFieldTypeCollectionResource.java | 1 + .../rest/schema/TestFieldTypeResource.java | 17 +- ...TestManagedSchemaDynamicFieldResource.java | 366 ----------------- .../TestManagedSchemaFieldResource.java | 369 ------------------ .../TestManagedSchemaFieldTypeResource.java | 350 ----------------- .../TestRemoveLastDynamicCopyField.java | 80 ---- .../schema/TestSchemaSimilarityResource.java | 1 - .../TestManagedStopFilterFactory.java | 2 +- .../TestManagedSynonymFilterFactory.java | 6 +- .../TestCloudManagedSchemaConcurrent.java | 3 +- .../solr/client/solrj/request/SchemaTest.java | 4 - 31 files changed, 125 insertions(+), 3102 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java delete mode 100644 solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java delete mode 100644 solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaDynamicFieldResource.java delete mode 100644 solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldResource.java delete mode 100644 solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldTypeResource.java delete mode 100644 solr/core/src/test/org/apache/solr/rest/schema/TestRemoveLastDynamicCopyField.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 645533b97e3c..14195ab7b695 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -102,6 +102,8 @@ Upgrading from Solr 5.x * When requesting stats in date fields, "sum" is now a double value instead of a date. See SOLR-8671 +* SOLR-8736: The deprecated GET methods for schema are now accessible and implemented differently + Detailed Change List ---------------------- @@ -394,6 +396,9 @@ Other Changes * SOLR-8423: DeleteShard and DeleteReplica should cleanup instance and data directory by default and add support for optionally retaining the directories. (Anshum Gupta) +* SOLR-8736: schema GET operations on fields, dynamicFields, fieldTypes, copyField are + reimplemented as a part of the bulk API with less details (noble) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index 046de46787ed..4279864b6ba6 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -20,15 +20,20 @@ import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Set; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.solr.cloud.ZkSolrResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; @@ -42,17 +47,29 @@ import org.slf4j.LoggerFactory; import static org.apache.solr.common.params.CommonParams.JSON; -import static org.apache.solr.core.ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG; public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private boolean isImmutableConfigSet = false; - @Override - public void init(NamedList args) { - super.init(args); + private static final Map level2; + + static { + Set s = ImmutableSet.of( + IndexSchema.FIELD_TYPES, + IndexSchema.FIELDS, + IndexSchema.DYNAMIC_FIELDS, + IndexSchema.COPY_FIELDS + ); + Map m = new HashMap<>(); + for (String s1 : s) { + m.put(s1, s1); + m.put(s1.toLowerCase(Locale.ROOT), s1); + } + level2 = ImmutableMap.copyOf(m); } + @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { SolrConfigHandler.setWt(req, JSON); @@ -150,6 +167,33 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { break; } default: { + List parts = StrUtils.splitSmart(path, '/'); + if (parts.get(0).isEmpty()) parts.remove(0); + if (parts.size() > 1 && level2.containsKey(parts.get(1))) { + String realName = level2.get(parts.get(1)); + SimpleOrderedMap propertyValues = req.getSchema().getNamedPropertyValues(req.getParams()); + Object o = propertyValues.get(realName); + if(parts.size()> 2) { + String name = parts.get(2); + if (o instanceof List) { + List list = (List) o; + for (Object obj : list) { + if (obj instanceof SimpleOrderedMap) { + SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj; + if(name.equals(simpleOrderedMap.get("name"))) { + rsp.add(realName.substring(0, realName.length() - 1), simpleOrderedMap); + return; + } + } + } + } + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such path " + path); + } else { + rsp.add(realName, o); + } + return; + } + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such path " + path); } } @@ -160,19 +204,25 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { } private static Set subPaths = new HashSet<>(Arrays.asList( - "/version", - "/uniquekey", - "/name", - "/similarity", - "/defaultsearchfield", - "/solrqueryparser", - "/zkversion", - "/solrqueryparser/defaultoperator" + "version", + "uniquekey", + "name", + "similarity", + "defaultsearchfield", + "solrqueryparser", + "zkversion" )); + static { + subPaths.addAll(level2.keySet()); + } @Override public SolrRequestHandler getSubHandler(String subPath) { - if (subPaths.contains(subPath)) return this; + List parts = StrUtils.splitSmart(subPath, '/'); + if (parts.get(0).isEmpty()) parts.remove(0); + String prefix = parts.get(0); + if(subPaths.contains(prefix)) return this; + return null; } diff --git a/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java b/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java index 0e40f73c8b2d..1310198f8ca2 100644 --- a/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java +++ b/solr/core/src/java/org/apache/solr/rest/SolrSchemaRestApi.java @@ -15,14 +15,14 @@ * limitations under the License. */ package org.apache.solr.rest; + +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.rest.schema.CopyFieldCollectionResource; -import org.apache.solr.rest.schema.DynamicFieldCollectionResource; -import org.apache.solr.rest.schema.DynamicFieldResource; -import org.apache.solr.rest.schema.FieldCollectionResource; -import org.apache.solr.rest.schema.FieldResource; -import org.apache.solr.rest.schema.FieldTypeCollectionResource; -import org.apache.solr.rest.schema.FieldTypeResource; import org.apache.solr.schema.IndexSchema; import org.restlet.Application; import org.restlet.Restlet; @@ -30,39 +30,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - /** * Restlet servlet handling /<context>/<collection>/schema/* URL paths */ public class SolrSchemaRestApi extends Application { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String FIELDS_PATH = "/" + IndexSchema.FIELDS; - - public static final String DYNAMIC_FIELDS = IndexSchema.DYNAMIC_FIELDS.toLowerCase(Locale.ROOT); - public static final String DYNAMIC_FIELDS_PATH = "/" + DYNAMIC_FIELDS; - - public static final String FIELDTYPES = IndexSchema.FIELD_TYPES.toLowerCase(Locale.ROOT); - public static final String FIELDTYPES_PATH = "/" + FIELDTYPES; - public static final String NAME_SEGMENT = "/{" + IndexSchema.NAME.toLowerCase(Locale.ROOT) + "}"; - - public static final String COPY_FIELDS = IndexSchema.COPY_FIELDS.toLowerCase(Locale.ROOT); - public static final String COPY_FIELDS_PATH = "/" + COPY_FIELDS; - + /** * Returns reserved endpoints under /schema */ public static Set getReservedEndpoints() { Set reservedEndpoints = new HashSet<>(); - reservedEndpoints.add(RestManager.SCHEMA_BASE_PATH + FIELDS_PATH); - reservedEndpoints.add(RestManager.SCHEMA_BASE_PATH + DYNAMIC_FIELDS_PATH); - reservedEndpoints.add(RestManager.SCHEMA_BASE_PATH + FIELDTYPES_PATH); - reservedEndpoints.add(RestManager.SCHEMA_BASE_PATH + COPY_FIELDS_PATH); return Collections.unmodifiableSet(reservedEndpoints); } @@ -88,25 +67,6 @@ public synchronized Restlet createInboundRoot() { log.info("createInboundRoot started for /schema"); - router.attach(FIELDS_PATH, FieldCollectionResource.class); - // Allow a trailing slash on collection requests - router.attach(FIELDS_PATH + "/", FieldCollectionResource.class); - router.attach(FIELDS_PATH + NAME_SEGMENT, FieldResource.class); - - router.attach(DYNAMIC_FIELDS_PATH, DynamicFieldCollectionResource.class); - // Allow a trailing slash on collection requests - router.attach(DYNAMIC_FIELDS_PATH + "/", DynamicFieldCollectionResource.class); - router.attach(DYNAMIC_FIELDS_PATH + NAME_SEGMENT, DynamicFieldResource.class); - - router.attach(FIELDTYPES_PATH, FieldTypeCollectionResource.class); - // Allow a trailing slash on collection requests - router.attach(FIELDTYPES_PATH + "/", FieldTypeCollectionResource.class); - router.attach(FIELDTYPES_PATH + NAME_SEGMENT, FieldTypeResource.class); - - router.attach(COPY_FIELDS_PATH, CopyFieldCollectionResource.class); - // Allow a trailing slash on collection requests - router.attach(COPY_FIELDS_PATH + "/", CopyFieldCollectionResource.class); - router.attachDefault(RestManager.ManagedEndpoint.class); // attach all the dynamically registered schema resources diff --git a/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java b/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java deleted file mode 100644 index 25f631d5e89e..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.rest.BaseSolrResource; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.restlet.resource.ResourceException; - -import java.util.LinkedHashSet; -import java.util.Map; - - -/** - * Base class for Schema Field and DynamicField requests. - */ -abstract class BaseFieldResource extends BaseSolrResource { - protected static final String INCLUDE_DYNAMIC_PARAM = "includeDynamic"; - private static final String DYNAMIC_BASE = "dynamicBase"; - - private LinkedHashSet requestedFields; - private boolean showDefaults; - - protected LinkedHashSet getRequestedFields() { - return requestedFields; - } - - - protected BaseFieldResource() { - super(); - } - - /** - * Pulls the "fl" param from the request and splits it to get the - * requested list of fields. The (Dynamic)FieldCollectionResource classes - * will then restrict the fields sent back in the response to those - * on this list. The (Dynamic)FieldResource classes ignore this list, - * since the (dynamic) field is specified in the URL path, rather than - * in a query parameter. - *

    - * Also pulls the "showDefaults" param from the request, for use by all - * subclasses to include default values from the associated field type - * in the response. By default this param is off. - */ - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - String flParam = getSolrRequest().getParams().get(CommonParams.FL); - if (null != flParam) { - String[] fields = flParam.trim().split("[,\\s]+"); - if (fields.length > 0) { - requestedFields = new LinkedHashSet<>(); - for (String field : fields) { - if ( ! field.trim().isEmpty()) { - requestedFields.add(field.trim()); - } - } - } - } - showDefaults = getSolrRequest().getParams().getBool(SHOW_DEFAULTS, false); - } - } - - /** Get the properties for a given field. - * - * @param field not required to exist in the schema - */ - protected SimpleOrderedMap getFieldProperties(SchemaField field) { - if (null == field) { - return null; - } - SimpleOrderedMap properties = field.getNamedPropertyValues(showDefaults); - if ( ! getSchema().getFields().containsKey(field.getName())) { - String dynamicBase = getSchema().getDynamicPattern(field.getName()); - // Add dynamicBase property if it's different from the field name. - if ( ! field.getName().equals(dynamicBase)) { - properties.add(DYNAMIC_BASE, dynamicBase); - } - } - if (field == getSchema().getUniqueKeyField()) { - properties.add(IndexSchema.UNIQUE_KEY, true); - } - return properties; - } - - /** - * When running in cloud mode, waits for a schema update to be - * applied by all active replicas of the current collection. - */ - protected void waitForSchemaUpdateToPropagate(IndexSchema newSchema) { - // If using ZooKeeper and the client application has requested an update timeout, then block until all - // active replicas for this collection process the updated schema - if (getUpdateTimeoutSecs() > 0 && newSchema != null && - newSchema.getResourceLoader() instanceof ZkSolrResourceLoader) - { - CoreDescriptor cd = getSolrCore().getCoreDescriptor(); - String collection = cd.getCollectionName(); - if (collection != null) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) newSchema.getResourceLoader(); - ManagedIndexSchema.waitForSchemaZkVersionAgreement(collection, - cd.getCloudDescriptor().getCoreNodeName(), - ((ManagedIndexSchema) newSchema).getSchemaZkVersion(), - zkLoader.getZkController(), - getUpdateTimeoutSecs()); - } - } - } - - // protected access on this class triggers a bug in javadoc generation caught by - // documentation-link: "BROKEN LINK" reported in javadoc for classes using - // NewFieldArguments because the link target file is BaseFieldResource.NewFieldArguments, - // but the actual file is BaseFieldResource$NewFieldArguments. - static class NewFieldArguments { - private String name; - private String type; - Map map; - NewFieldArguments(String name, String type, Map map) { - this.name = name; - this.type = type; - this.map = map; - } - - public String getName() { return name; } - public String getType() { return type; } - public Map getMap() { return map; } - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java b/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java deleted file mode 100644 index c475dd0c3238..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/BaseFieldTypeResource.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; - -import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.rest.BaseSolrResource; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.restlet.resource.ResourceException; - -import java.util.List; - -/** - * Base class for the FieldType resource classes. - */ -abstract class BaseFieldTypeResource extends BaseSolrResource { - private boolean showDefaults; - - protected BaseFieldTypeResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - showDefaults = getSolrRequest().getParams().getBool(SHOW_DEFAULTS, false); - } - } - - /** Used by subclasses to collect field type properties */ - protected SimpleOrderedMap getFieldTypeProperties(FieldType fieldType) { - SimpleOrderedMap properties = fieldType.getNamedPropertyValues(showDefaults); - properties.add(IndexSchema.FIELDS, getFieldsWithFieldType(fieldType)); - properties.add(IndexSchema.DYNAMIC_FIELDS, getDynamicFieldsWithFieldType(fieldType)); - return properties; - } - - - /** Return a list of names of Fields that have the given FieldType */ - protected abstract List getFieldsWithFieldType(FieldType fieldType); - - /** Return a list of names of DynamicFields that have the given FieldType */ - protected abstract List getDynamicFieldsWithFieldType(FieldType fieldType); - - /** - * Adds one or more new FieldType definitions to the managed schema for the given core. - */ - protected void addNewFieldTypes(List newFieldTypes, ManagedIndexSchema oldSchema) { - IndexSchema newSchema = null; - boolean success = false; - while (!success) { - try { - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addFieldTypes(newFieldTypes, true); - getSolrCore().setLatestSchema(newSchema); - success = true; - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema(); - } - } - - // If using ZooKeeper and the client application has requested an update timeout, then block until all - // active replicas for this collection process the updated schema - if (getUpdateTimeoutSecs() > 0 && newSchema != null && - newSchema.getResourceLoader() instanceof ZkSolrResourceLoader) - { - CoreDescriptor cd = getSolrCore().getCoreDescriptor(); - String collection = cd.getCollectionName(); - if (collection != null) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) newSchema.getResourceLoader(); - ManagedIndexSchema.waitForSchemaZkVersionAgreement(collection, - cd.getCloudDescriptor().getCoreNodeName(), - ((ManagedIndexSchema) newSchema).getSchemaZkVersion(), - zkLoader.getZkController(), - getUpdateTimeoutSecs()); - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java deleted file mode 100644 index 610c0542af92..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/CopyFieldCollectionResource.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.POSTable; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.solr.common.SolrException.ErrorCode; - -/** - * This class responds to requests at /solr/(corename)/schema/copyfields - *

    - * - * To restrict the set of copyFields in the response, specify one or both - * of the following as query parameters, with values as space and/or comma - * separated dynamic or explicit field names: - * - *

      - *
    • dest.fl: include copyFields that have one of these as a destination
      • - *
    • source.fl: include copyFields that have one of these as a source
      • - *
    - * - * If both dest.fl and source.fl are given as query parameters, the copyfields - * in the response will be restricted to those that match any of the destinations - * in dest.fl and also match any of the sources in source.fl. - */ -public class CopyFieldCollectionResource extends BaseFieldResource implements GETable, POSTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String SOURCE_FIELD_LIST = IndexSchema.SOURCE + "." + CommonParams.FL; - private static final String DESTINATION_FIELD_LIST = IndexSchema.DESTINATION + "." + CommonParams.FL; - - private Set requestedSourceFields; - private Set requestedDestinationFields; - - public CopyFieldCollectionResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - String sourceFieldListParam = getSolrRequest().getParams().get(SOURCE_FIELD_LIST); - if (null != sourceFieldListParam) { - String[] fields = sourceFieldListParam.trim().split("[,\\s]+"); - if (fields.length > 0) { - requestedSourceFields = new HashSet<>(Arrays.asList(fields)); - requestedSourceFields.remove(""); // Remove empty values, if any - } - } - String destinationFieldListParam = getSolrRequest().getParams().get(DESTINATION_FIELD_LIST); - if (null != destinationFieldListParam) { - String[] fields = destinationFieldListParam.trim().split("[,\\s]+"); - if (fields.length > 0) { - requestedDestinationFields = new HashSet<>(Arrays.asList(fields)); - requestedDestinationFields.remove(""); // Remove empty values, if any - } - } - } - } - - @Override - public Representation get() { - try { - getSolrResponse().add(IndexSchema.COPY_FIELDS, - getSchema().getCopyFieldProperties(true, requestedSourceFields, requestedDestinationFields)); - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - @Override - public Representation post(Representation entity) throws ResourceException { - try { - if (!getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Object object = ObjectBuilder.fromJSON(entity.getText()); - - if (!(object instanceof List)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form" - + " (ignore the backslashes): [{\"source\":\"foo\",\"dest\":\"comma-separated list of targets\"}, {...}, ...]"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - List> list = (List>) object; - Map> fieldsToCopy = new HashMap<>(); - ManagedIndexSchema oldSchema = (ManagedIndexSchema) getSchema(); - Set malformed = new HashSet<>(); - for (Map map : list) { - String fieldName = (String)map.get(IndexSchema.SOURCE); - if (null == fieldName) { - String message = "Missing '" + IndexSchema.SOURCE + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - Object dest = map.get(IndexSchema.DESTINATION); - List destinations = null; - if (dest != null) { - if (dest instanceof List){ - destinations = (List)dest; - } else if (dest instanceof String){ - destinations = Collections.singletonList(dest.toString()); - } else { - String message = "Invalid '" + IndexSchema.DESTINATION + "' type."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - if (destinations == null) { - malformed.add(fieldName); - } else { - fieldsToCopy.put(fieldName, destinations); - } - } - if (malformed.size() > 0){ - StringBuilder message = new StringBuilder("Malformed destination(s) for: "); - for (String s : malformed) { - message.append(s).append(", "); - } - if (message.length() > 2) { - message.setLength(message.length() - 2);//drop the last , - } - log.error(message.toString().trim()); - throw new SolrException(ErrorCode.BAD_REQUEST, message.toString().trim()); - } - IndexSchema newSchema = null; - boolean success = false; - while (!success) { - try { - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addCopyFields(fieldsToCopy,true); - if (null != newSchema) { - getSolrCore().setLatestSchema(newSchema); - success = true; - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add fields."); - } - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - log.debug("Schema changed while processing request, retrying"); - oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema(); - } - } - waitForSchemaUpdateToPropagate(newSchema); - } - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - return new SolrOutputRepresentation(); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java deleted file mode 100644 index bf9423404b78..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.POSTable; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * This class responds to requests at /solr/(corename)/schema/dynamicfields - *

    - * To restrict the set of dynamic fields in the response, specify a comma - * and/or space separated list of dynamic field patterns in the "fl" query - * parameter. - */ -public class DynamicFieldCollectionResource extends BaseFieldResource implements GETable, POSTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public DynamicFieldCollectionResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - } - - @Override - public Representation get() { - - try { - List> props = new ArrayList<>(); - if (null == getRequestedFields()) { - for (IndexSchema.DynamicField dynamicField : getSchema().getDynamicFields()) { - if ( ! dynamicField.getRegex().startsWith(IndexSchema.INTERNAL_POLY_FIELD_PREFIX)) { // omit internal polyfields - props.add(getFieldProperties(dynamicField.getPrototype())); - } - } - } else { - if (0 == getRequestedFields().size()) { - String message = "Empty " + CommonParams.FL + " parameter value"; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - Map dynamicFieldsByName = new HashMap<>(); - for (IndexSchema.DynamicField dynamicField : getSchema().getDynamicFields()) { - dynamicFieldsByName.put(dynamicField.getRegex(), dynamicField.getPrototype()); - } - // Use the same order as the fl parameter - for (String dynamicFieldName : getRequestedFields()) { - final SchemaField dynamicSchemaField = dynamicFieldsByName.get(dynamicFieldName); - if (null == dynamicSchemaField) { - log.info("Requested dynamic field '" + dynamicFieldName + "' not found."); - } else { - props.add(getFieldProperties(dynamicSchemaField)); - } - } - } - getSolrResponse().add(IndexSchema.DYNAMIC_FIELDS, props); - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - @Override - public Representation post(Representation entity) { - try { - if ( ! getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - if (null == entity.getMediaType()) { - entity.setMediaType(MediaType.APPLICATION_JSON); - } - if ( ! entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Object object = ObjectBuilder.fromJSON(entity.getText()); - if ( ! (object instanceof List)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form" - + " (ignore the backslashes): [{\"name\":\"*_foo\",\"type\":\"text_general\", ...}, {...}, ...]"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - List> list = (List>)object; - List newDynamicFields = new ArrayList<>(); - List newDynamicFieldArguments = new ArrayList<>(); - ManagedIndexSchema oldSchema = (ManagedIndexSchema)getSchema(); - Map> copyFields = new HashMap<>(); - for (Map map : list) { - String fieldNamePattern = (String)map.remove(IndexSchema.NAME); - if (null == fieldNamePattern) { - String message = "Missing '" + IndexSchema.NAME + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - String fieldType = (String)map.remove(IndexSchema.TYPE); - if (null == fieldType) { - String message = "Missing '" + IndexSchema.TYPE + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - // copyFields:"comma separated list of destination fields" - Object copies = map.get(IndexSchema.COPY_FIELDS); - List copyTo = null; - if (copies != null) { - if (copies instanceof List){ - copyTo = (List)copies; - } else if (copies instanceof String){ - copyTo = Collections.singletonList(copies.toString()); - } else { - String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - if (copyTo != null) { - map.remove(IndexSchema.COPY_FIELDS); - copyFields.put(fieldNamePattern, copyTo); - } - newDynamicFields.add(oldSchema.newDynamicField(fieldNamePattern, fieldType, map)); - newDynamicFieldArguments.add(new NewFieldArguments(fieldNamePattern, fieldType, map)); - } - IndexSchema newSchema = null; - boolean firstAttempt = true; - boolean success = false; - while ( ! success) { - try { - if ( ! firstAttempt) { - // If this isn't the first attempt, we must have failed due to - // the schema changing in Zk during optimistic concurrency control. - // In that case, rerun creating the new fields, because they may - // fail now due to changes in the schema. This behavior is consistent - // with what would happen if we locked the schema and the other schema - // change went first. - newDynamicFields.clear(); - for (NewFieldArguments args : newDynamicFieldArguments) { - newDynamicFields.add(oldSchema.newDynamicField(args.getName(), args.getType(), args.getMap())); - } - } - firstAttempt = false; - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addDynamicFields(newDynamicFields, copyFields, true); - if (null != newSchema) { - getSolrCore().setLatestSchema(newSchema); - success = true; - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add dynamic fields."); - } - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - log.debug("Schema changed while processing request, retrying"); - oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema(); - } - } - - waitForSchemaUpdateToPropagate(newSchema); - - } - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java b/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java deleted file mode 100644 index bf67608cc496..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.PUTable; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; - -/** - * This class responds to requests at /solr/(corename)/schema/dynamicfields/(pattern) - * where pattern is a field name pattern (with an asterisk at the beginning or the end). - */ -public class DynamicFieldResource extends BaseFieldResource implements GETable, PUTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private String fieldNamePattern; - - public DynamicFieldResource() { - super(); - } - - /** - * Gets the field name pattern from the request attribute where it's stored by Restlet. - */ - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - fieldNamePattern = (String)getRequestAttributes().get(IndexSchema.NAME); - try { - fieldNamePattern = null == fieldNamePattern ? "" : urlDecode(fieldNamePattern.trim()).trim(); - } catch (UnsupportedEncodingException e) { - throw new ResourceException(e); - } - } - } - - @Override - public Representation get() { - try { - if (fieldNamePattern.isEmpty()) { - final String message = "Dynamic field name is missing"; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - SchemaField field = null; - for (SchemaField prototype : getSchema().getDynamicFieldPrototypes()) { - if (prototype.getName().equals(fieldNamePattern)) { - field = prototype; - break; - } - } - if (null == field) { - final String message = "Dynamic field '" + fieldNamePattern + "' not found."; - throw new SolrException(ErrorCode.NOT_FOUND, message); - } else { - getSolrResponse().add(IndexSchema.DYNAMIC_FIELD, getFieldProperties(field)); - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - /** - * Accepts JSON add dynamic field request - */ - @Override - public Representation put(Representation entity) { - try { - if ( ! getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - if (null == entity.getMediaType()) { - entity.setMediaType(MediaType.APPLICATION_JSON); - } - if ( ! entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Object object = ObjectBuilder.fromJSON(entity.getText()); - if ( ! (object instanceof Map)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected Map of the form" - + " (ignore the backslashes): {\"type\":\"text_general\", ...}, either with or" - + " without a \"name\" mapping. If the \"name\" is specified, it must match the" - + " name given in the request URL: /schema/dynamicfields/(name)"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Map map = (Map)object; - if (1 == map.size() && map.containsKey(IndexSchema.DYNAMIC_FIELD)) { - map = (Map)map.get(IndexSchema.DYNAMIC_FIELD); - } - String bodyFieldName; - if (null != (bodyFieldName = (String)map.remove(IndexSchema.NAME)) - && ! fieldNamePattern.equals(bodyFieldName)) { - String message = "Dynamic field name in the request body '" + bodyFieldName - + "' doesn't match dynamic field name in the request URL '" + fieldNamePattern + "'"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - String fieldType; - if (null == (fieldType = (String) map.remove(IndexSchema.TYPE))) { - String message = "Missing '" + IndexSchema.TYPE + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - ManagedIndexSchema oldSchema = (ManagedIndexSchema)getSchema(); - Object copies = map.get(IndexSchema.COPY_FIELDS); - Collection copyFieldNames = null; - if (copies != null) { - if (copies instanceof List) { - copyFieldNames = (List)copies; - } else if (copies instanceof String) { - copyFieldNames = singletonList(copies.toString()); - } else { - String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - if (copyFieldNames != null) { - map.remove(IndexSchema.COPY_FIELDS); - } - IndexSchema newSchema = null; - boolean success = false; - while ( ! success) { - try { - SchemaField newDynamicField = oldSchema.newDynamicField(fieldNamePattern, fieldType, map); - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addDynamicFields(singletonList(newDynamicField), singletonMap(newDynamicField.getName(), copyFieldNames), true); - if (null != newSchema) { - getSolrCore().setLatestSchema(newSchema); - success = true; - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add dynamic field."); - } - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - log.debug("Schema changed while processing request, retrying"); - oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema(); - } - } - // if in cloud mode, wait for schema updates to propagate to all replicas - waitForSchemaUpdateToPropagate(newSchema); - } - } - } - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java deleted file mode 100644 index f1bf6a4d3729..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.POSTable; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * This class responds to requests at /solr/(corename)/schema/fields - *

    - * Two query parameters are supported: - *

      - *
    • - * "fl": a comma- and/or space-separated list of fields to send properties - * for in the response, rather than the default: all of them. - *
      • - *
    • - * "includeDynamic": if the "fl" parameter is specified, matching dynamic - * fields are included in the response and identified with the "dynamicBase" - * property. If the "fl" parameter is not specified, the "includeDynamic" - * query parameter is ignored. - *
      • - *
    - */ -public class FieldCollectionResource extends BaseFieldResource implements GETable, POSTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private boolean includeDynamic; - - public FieldCollectionResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - includeDynamic = getSolrRequest().getParams().getBool(INCLUDE_DYNAMIC_PARAM, false); - } - } - - @Override - public Representation get() { - try { - final List> props = new ArrayList<>(); - if (null == getRequestedFields()) { - SortedSet fieldNames = new TreeSet<>(getSchema().getFields().keySet()); - for (String fieldName : fieldNames) { - props.add(getFieldProperties(getSchema().getFields().get(fieldName))); - } - } else { - if (0 == getRequestedFields().size()) { - String message = "Empty " + CommonParams.FL + " parameter value"; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - // Use the same order as the fl parameter - for (String fieldName : getRequestedFields()) { - final SchemaField field; - if (includeDynamic) { - field = getSchema().getFieldOrNull(fieldName); - } else { - field = getSchema().getFields().get(fieldName); - } - if (null == field) { - log.info("Requested field '" + fieldName + "' not found."); - } else { - props.add(getFieldProperties(field)); - } - } - } - getSolrResponse().add(IndexSchema.FIELDS, props); - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - @Override - public Representation post(Representation entity) { - try { - if (!getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - if (null == entity.getMediaType()) { - entity.setMediaType(MediaType.APPLICATION_JSON); - } - if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Object object = ObjectBuilder.fromJSON(entity.getText()); - if (!(object instanceof List)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form" - + " (ignore the backslashes): [{\"name\":\"foo\",\"type\":\"text_general\", ...}, {...}, ...]"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - List> list = (List>) object; - List newFields = new ArrayList<>(); - List newFieldArguments = new ArrayList<>(); - IndexSchema oldSchema = getSchema(); - Map> copyFields = new HashMap<>(); - for (Map map : list) { - String fieldName = (String) map.remove(IndexSchema.NAME); - if (null == fieldName) { - String message = "Missing '" + IndexSchema.NAME + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - String fieldType = (String) map.remove(IndexSchema.TYPE); - if (null == fieldType) { - String message = "Missing '" + IndexSchema.TYPE + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - // copyFields:"comma separated list of destination fields" - Object copies = map.get(IndexSchema.COPY_FIELDS); - List copyTo = null; - if (copies != null) { - if (copies instanceof List){ - copyTo = (List) copies; - } else if (copies instanceof String){ - copyTo = Collections.singletonList(copies.toString()); - } else { - String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - if (copyTo != null) { - map.remove(IndexSchema.COPY_FIELDS); - copyFields.put(fieldName, copyTo); - } - newFields.add(oldSchema.newField(fieldName, fieldType, map)); - newFieldArguments.add(new NewFieldArguments(fieldName, fieldType, map)); - } - IndexSchema newSchema = null; - boolean firstAttempt = true; - boolean success = false; - while (!success) { - try { - if (!firstAttempt) { - // If this isn't the first attempt, we must have failed due to - // the schema changing in Zk during optimistic concurrency control. - // In that case, rerun creating the new fields, because they may - // fail now due to changes in the schema. This behavior is consistent - // with what would happen if we locked the schema and the other schema - // change went first. - newFields.clear(); - for (NewFieldArguments args : newFieldArguments) { - newFields.add(oldSchema.newField( - args.getName(), args.getType(), args.getMap())); - } - } - firstAttempt = false; - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addFields(newFields, copyFields, true); - if (null != newSchema) { - getSolrCore().setLatestSchema(newSchema); - success = true; - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add fields."); - } - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - log.debug("Schema changed while processing request, retrying"); - oldSchema = getSolrCore().getLatestSchema(); - } - } - waitForSchemaUpdateToPropagate(newSchema); - } - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java b/solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java deleted file mode 100644 index 2634bbdb759e..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/FieldResource.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.cloud.ZkSolrResourceLoader; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.PUTable; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -/** - * This class responds to requests at /solr/(corename)/schema/fields/(fieldname) - * where "fieldname" is the name of a field. - *

    - * The GET method returns properties for the given fieldname. - * The "includeDynamic" query parameter, if specified, will cause the - * dynamic field matching the given fieldname to be returned if fieldname - * is not explicitly declared in the schema. - *

    - * The PUT method accepts field addition requests in JSON format. - */ -public class FieldResource extends BaseFieldResource implements GETable, PUTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private boolean includeDynamic; - private String fieldName; - - public FieldResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - includeDynamic = getSolrRequest().getParams().getBool(INCLUDE_DYNAMIC_PARAM, false); - fieldName = (String) getRequestAttributes().get(IndexSchema.NAME); - try { - fieldName = null == fieldName ? "" : urlDecode(fieldName.trim()).trim(); - } catch (UnsupportedEncodingException e) { - throw new ResourceException(e); - } - } - } - - @Override - public Representation get() { - try { - if (fieldName.isEmpty()) { - final String message = "Field name is missing"; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - final SchemaField field; - if (includeDynamic) { - field = getSchema().getFieldOrNull(fieldName); - } else { - field = getSchema().getFields().get(fieldName); - } - if (null == field) { - final String message = "Field '" + fieldName + "' not found."; - throw new SolrException(ErrorCode.NOT_FOUND, message); - } else { - getSolrResponse().add(IndexSchema.FIELD, getFieldProperties(field)); - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - /** - * Accepts JSON add field request, to URL - */ - @Override - public Representation put(Representation entity) { - try { - if (!getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - if (null == entity.getMediaType()) { - entity.setMediaType(MediaType.APPLICATION_JSON); - } - if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Object object = ObjectBuilder.fromJSON(entity.getText()); - if (!(object instanceof Map)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected Map of the form" - + " (ignore the backslashes): {\"type\":\"text_general\", ...}, either with or" - + " without a \"name\" mapping. If the \"name\" is specified, it must match the" - + " name given in the request URL: /schema/fields/(name)"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - Map map = (Map) object; - if (1 == map.size() && map.containsKey(IndexSchema.FIELD)) { - map = (Map) map.get(IndexSchema.FIELD); - } - String bodyFieldName; - if (null != (bodyFieldName = (String) map.remove(IndexSchema.NAME)) && !fieldName.equals(bodyFieldName)) { - String message = "Field name in the request body '" + bodyFieldName - + "' doesn't match field name in the request URL '" + fieldName + "'"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - String fieldType; - if (null == (fieldType = (String) map.remove(IndexSchema.TYPE))) { - String message = "Missing '" + IndexSchema.TYPE + "' mapping."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - ManagedIndexSchema oldSchema = (ManagedIndexSchema) getSchema(); - Object copies = map.get(IndexSchema.COPY_FIELDS); - List copyFieldNames = null; - if (copies != null) { - if (copies instanceof List) { - copyFieldNames = (List) copies; - } else if (copies instanceof String) { - copyFieldNames = Collections.singletonList(copies.toString()); - } else { - String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - if (copyFieldNames != null) { - map.remove(IndexSchema.COPY_FIELDS); - } - - IndexSchema newSchema = null; - boolean success = false; - while (!success) { - try { - SchemaField newField = oldSchema.newField(fieldName, fieldType, map); - synchronized (oldSchema.getSchemaUpdateLock()) { - newSchema = oldSchema.addField(newField, copyFieldNames); - if (null != newSchema) { - getSolrCore().setLatestSchema(newSchema); - success = true; - } else { - throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add field."); - } - } - } catch (ManagedIndexSchema.SchemaChangedInZkException e) { - log.debug("Schema changed while processing request, retrying"); - oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema(); - } - } - waitForSchemaUpdateToPropagate(newSchema); - } - } - } - } - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java deleted file mode 100644 index d2eb1bd812d2..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeCollectionResource.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.POSTable; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -/** - * This class responds to requests at /solr/(corename)/schema/fieldtypes - * - * The GET method returns properties for all field types defined in the schema. - */ -public class FieldTypeCollectionResource extends BaseFieldTypeResource implements GETable, POSTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private Map> fieldsByFieldType; - private Map> dynamicFieldsByFieldType; - - public FieldTypeCollectionResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - fieldsByFieldType = getFieldsByFieldType(); - dynamicFieldsByFieldType = getDynamicFieldsByFieldType(); - } - } - - @Override - public Representation get() { - try { - List> props = new ArrayList<>(); - Map sortedFieldTypes = new TreeMap<>(getSchema().getFieldTypes()); - for (FieldType fieldType : sortedFieldTypes.values()) { - props.add(getFieldTypeProperties(fieldType)); - } - getSolrResponse().add(IndexSchema.FIELD_TYPES, props); - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - /** Returns field lists from the map constructed in doInit() */ - @Override - protected List getFieldsWithFieldType(FieldType fieldType) { - List fields = fieldsByFieldType.get(fieldType.getTypeName()); - if (null == fields) { - fields = Collections.emptyList(); - } - return fields; - } - - /** Returns dynamic field lists from the map constructed in doInit() */ - @Override - protected List getDynamicFieldsWithFieldType(FieldType fieldType) { - List dynamicFields = dynamicFieldsByFieldType.get(fieldType.getTypeName()); - if (null == dynamicFields) { - dynamicFields = Collections.emptyList(); - } - return dynamicFields; - } - - /** - * Returns a map from field type names to a sorted list of fields that use the field type. - * The map only includes field types that are used by at least one field. - */ - private Map> getFieldsByFieldType() { - Map> fieldsByFieldType = new HashMap<>(); - for (SchemaField schemaField : getSchema().getFields().values()) { - final String fieldType = schemaField.getType().getTypeName(); - List fields = fieldsByFieldType.get(fieldType); - if (null == fields) { - fields = new ArrayList<>(); - fieldsByFieldType.put(fieldType, fields); - } - fields.add(schemaField.getName()); - } - for (List fields : fieldsByFieldType.values()) { - Collections.sort(fields); - } - return fieldsByFieldType; - } - - /** - * Returns a map from field type names to a list of dynamic fields that use the field type. - * The map only includes field types that are used by at least one dynamic field. - */ - private Map> getDynamicFieldsByFieldType() { - Map> dynamicFieldsByFieldType = new HashMap<>(); - for (SchemaField schemaField : getSchema().getDynamicFieldPrototypes()) { - final String fieldType = schemaField.getType().getTypeName(); - List dynamicFields = dynamicFieldsByFieldType.get(fieldType); - if (null == dynamicFields) { - dynamicFields = new ArrayList<>(); - dynamicFieldsByFieldType.put(fieldType, dynamicFields); - } - dynamicFields.add(schemaField.getName()); - } - return dynamicFieldsByFieldType; - } - - @SuppressWarnings("unchecked") - @Override - public Representation post(Representation entity) { - try { - if (!getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - if (null == entity.getMediaType()) - entity.setMediaType(MediaType.APPLICATION_JSON); - - if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - Object object = ObjectBuilder.fromJSON(entity.getText()); - if (!(object instanceof List)) { - String message = "Invalid JSON type " + object.getClass().getName() - + ", expected List of field type definitions in the form of" - + " (ignore the backslashes): [{\"name\":\"text_general\",\"class\":\"solr.TextField\", ...}, {...}, ...]"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - List> fieldTypeList = (List>) object; - if (fieldTypeList.size() > 0) - addOrUpdateFieldTypes(fieldTypeList); - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - @SuppressWarnings("unchecked") - protected void addOrUpdateFieldTypes(List> fieldTypeList) throws Exception { - List newFieldTypes = new ArrayList<>(fieldTypeList.size()); - ManagedIndexSchema oldSchema = (ManagedIndexSchema) getSchema(); - for (Map fieldTypeJson : fieldTypeList) { - if (1 == fieldTypeJson.size() && fieldTypeJson.containsKey(IndexSchema.FIELD_TYPE)) { - fieldTypeJson = (Map) fieldTypeJson.get(IndexSchema.FIELD_TYPE); - } - FieldType newFieldType = - FieldTypeResource.buildFieldTypeFromJson(oldSchema, - (String)fieldTypeJson.get(IndexSchema.NAME), fieldTypeJson); - newFieldTypes.add(newFieldType); - } - // now deploy the added types (all or nothing) - addNewFieldTypes(newFieldTypes, oldSchema); - } -} diff --git a/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java b/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java deleted file mode 100644 index 361c8c216baa..000000000000 --- a/solr/core/src/java/org/apache/solr/rest/schema/FieldTypeResource.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.rest.GETable; -import org.apache.solr.rest.PUTable; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.ManagedIndexSchema; -import org.apache.solr.schema.SchemaField; -import org.noggit.ObjectBuilder; -import org.restlet.data.MediaType; -import org.restlet.representation.Representation; -import org.restlet.resource.ResourceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * This class responds to requests at /solr/(corename)/schema/fieldtype/(typename) - * where "typename" is the name of a field type in the schema. - * - * The GET method returns properties for the named field type. - */ -public class FieldTypeResource extends BaseFieldTypeResource implements GETable, PUTable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private String typeName; - - public FieldTypeResource() { - super(); - } - - @Override - public void doInit() throws ResourceException { - super.doInit(); - if (isExisting()) { - typeName = (String)getRequestAttributes().get(IndexSchema.NAME); - try { - typeName = null == typeName ? "" : urlDecode(typeName.trim()).trim(); - } catch (UnsupportedEncodingException e) { - throw new ResourceException(e); - } - } - } - - @Override - public Representation get() { - try { - if (typeName.isEmpty()) { - final String message = "Field type name is missing"; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } else { - FieldType fieldType = getSchema().getFieldTypes().get(typeName); - if (null == fieldType) { - final String message = "Field type '" + typeName + "' not found."; - throw new SolrException(ErrorCode.NOT_FOUND, message); - } - getSolrResponse().add(IndexSchema.FIELD_TYPE, getFieldTypeProperties(fieldType)); - } - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - /** - * Returns a field list using the given field type by iterating over all fields - * defined in the schema. - */ - @Override - protected List getFieldsWithFieldType(FieldType fieldType) { - List fields = new ArrayList<>(); - for (SchemaField schemaField : getSchema().getFields().values()) { - if (schemaField.getType().getTypeName().equals(fieldType.getTypeName())) { - fields.add(schemaField.getName()); - } - } - Collections.sort(fields); - return fields; - } - - /** - * Returns a dynamic field list using the given field type by iterating over all - * dynamic fields defined in the schema. - */ - @Override - protected List getDynamicFieldsWithFieldType(FieldType fieldType) { - List dynamicFields = new ArrayList<>(); - for (SchemaField prototype : getSchema().getDynamicFieldPrototypes()) { - if (prototype.getType().getTypeName().equals(fieldType.getTypeName())) { - dynamicFields.add(prototype.getName()); - } - } - return dynamicFields; // Don't sort these - they're matched in order - } - - /** - * Accepts JSON add fieldtype request, to URL - */ - @SuppressWarnings("unchecked") - @Override - public Representation put(Representation entity) { - try { - if (!getSchema().isMutable()) { - final String message = "This IndexSchema is not mutable."; - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - if (null == entity.getMediaType()) - entity.setMediaType(MediaType.APPLICATION_JSON); - - if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) { - String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted." - + " Request has media type " + entity.getMediaType().toString() + "."; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - Object object = ObjectBuilder.fromJSON(entity.getText()); - if (!(object instanceof Map)) { - String message = "Invalid JSON type " + object.getClass().getName() + ", expected Map of the form" - + " (ignore the backslashes): {\"name\":\"text_general\", \"class\":\"solr.TextField\" ...}," - + " either with or without a \"name\" mapping. If the \"name\" is specified, it must match the" - + " name given in the request URL: /schema/fieldtypes/(name)"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - // basic validation passed, let's try to create it! - addOrUpdateFieldType((Map)object); - - } catch (Exception e) { - getSolrResponse().setException(e); - } - handlePostExecution(log); - - return new SolrOutputRepresentation(); - } - - protected void addOrUpdateFieldType(Map fieldTypeJson) { - ManagedIndexSchema oldSchema = (ManagedIndexSchema) getSchema(); - FieldType newFieldType = buildFieldTypeFromJson(oldSchema, typeName, fieldTypeJson); - addNewFieldTypes(Collections.singletonList(newFieldType), oldSchema); - } - - /** - * Builds a FieldType definition from a JSON object. - */ - @SuppressWarnings("unchecked") - static FieldType buildFieldTypeFromJson(ManagedIndexSchema oldSchema, String fieldTypeName, Map fieldTypeJson) { - if (1 == fieldTypeJson.size() && fieldTypeJson.containsKey(IndexSchema.FIELD_TYPE)) { - fieldTypeJson = (Map)fieldTypeJson.get(IndexSchema.FIELD_TYPE); - } - - String bodyTypeName = (String) fieldTypeJson.get(IndexSchema.NAME); - if (bodyTypeName == null) { - // must provide the name in the JSON for converting to the XML format needed - // to create FieldType objects using the FieldTypePluginLoader - fieldTypeJson.put(IndexSchema.NAME, fieldTypeName); - } else { - // if they provide it in the JSON, then it must match the value from the path - if (!fieldTypeName.equals(bodyTypeName)) { - String message = "Field type name in the request body '" + bodyTypeName - + "' doesn't match field type name in the request URL '" + fieldTypeName + "'"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - } - - String className = (String)fieldTypeJson.get(FieldType.CLASS_NAME); - if (className == null) { - String message = "Missing required '" + FieldType.CLASS_NAME + "' property!"; - log.error(message); - throw new SolrException(ErrorCode.BAD_REQUEST, message); - } - - return oldSchema.newFieldType(fieldTypeName, className, fieldTypeJson); - } -} diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index acc8c1302714..4319c3ebba33 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -39,6 +39,7 @@ import org.apache.lucene.util.Version; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; @@ -1352,6 +1353,10 @@ public boolean isCopyFieldTarget( SchemaField f ) { * Get a map of property name -> value for the whole schema. */ public SimpleOrderedMap getNamedPropertyValues() { + return getNamedPropertyValues(new MapSolrParams(Collections.EMPTY_MAP)); + + } + public SimpleOrderedMap getNamedPropertyValues(SolrParams params) { SimpleOrderedMap topLevel = new SimpleOrderedMap<>(); topLevel.add(NAME, getSchemaName()); topLevel.add(VERSION, getVersion()); @@ -1372,19 +1377,19 @@ public SimpleOrderedMap getNamedPropertyValues() { List> fieldTypeProperties = new ArrayList<>(); SortedMap sortedFieldTypes = new TreeMap<>(fieldTypes); for (FieldType fieldType : sortedFieldTypes.values()) { - fieldTypeProperties.add(fieldType.getNamedPropertyValues(false)); + fieldTypeProperties.add(fieldType.getNamedPropertyValues(params.getBool("showDefaults", false))); } topLevel.add(FIELD_TYPES, fieldTypeProperties); List> fieldProperties = new ArrayList<>(); SortedSet fieldNames = new TreeSet<>(fields.keySet()); for (String fieldName : fieldNames) { - fieldProperties.add(fields.get(fieldName).getNamedPropertyValues(false)); + fieldProperties.add(fields.get(fieldName).getNamedPropertyValues(params.getBool("showDefaults", false))); } topLevel.add(FIELDS, fieldProperties); List> dynamicFieldProperties = new ArrayList<>(); for (IndexSchema.DynamicField dynamicField : dynamicFields) { if ( ! dynamicField.getRegex().startsWith(INTERNAL_POLY_FIELD_PREFIX)) { // omit internal polyfields - dynamicFieldProperties.add(dynamicField.getPrototype().getNamedPropertyValues(false)); + dynamicFieldProperties.add(dynamicField.getPrototype().getNamedPropertyValues(params.getBool("showDefaults", false))); } } topLevel.add(DYNAMIC_FIELDS, dynamicFieldProperties); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index d87eb690a110..f291b2f97862 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -35,10 +35,12 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.Set; +import com.google.common.collect.ImmutableSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; @@ -84,6 +86,7 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.QueryResponseWriterUtil; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.AuthorizationContext.CollectionRequest; diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestClassNameShortening.java b/solr/core/src/test/org/apache/solr/rest/schema/TestClassNameShortening.java index ddbd331ee876..52ba10b789fe 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestClassNameShortening.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestClassNameShortening.java @@ -18,13 +18,14 @@ import org.apache.solr.util.RestTestBase; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.restlet.ext.servlet.ServerServlet; import java.util.SortedMap; import java.util.TreeMap; - +@Ignore public class TestClassNameShortening extends RestTestBase { @BeforeClass diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java index 5eeee8c7a334..c0f936d929e5 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java @@ -30,73 +30,51 @@ public void testGetAllCopyFields() throws Exception { +" and int[@name='maxChars'][.='200']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + +" and str[@name='dest'][.='dest_sub_no_ast_s']]", - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" + "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" +" and str[@name='dest'][.='title']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" +" and str[@name='dest'][.='*_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" - +" and str[@name='dest'][.='*_dest_sub_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + +" and str[@name='dest'][.='*_dest_sub_s']]", - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" + +" and str[@name='dest'][.='dest_sub_no_ast_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_src_sub_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" +" and str[@name='dest'][.='title']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_src_sub_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" +" and str[@name='dest'][.='*_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_src_sub_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" - +" and str[@name='dest'][.='*_dest_sub_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + +" and str[@name='dest'][.='*_dest_sub_s']]", - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_src_sub_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_src_sub_i']" + +" and str[@name='dest'][.='dest_sub_no_ast_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" +" and str[@name='dest'][.='*_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" - +" and str[@name='dest'][.='*_dest_sub_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + +" and str[@name='dest'][.='*_dest_sub_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']" - +" and str[@name='sourceDynamicBase'][.='*_i']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']" - +" and str[@name='destDynamicBase'][.='*_s']]", + +" and str[@name='dest'][.='dest_sub_no_ast_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title_*']" - +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']" - +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']" +" and str[@name='dest'][.='text']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title_*']" - +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']" - +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']" +" and str[@name='dest'][.='*_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title_*']" - +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']" - +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']" +" and str[@name='dest'][.='*_dest_sub_s']]", "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title_*']" - +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']" - +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']" +" and str[@name='dest'][.='dest_sub_no_ast_s']]"); } @@ -104,56 +82,22 @@ public void testGetAllCopyFields() throws Exception { public void testJsonGetAllCopyFields() throws Exception { assertJQ("/schema/copyfields?indent=on&wt=json", "/copyFields/[1]=={'source':'src_sub_no_ast_i','dest':'title'}", - "/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}", + "/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s'}", "/copyFields/[8]=={'source':'*_i','dest':'title'}", "/copyFields/[9]=={'source':'*_i','dest':'*_s'}", - "/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}", - "/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}", + "/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s'}", + "/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s'}", - "/copyFields/[12]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'title'}", - "/copyFields/[13]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_s'}", - "/copyFields/[14]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}", - "/copyFields/[15]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}", + "/copyFields/[12]=={'source':'*_src_sub_i','dest':'title'}", + "/copyFields/[13]=={'source':'*_src_sub_i','dest':'*_s'}", + "/copyFields/[14]=={'source':'*_src_sub_i','dest':'*_dest_sub_s'}", + "/copyFields/[15]=={'source':'*_src_sub_i','dest':'dest_sub_no_ast_s'}", - "/copyFields/[16]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_s'}", - "/copyFields/[17]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}", - "/copyFields/[18]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}"); + "/copyFields/[16]=={'source':'src_sub_no_ast_i','dest':'*_s'}", + "/copyFields/[17]=={'source':'src_sub_no_ast_i','dest':'*_dest_sub_s'}", + "/copyFields/[18]=={'source':'src_sub_no_ast_i','dest':'dest_sub_no_ast_s'}"); } - @Test - public void testRestrictSource() throws Exception { - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=title,*_i,*_src_sub_i,src_sub_no_ast_i", - "count(/response/arr[@name='copyFields']/lst) = 16", // 4 + 4 + 4 + 4 - "count(/response/arr[@name='copyFields']/lst/str[@name='source'][.='title']) = 4", - "count(/response/arr[@name='copyFields']/lst/str[@name='source'][.='*_i']) = 4", - "count(/response/arr[@name='copyFields']/lst/str[@name='source'][.='*_src_sub_i']) = 4", - "count(/response/arr[@name='copyFields']/lst/str[@name='source'][.='src_sub_no_ast_i']) = 4"); - } - - @Test - public void testRestrictDest() throws Exception { - assertQ("/schema/copyfields/?indent=on&wt=xml&dest.fl=title,*_s,*_dest_sub_s,dest_sub_no_ast_s", - "count(/response/arr[@name='copyFields']/lst) = 16", // 3 + 4 + 4 + 5 - "count(/response/arr[@name='copyFields']/lst/str[@name='dest'][.='title']) = 3", - "count(/response/arr[@name='copyFields']/lst/str[@name='dest'][.='*_s']) = 4", - "count(/response/arr[@name='copyFields']/lst/str[@name='dest'][.='*_dest_sub_s']) = 4", - "count(/response/arr[@name='copyFields']/lst/str[@name='dest'][.='dest_sub_no_ast_s']) = 5"); - } - - @Test - public void testRestrictSourceAndDest() throws Exception { - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=title,*_i&dest.fl=title,dest_sub_no_ast_s", - "count(/response/arr[@name='copyFields']/lst) = 3", - - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']]", - - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" - +" and str[@name='dest'][.='title']]", - - "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='*_i']" - +" and str[@name='dest'][.='dest_sub_no_ast_s']]"); - } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldCollectionResource.java index 318b28a78ad9..032bbad08ccd 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldCollectionResource.java @@ -29,21 +29,6 @@ public void testGetAllDynamicFields() throws Exception { "(/response/arr[@name='dynamicFields']/lst/str[@name='name'])[3] = '*_mfacet'"); } - @Test - public void testGetTwoDynamicFields() throws IOException { - assertQ("/schema/dynamicfields?indent=on&wt=xml&fl=*_i,*_s", - "count(/response/arr[@name='dynamicFields']/lst/str[@name='name']) = 2", - "(/response/arr[@name='dynamicFields']/lst/str[@name='name'])[1] = '*_i'", - "(/response/arr[@name='dynamicFields']/lst/str[@name='name'])[2] = '*_s'"); - } - - @Test - public void testNotFoundDynamicFields() throws IOException { - assertQ("/schema/dynamicfields?indent=on&wt=xml&fl=*_not_in_there,this_one_isnt_either_*", - "count(/response/arr[@name='dynamicFields']) = 1", - "count(/response/arr[@name='dynamicfields']/lst/str[@name='name']) = 0"); - } - @Test public void testJsonGetAllDynamicFields() throws Exception { assertJQ("/schema/dynamicfields?indent=on", @@ -51,18 +36,4 @@ public void testJsonGetAllDynamicFields() throws Exception { "/dynamicFields/[1]/name=='ignored_*'", "/dynamicFields/[2]/name=='*_mfacet'"); } - - @Test - public void testJsonGetTwoDynamicFields() throws Exception { - assertJQ("/schema/dynamicfields?indent=on&fl=*_i,*_s&wt=xml", // assertJQ will fix the wt param to be json - "/dynamicFields/[0]/name=='*_i'", - "/dynamicFields/[1]/name=='*_s'"); - } - - @Test - public void testJsonPostFieldsToNonMutableIndexSchema() throws Exception { - assertJPost("/schema/dynamicfields", - "[{\"name\":\"foobarbaz\", \"type\":\"text_general\", \"stored\":\"false\"}]", - "/error/msg=='This IndexSchema is not mutable.'"); - } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldResource.java index 54b17fcdd9fc..7ca7953d393d 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldResource.java @@ -67,11 +67,4 @@ public void testJsonGetDynamicField() throws Exception { "/dynamicField/required==false", "/dynamicField/tokenized==false"); } - - @Test - public void testJsonPutFieldToNonMutableIndexSchema() throws Exception { - assertJPut("/schema/dynamicfields/newfield_*", - "{\"type\":\"text_general\", \"stored\":\"false\"}", - "/error/msg=='This IndexSchema is not mutable.'"); - } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java index 571acc52bffc..dd554158eab6 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java @@ -30,45 +30,6 @@ public void testGetAllFields() throws Exception { "(/response/arr[@name='fields']/lst/str[@name='name'])[3] = '_version_'"); } - @Test - public void testGetTwoFields() throws IOException { - assertQ("/schema/fields?indent=on&wt=xml&fl=id,_version_", - "count(/response/arr[@name='fields']/lst/str[@name='name']) = 2", - "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", - "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'"); - } - - @Test - public void testGetThreeFieldsDontIncludeDynamic() throws IOException { - // - assertQ("/schema/fields?indent=on&wt=xml&fl=id,_version_,price_i", - "count(/response/arr[@name='fields']/lst/str[@name='name']) = 2", - "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", - "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'"); - } - - @Test - public void testGetThreeFieldsIncludeDynamic() throws IOException { - assertQ("/schema/fields?indent=on&wt=xml&fl=id,_version_,price_i&includeDynamic=on", - - "count(/response/arr[@name='fields']/lst/str[@name='name']) = 3", - - "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", - - "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'", - - "(/response/arr[@name='fields']/lst/str[@name='name'])[3] = 'price_i'", - - "/response/arr[@name='fields']/lst[ str[@name='name']='price_i' " - +" and str[@name='dynamicBase']='*_i']"); - } - - @Test - public void testNotFoundFields() throws IOException { - assertQ("/schema/fields?indent=on&wt=xml&fl=not_in_there,this_one_either", - "count(/response/arr[@name='fields']) = 1", - "count(/response/arr[@name='fields']/lst/str[@name='name']) = 0"); - } @Test public void testJsonGetAllFields() throws Exception { @@ -78,10 +39,4 @@ public void testJsonGetAllFields() throws Exception { "/fields/[2]/name=='_version_'"); } - @Test - public void testJsonGetTwoFields() throws Exception { - assertJQ("/schema/fields?indent=on&fl=id,_version_&wt=xml", // assertJQ should fix the wt param to be json - "/fields/[0]/name=='id'", - "/fields/[1]/name=='_version_'"); - } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java index 711e3c05face..627aee09e780 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java @@ -72,18 +72,12 @@ public void testJsonGetField() throws Exception { "/field/tokenized==true"); } - @Test - public void testGetFieldIncludeDynamic() throws Exception { - assertQ("/schema/fields/some_crazy_name_i?indent=on&wt=xml&includeDynamic=true", - "/response/lst[@name='field']/str[@name='name'] = 'some_crazy_name_i'", - "/response/lst[@name='field']/str[@name='dynamicBase'] = '*_i'"); - } - + @Test public void testGetFieldDontShowDefaults() throws Exception { String[] tests = { "count(/response/lst[@name='field']) = 1", - "count(/response/lst[@name='field']/*) = 7", + "count(/response/lst[@name='field']/*) = 6", "/response/lst[@name='field']/str[@name='name'] = 'id'", "/response/lst[@name='field']/str[@name='type'] = 'string'", "/response/lst[@name='field']/bool[@name='indexed'] = 'true'", @@ -95,17 +89,4 @@ public void testGetFieldDontShowDefaults() throws Exception { assertQ("/schema/fields/id?indent=on&wt=xml&showDefaults=false", tests); } - @Test - public void testJsonPutFieldToNonMutableIndexSchema() throws Exception { - assertJPut("/schema/fields/newfield", - "{\"type\":\"text_general\", \"stored\":\"false\"}", - "/error/msg=='This IndexSchema is not mutable.'"); - } - - @Test - public void testJsonPostFieldsToNonMutableIndexSchema() throws Exception { - assertJPost("/schema/fields", - "[{\"name\":\"foobarbaz\", \"type\":\"text_general\", \"stored\":\"false\"}]", - "/error/msg=='This IndexSchema is not mutable.'"); - } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeCollectionResource.java index a61f6444dccd..53cd1c0c8604 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeCollectionResource.java @@ -19,6 +19,7 @@ import org.junit.Test; public class TestFieldTypeCollectionResource extends SolrRestletTestBase { + @Test public void testGetAllFieldTypes() throws Exception { assertQ("/schema/fieldtypes?indent=on&wt=xml", diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java index 2f0c2410365b..eb72aeda63e6 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java @@ -24,7 +24,7 @@ public class TestFieldTypeResource extends SolrRestletTestBase { public void testGetFieldType() throws Exception { assertQ("/schema/fieldtypes/float?indent=on&wt=xml&showDefaults=true", "count(/response/lst[@name='fieldType']) = 1", - "count(/response/lst[@name='fieldType']/*) = 18", + "count(/response/lst[@name='fieldType']/*) = 16", "/response/lst[@name='fieldType']/str[@name='name'] = 'float'", "/response/lst[@name='fieldType']/str[@name='class'] = 'solr.TrieFloatField'", "/response/lst[@name='fieldType']/str[@name='precisionStep'] ='0'", @@ -39,9 +39,7 @@ public void testGetFieldType() throws Exception { "/response/lst[@name='fieldType']/bool[@name='omitPositions'] = 'false'", "/response/lst[@name='fieldType']/bool[@name='storeOffsetsWithPositions'] = 'false'", "/response/lst[@name='fieldType']/bool[@name='multiValued'] = 'false'", - "/response/lst[@name='fieldType']/bool[@name='tokenized'] = 'false'", - "/response/lst[@name='fieldType']/arr[@name='fields']/str = 'weight'", - "/response/lst[@name='fieldType']/arr[@name='dynamicFields']/str = '*_f'"); + "/response/lst[@name='fieldType']/bool[@name='tokenized'] = 'false'"); } @Test @@ -69,22 +67,19 @@ public void testJsonGetFieldType() throws Exception { "/fieldType/omitPositions==false", "/fieldType/storeOffsetsWithPositions==false", "/fieldType/multiValued==false", - "/fieldType/tokenized==false", - "/fieldType/fields==['weight']", - "/fieldType/dynamicFields==['*_f']"); + "/fieldType/tokenized==false"); } @Test public void testGetFieldTypeDontShowDefaults() throws Exception { assertQ("/schema/fieldtypes/teststop?wt=xml&indent=on", - "count(/response/lst[@name='fieldType']/*) = 5", + "count(/response/lst[@name='fieldType']/*) = 3", "/response/lst[@name='fieldType']/str[@name='name'] = 'teststop'", "/response/lst[@name='fieldType']/str[@name='class'] = 'solr.TextField'", "/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.LowerCaseTokenizerFactory'", "/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='class'][.='solr.StandardFilterFactory']", "/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='class'][.='solr.StopFilterFactory']", - "/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='words'][.='stopwords.txt']", - "/response/lst[@name='fieldType']/arr[@name='fields']/str[.='teststop']", - "/response/lst[@name='fieldType']/arr[@name='dynamicFields']"); + "/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='words'][.='stopwords.txt']" + ); } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaDynamicFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaDynamicFieldResource.java deleted file mode 100644 index 65727099eabf..000000000000 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaDynamicFieldResource.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.commons.io.FileUtils; -import org.apache.solr.util.RestTestBase; -import org.eclipse.jetty.servlet.ServletHolder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.restlet.ext.servlet.ServerServlet; - -import java.io.File; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.regex.Pattern; - -public class TestManagedSchemaDynamicFieldResource extends RestTestBase { - - private static File tmpSolrHome; - private static File tmpConfDir; - - private static final String collection = "collection1"; - private static final String confDir = collection + "/conf"; - - - @Before - public void before() throws Exception { - tmpSolrHome = createTempDir().toFile(); - tmpConfDir = new File(tmpSolrHome, confDir); - FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile()); - - final SortedMap extraServlets = new TreeMap<>(); - final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class); - solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi"); - extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...' - - System.setProperty("managed.schema.mutable", "true"); - System.setProperty("enable.update.log", "false"); - - createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml", - "/solr", true, extraServlets); - } - - @After - public void after() throws Exception { - if (jetty != null) { - jetty.stop(); - jetty = null; - } - client = null; - if (restTestHarness != null) { - restTestHarness.close(); - } - restTestHarness = null; - } - - @Test - public void testAddDynamicFieldBadFieldType() throws Exception { - assertJPut("/schema/dynamicfields/*_newdynamicfield", - json( "{'type':'not_in_there_at_all','stored':false}" ), - "/error/msg==\"Dynamic field \\'*_newdynamicfield\\': Field type \\'not_in_there_at_all\\' not found.\""); - } - - @Test - public void testAddDynamicFieldMismatchedName() throws Exception { - assertJPut("/schema/dynamicfields/*_newdynamicfield", - json( "{'name':'*_something_else','type':'text','stored':false}" ), - "/error/msg=='///regex:\\\\*_newdynamicfield///'"); - } - - @Test - public void testAddDynamicFieldBadProperty() throws Exception { - assertJPut("/schema/dynamicfields/*_newdynamicfield", - json( "{'type':'text','no_property_with_this_name':false}" ), - "/error/msg==\"java.lang.IllegalArgumentException: Invalid field property: no_property_with_this_name\""); - } - - @Test - public void testAddDynamicField() throws Exception { - assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml", - "count(/response/lst[@name='newdynamicfield_*']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/dynamicfields/newdynamicfield_*", - json("{'type':'text','stored':false}"), - "/responseHeader/status==0"); - - assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertU(adoc("newdynamicfield_A", "value1 value2", "id", "123")); - assertU(commit()); - - assertQ("/select?q=newdynamicfield_A:value1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='123']"); - } - - @Test - public void testAddDynamicFieldWithMulipleOptions() throws Exception { - assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/dynamicfields/newdynamicfield_*", - json("{'type':'text_en','stored':true,'indexed':false}"), - "/responseHeader/status==0"); - - File managedSchemaFile = new File(tmpConfDir, "managed-schema"); - assertTrue(managedSchemaFile.exists()); - String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newdynamicfieldStoredTrueIndexedFalsePattern - = Pattern.compile( ""); - assertTrue(newdynamicfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/lst[@name='dynamicField']/str[@name='name'] = 'newdynamicfield_*'", - "/response/lst[@name='dynamicField']/str[@name='type'] = 'text_en'", - "/response/lst[@name='dynamicField']/bool[@name='indexed'] = 'false'", - "/response/lst[@name='dynamicField']/bool[@name='stored'] = 'true'"); - - assertU(adoc("newdynamicfield_A", "value1 value2", "id", "1234")); - assertU(commit()); - - assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/dynamicfields/newdynamicfield2_*", - json("{'type':'text_en','stored':true,'indexed':true,'multiValued':true}"), - "/responseHeader/status==0"); - - managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern - = Pattern.compile( ""); - assertTrue(newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/lst[@name='dynamicField']/str[@name='name'] = 'newdynamicfield2_*'", - "/response/lst[@name='dynamicField']/str[@name='type'] = 'text_en'", - "/response/lst[@name='dynamicField']/bool[@name='indexed'] = 'true'", - "/response/lst[@name='dynamicField']/bool[@name='stored'] = 'true'", - "/response/lst[@name='dynamicField']/bool[@name='multiValued'] = 'true'"); - - assertU(adoc("newdynamicfield2_A", "value1 value2", "newdynamicfield2_A", "value3 value4", "id", "5678")); - assertU(commit()); - - assertQ("/select?q=newdynamicfield2_A:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='5678']"); - } - - @Test - public void testAddDynamicFieldCollectionWithMultipleOptions() throws Exception { - assertQ("/schema/dynamicfields?indent=on&wt=xml", - "count(/response/arr[@name='dynamicFields']/lst/str[@name]) > 0", // there are fields - "count(/response/arr[@name='dynamicFields']/lst/str[starts-with(@name,'newfield')]) = 0"); // but none named newfield* - - assertJPost("/schema/dynamicfields", - json("[{'name':'newdynamicfield_*','type':'text_en','stored':true,'indexed':false}]"), - "/responseHeader/status==0"); - - File managedSchemaFile = new File(tmpConfDir, "managed-schema"); - assertTrue(managedSchemaFile.exists()); - String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newfieldStoredTrueIndexedFalsePattern - = Pattern.compile( ""); - assertTrue(newfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/dynamicfields?indent=on&wt=xml", - "/response/arr[@name='dynamicFields']/lst" - + "[str[@name='name']='newdynamicfield_*' and str[@name='type']='text_en'" - + " and bool[@name='stored']='true' and bool[@name='indexed']='false']"); - - assertU(adoc("newdynamicfield_A", "value1 value2", "id", "789")); - assertU(commit()); - - assertJPost("/schema/dynamicfields", - json("[{'name':'newdynamicfield2_*','type':'text_en','stored':true,'indexed':true,'multiValued':true}]"), - "/responseHeader/status==0"); - - managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern - = Pattern.compile( ""); - assertTrue(newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/dynamicfields?indent=on&wt=xml", - "/response/arr[@name='dynamicFields']/lst" - + "[str[@name='name']='newdynamicfield2_*' and str[@name='type']='text_en'" - + " and bool[@name='stored']='true' and bool[@name='indexed']='true' and bool[@name='multiValued']='true']"); - - assertU(adoc("newdynamicfield2_A", "value1 value2", "newdynamicfield2_A", "value3 value4", "id", "790")); - assertU(commit()); - - assertQ("/select?q=newdynamicfield2_A:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='790']"); - } - - - @Test - public void testAddCopyField() throws Exception { - assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/dynamicfields/dynamicfieldA_*", - json("{'type':'text','stored':false}"), - "/responseHeader/status==0"); - assertJPut("/schema/dynamicfields/dynamicfieldB_*", - json("{'type':'text','stored':false, 'copyFields':['dynamicfieldA_*']}"), - "/responseHeader/status==0"); - assertJPut("/schema/dynamicfields/dynamicfieldC_*", - json("{'type':'text','stored':false, 'copyFields':'dynamicfieldA_*'}"), - "/responseHeader/status==0"); - - assertQ("/schema/dynamicfields/dynamicfieldB_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldB_*", - "count(/response/arr[@name='copyFields']/lst) = 1"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldC_*", - "count(/response/arr[@name='copyFields']/lst) = 1"); - //fine to pass in empty list, just won't do anything - assertJPut("/schema/dynamicfields/dynamicfieldD_*", - json("{'type':'text','stored':false, 'copyFields':[]}"), - "/responseHeader/status==0"); - //some bad usages - assertJPut("/schema/dynamicfields/dynamicfieldF_*", - json("{'type':'text','stored':false, 'copyFields':['some_nonexistent_dynamicfield_ignore_exception_*']}"), - "/error/msg==\"copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.\""); - } - - @Test - public void testPostMultipleDynamicFields() throws Exception { - assertQ("/schema/dynamicfields/newdynamicfield1_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPost("/schema/dynamicfields", - json( "[{'name':'newdynamicfield1_*','type':'text','stored':false}," - + " {'name':'newdynamicfield2_*','type':'text','stored':false}]"), - "/responseHeader/status==0"); - - assertQ("/schema/dynamicfields/newdynamicfield1_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml", - "count(/response/lst[@name='dynamicField']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertU(adoc("newdynamicfield1_A", "value1 value2", "id", "123")); - assertU(adoc("newdynamicfield2_A", "value3 value4", "id", "456")); - assertU(commit()); - - assertQ("/select?q=newdynamicfield1_A:value1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='123']"); - assertQ("/select?q=newdynamicfield2_A:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='456']"); - } - - @Test - public void testPostCopy() throws Exception { - assertJPost("/schema/dynamicfields", - json( "[{'name':'dynamicfieldA_*','type':'text','stored':false}," - + " {'name':'dynamicfieldB_*','type':'text','stored':false}," - + " {'name':'dynamicfieldC_*','type':'text','stored':false, 'copyFields':['dynamicfieldB_*']}]"), - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldC_*", - "count(/response/arr[@name='copyFields']/lst) = 1"); - assertJPost("/schema/dynamicfields", - json( "[{'name':'dynamicfieldD_*','type':'text','stored':false}," - + " {'name':'dynamicfieldE_*','type':'text','stored':false}," - + " {'name':'dynamicfieldF_*','type':'text','stored':false, 'copyFields':['dynamicfieldD_*','dynamicfieldE_*']}," - + " {'name':'dynamicfieldG_*','type':'text','stored':false, 'copyFields':'dynamicfieldD_*'}]"),//single - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldF_*", - "count(/response/arr[@name='copyFields']/lst) = 2"); - //passing in an empty list is perfectly acceptable, it just won't do anything - assertJPost("/schema/dynamicfields", - json( "[{'name':'dynamicfieldX_*','type':'text','stored':false}," - + " {'name':'dynamicfieldY_*','type':'text','stored':false}," - + " {'name':'dynamicfieldZ_*','type':'text','stored':false, 'copyFields':[]}]"), - "/responseHeader/status==0"); - //some bad usages - - assertJPost("/schema/dynamicfields", - json( "[{'name':'dynamicfieldH_*','type':'text','stored':false}," - + " {'name':'dynamicfieldI_*','type':'text','stored':false}," - + " {'name':'dynamicfieldJ_*','type':'text','stored':false, 'copyFields':['some_nonexistent_dynamicfield_ignore_exception_*']}]"), - "/error/msg=='copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.'"); - } - - @Test - public void testPostCopyFields() throws Exception { - assertJPost("/schema/dynamicfields", - json( "[{'name':'dynamicfieldA_*','type':'text','stored':false}," - + " {'name':'dynamicfieldB_*','type':'text','stored':false}," - + " {'name':'dynamicfieldC_*','type':'text','stored':false}," - + " {'name':'dynamicfieldD_*','type':'text','stored':false}," - + " {'name':'dynamicfieldE_*','type':'text','stored':false}]"), - "/responseHeader/status==0"); - assertJPost("/schema/copyfields", - json( "[{'source':'dynamicfieldA_*', 'dest':'dynamicfieldB_*'}," - + " {'source':'dynamicfieldD_*', 'dest':['dynamicfieldC_*', 'dynamicfieldE_*']}]"), - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldA_*", - "count(/response/arr[@name='copyFields']/lst) = 1"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldD_*", - "count(/response/arr[@name='copyFields']/lst) = 2"); - assertJPost("/schema/copyfields", // copyField glob sources are not required to match a dynamic field - json("[{'source':'some_glob_not_necessarily_matching_any_dynamicfield_*', 'dest':['dynamicfieldA_*']}," - +" {'source':'*', 'dest':['dynamicfieldD_*']}]"), - "/responseHeader/status==0"); - assertJPost("/schema/copyfields", - json("[{'source':'dynamicfieldD_*', 'dest':['some_nonexistent_dynamicfield_ignore_exception_*']}]"), - "/error/msg=='copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.'"); - } -} - diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldResource.java deleted file mode 100644 index b39d266f2238..000000000000 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldResource.java +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import org.apache.commons.io.FileUtils; -import org.apache.solr.util.RestTestBase; -import org.eclipse.jetty.servlet.ServletHolder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.restlet.ext.servlet.ServerServlet; - -import java.io.File; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.regex.Pattern; - -public class TestManagedSchemaFieldResource extends RestTestBase { - - private static File tmpSolrHome; - private static File tmpConfDir; - - private static final String collection = "collection1"; - private static final String confDir = collection + "/conf"; - - - @Before - public void before() throws Exception { - tmpSolrHome = createTempDir().toFile(); - tmpConfDir = new File(tmpSolrHome, confDir); - FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile()); - - final SortedMap extraServlets = new TreeMap<>(); - final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class); - solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi"); - extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...' - - System.setProperty("managed.schema.mutable", "true"); - System.setProperty("enable.update.log", "false"); - - createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml", - "/solr", true, extraServlets); - } - - @After - public void after() throws Exception { - if (jetty != null) { - jetty.stop(); - jetty = null; - } - client = null; - if (restTestHarness != null) { - restTestHarness.close(); - } - restTestHarness = null; - } - - @Test - public void testAddFieldBadFieldType() throws Exception { - assertJPut("/schema/fields/newfield", - json( "{'type':'not_in_there_at_all','stored':false}" ), - "/error/msg==\"Field \\'newfield\\': Field type \\'not_in_there_at_all\\' not found.\""); - } - - @Test - public void testAddFieldMismatchedName() throws Exception { - assertJPut("/schema/fields/newfield", - json( "{'name':'something_else','type':'text','stored':false}" ), - "/error/msg=='///regex:newfield///'"); - } - - @Test - public void testAddFieldBadProperty() throws Exception { - assertJPut("/schema/fields/newfield", - json( "{'type':'text','no_property_with_this_name':false}" ), - "/error/msg==\"java.lang.IllegalArgumentException: Invalid field property: no_property_with_this_name\""); - } - - @Test - public void testAddField() throws Exception { - assertQ("/schema/fields/newfield?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/fields/newfield", - json("{'type':'text','stored':false}"), - "/responseHeader/status==0"); - - assertQ("/schema/fields/newfield?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertU(adoc("newfield", "value1 value2", "id", "123")); - assertU(commit()); - - assertQ("/select?q=newfield:value1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='123']"); - } - - @Test - public void testAddFieldWithMulipleOptions() throws Exception { - assertQ("/schema/fields/newfield?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/fields/newfield", - json("{'type':'text_en','stored':true,'indexed':false}"), - "/responseHeader/status==0"); - - File managedSchemaFile = new File(tmpConfDir, "managed-schema"); - assertTrue(managedSchemaFile.exists()); - String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newfieldStoredTrueIndexedFalsePattern - = Pattern.compile( ""); - assertTrue(newfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/fields/newfield?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/lst[@name='field']/str[@name='name'] = 'newfield'", - "/response/lst[@name='field']/str[@name='type'] = 'text_en'", - "/response/lst[@name='field']/bool[@name='indexed'] = 'false'", - "/response/lst[@name='field']/bool[@name='stored'] = 'true'"); - - assertU(adoc("newfield", "value1 value2", "id", "1234")); - assertU(commit()); - - assertQ("/schema/fields/newfield2?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/fields/newfield2", - json("{'type':'text_en','stored':true,'indexed':true,'multiValued':true}"), - "/responseHeader/status==0"); - - managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newfield2StoredTrueIndexedTrueMultiValuedTruePattern - = Pattern.compile( ""); - assertTrue(newfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/fields/newfield2?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/lst[@name='field']/str[@name='name'] = 'newfield2'", - "/response/lst[@name='field']/str[@name='type'] = 'text_en'", - "/response/lst[@name='field']/bool[@name='indexed'] = 'true'", - "/response/lst[@name='field']/bool[@name='stored'] = 'true'", - "/response/lst[@name='field']/bool[@name='multiValued'] = 'true'"); - - assertU(adoc("newfield2", "value1 value2", "newfield2", "value3 value4", "id", "5678")); - assertU(commit()); - - assertQ("/select?q=newfield2:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='5678']"); - } - - @Test - public void testAddFieldCollectionWithMultipleOptions() throws Exception { - assertQ("/schema/fields?indent=on&wt=xml", - "count(/response/arr[@name='fields']/lst/str[@name]) > 0", // there are fields - "count(/response/arr[@name='fields']/lst/str[starts-with(@name,'newfield')]) = 0"); // but none named newfield* - - assertJPost("/schema/fields", - json("[{'name':'newfield','type':'text_en','stored':true,'indexed':false}]"), - "/responseHeader/status==0"); - - File managedSchemaFile = new File(tmpConfDir, "managed-schema"); - assertTrue(managedSchemaFile.exists()); - String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newfieldStoredTrueIndexedFalsePattern - = Pattern.compile( ""); - assertTrue(newfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/fields?indent=on&wt=xml", - "/response/arr[@name='fields']/lst" - + "[str[@name='name']='newfield' and str[@name='type']='text_en'" - + " and bool[@name='stored']='true' and bool[@name='indexed']='false']"); - - assertU(adoc("newfield", "value1 value2", "id", "789")); - assertU(commit()); - - assertJPost("/schema/fields", - json("[{'name':'newfield2','type':'text_en','stored':true,'indexed':true,'multiValued':true}]"), - "/responseHeader/status==0"); - - managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8"); - Pattern newfield2StoredTrueIndexedTrueMultiValuedTruePattern - = Pattern.compile( ""); - assertTrue(newfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find()); - - assertQ("/schema/fields?indent=on&wt=xml", - "/response/arr[@name='fields']/lst" - + "[str[@name='name']='newfield2' and str[@name='type']='text_en'" - + " and bool[@name='stored']='true' and bool[@name='indexed']='true' and bool[@name='multiValued']='true']"); - - assertU(adoc("newfield2", "value1 value2", "newfield2", "value3 value4", "id", "790")); - assertU(commit()); - - assertQ("/select?q=newfield2:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='790']"); - } - - - @Test - public void testAddCopyField() throws Exception { - assertQ("/schema/fields/newfield2?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPut("/schema/fields/fieldA", - json("{'type':'text','stored':false}"), - "/responseHeader/status==0"); - assertJPut("/schema/fields/fieldB", - json("{'type':'text','stored':false, 'copyFields':['fieldA']}"), - "/responseHeader/status==0"); - assertJPut("/schema/fields/fieldC", - json("{'type':'text','stored':false, 'copyFields':'fieldA'}"), - "/responseHeader/status==0"); - - assertQ("/schema/fields/fieldB?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldB", - "count(/response/arr[@name='copyFields']/lst) = 1" - ); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldC", - "count(/response/arr[@name='copyFields']/lst) = 1" - ); - //fine to pass in empty list, just won't do anything - assertJPut("/schema/fields/fieldD", - json("{'type':'text','stored':false, 'copyFields':[]}"), - "/responseHeader/status==0"); - //some bad usages - assertJPut("/schema/fields/fieldF", - json("{'type':'text','stored':false, 'copyFields':['some_nonexistent_field_ignore_exception']}"), - "/error/msg==\"copyField dest :\\'some_nonexistent_field_ignore_exception\\' is not an explicit field and doesn\\'t match a dynamicField.\""); - } - - @Test - public void testPostMultipleFields() throws Exception { - assertQ("/schema/fields/newfield1?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertQ("/schema/fields/newfield2?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 0", - "/response/lst[@name='responseHeader']/int[@name='status'] = '404'", - "/response/lst[@name='error']/int[@name='code'] = '404'"); - - assertJPost("/schema/fields", - json( "[{'name':'newfield1','type':'text','stored':false}," - + " {'name':'newfield2','type':'text','stored':false}]"), - "/responseHeader/status==0"); - - assertQ("/schema/fields/newfield1?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertQ("/schema/fields/newfield2?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - assertU(adoc("newfield1", "value1 value2", "id", "123")); - assertU(adoc("newfield2", "value3 value4", "id", "456")); - assertU(commit()); - - assertQ("/select?q=newfield1:value1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='123']"); - assertQ("/select?q=newfield2:value3", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'", - "/response/result[@name='response'][@numFound='1']", - "count(/response/result[@name='response']/doc/*) = 1", - "/response/result[@name='response']/doc/str[@name='id'][.='456']"); - } - - @Test - public void testPostCopy() throws Exception { - assertJPost("/schema/fields", - json( "[{'name':'fieldA','type':'text','stored':false}," - + " {'name':'fieldB','type':'text','stored':false}," - + " {'name':'fieldC','type':'text','stored':false, 'copyFields':['fieldB']}]"), - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldC", - "count(/response/arr[@name='copyFields']/lst) = 1" - ); - assertJPost("/schema/fields", - json( "[{'name':'fieldD','type':'text','stored':false}," - + " {'name':'fieldE','type':'text','stored':false}," - + " {'name':'fieldF','type':'text','stored':false, 'copyFields':['fieldD','fieldE']}," - + " {'name':'fieldG','type':'text','stored':false, 'copyFields':'fieldD'}]"),//single - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldF", - "count(/response/arr[@name='copyFields']/lst) = 2" - ); - //passing in an empty list is perfectly acceptable, it just won't do anything - assertJPost("/schema/fields", - json( "[{'name':'fieldX','type':'text','stored':false}," - + " {'name':'fieldY','type':'text','stored':false}," - + " {'name':'fieldZ','type':'text','stored':false, 'copyFields':[]}]"), - "/responseHeader/status==0"); - //some bad usages - - assertJPost("/schema/fields", - json( "[{'name':'fieldH','type':'text','stored':false}," - + " {'name':'fieldI','type':'text','stored':false}," - + " {'name':'fieldJ','type':'text','stored':false, 'copyFields':['some_nonexistent_field_ignore_exception']}]"), - "/error/msg=='copyField dest :\\'some_nonexistent_field_ignore_exception\\' is not an explicit field and doesn\\'t match a dynamicField.'"); - } - - @Test - public void testPostCopyFields() throws Exception { - assertJPost("/schema/fields", - json( "[{'name':'fieldA','type':'text','stored':false}," - + " {'name':'fieldB','type':'text','stored':false}," - + " {'name':'fieldC','type':'text','stored':false}," - + " {'name':'fieldD','type':'text','stored':false}," - + " {'name':'fieldE','type':'text','stored':false}]"), - "/responseHeader/status==0"); - assertJPost("/schema/copyfields", - json( "[{'source':'fieldA', 'dest':'fieldB'}," - + " {'source':'fieldD', 'dest':['fieldC', 'fieldE']}]"), - "/responseHeader/status==0"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldA", - "count(/response/arr[@name='copyFields']/lst) = 1"); - assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldD", - "count(/response/arr[@name='copyFields']/lst) = 2"); - assertJPost("/schema/copyfields", - json("[{'source':'some_nonexistent_field_ignore_exception', 'dest':['fieldA']}]"), - "/error/msg=='copyField source :\\'some_nonexistent_field_ignore_exception\\' is not a glob and doesn\\'t match any explicit field or dynamicField.'"); - assertJPost("/schema/copyfields", - json("[{'source':'fieldD', 'dest':['some_nonexistent_field_ignore_exception']}]"), - "/error/msg=='copyField dest :\\'some_nonexistent_field_ignore_exception\\' is not an explicit field and doesn\\'t match a dynamicField.'"); - } -} - diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldTypeResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldTypeResource.java deleted file mode 100644 index a0f4e2538a74..000000000000 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaFieldTypeResource.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.commons.io.FileUtils; -import org.apache.solr.util.RestTestBase; -import org.eclipse.jetty.servlet.ServletHolder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.noggit.JSONUtil; -import org.restlet.ext.servlet.ServerServlet; - -public class TestManagedSchemaFieldTypeResource extends RestTestBase { - - private static File tmpSolrHome; - private static File tmpConfDir; - - private static final String collection = "collection1"; - private static final String confDir = collection + "/conf"; - - @Before - public void before() throws Exception { - tmpSolrHome = createTempDir().toFile(); - tmpConfDir = new File(tmpSolrHome, confDir); - FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile()); - - final SortedMap extraServlets = new TreeMap<>(); - final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class); - solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi"); - extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...' - - System.setProperty("managed.schema.mutable", "true"); - System.setProperty("enable.update.log", "false"); - - createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml", - "/solr", true, extraServlets); - } - - @After - private void after() throws Exception { - jetty.stop(); - jetty = null; - System.clearProperty("managed.schema.mutable"); - System.clearProperty("enable.update.log"); - - if (restTestHarness != null) { - restTestHarness.close(); - } - restTestHarness = null; - } - - @Test - public void testAddFieldTypes() throws Exception { - - // name mismatch - assertJPut("/schema/fieldtypes/myIntFieldType", - json("{'name':'badNameEh','class':'solr.TrieIntField','stored':false}"), - "/responseHeader/status==400"); - - // no class - assertJPut("/schema/fieldtypes/myIntFieldType", - json("{'stored':false}"), - "/responseHeader/status==400"); - - // invalid attribute - assertJPut("/schema/fieldtypes/myIntFieldType", - json("{'foo':'bar'}"), - "/responseHeader/status==400"); - - // empty analyzer - String ftdef = ""; - ftdef += "{"; - ftdef += " 'class':'solr.TextField','positionIncrementGap':'100',"; - ftdef += " 'analyzer':''"; - ftdef += "}"; - assertJPut("/schema/fieldtypes/emptyAnalyzerFieldType", - json(ftdef), - "/responseHeader/status==400"); - - // basic field types - assertJPut("/schema/fieldtypes/myIntFieldType", - json("{'name':'myIntFieldType','class':'solr.TrieIntField','stored':false}"), - "/responseHeader/status==0"); - checkFieldTypeProps(getExpectedProps("myIntFieldType", "solr.TrieIntField", true, false), 16); - - assertJPut("/schema/fieldtypes/myDoubleFieldType", - json("{'class':'solr.TrieDoubleField','precisionStep':'0','positionIncrementGap':'0'}"), - "/responseHeader/status==0"); - Map expProps = - getExpectedProps("myDoubleFieldType", "solr.TrieDoubleField", true, true); - // add some additional expected props for this type - expProps.put("precisionStep", "0"); - expProps.put("positionIncrementGap", "0"); - checkFieldTypeProps(expProps, 18); - - assertJPut("/schema/fieldtypes/myBoolFieldType", - json("{'class':'solr.BoolField','sortMissingLast':true}"), - "/responseHeader/status==0"); - expProps = getExpectedProps("myBoolFieldType", "solr.BoolField", true, true); - expProps.put("sortMissingLast", true); - checkFieldTypeProps(expProps, 17); - - // a text analyzing field type - ftdef = "{"; - ftdef += " 'class':'solr.TextField','positionIncrementGap':'100',"; - ftdef += " 'analyzer':{"; - ftdef += " 'charFilters':["; - ftdef += " {'class':'solr.PatternReplaceCharFilterFactory','replacement':'$1$1','pattern':'([a-zA-Z])\\\\1+'}"; - ftdef += " ],"; - ftdef += " 'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'},"; - ftdef += " 'filters':["; - ftdef += " {'class':'solr.WordDelimiterFilterFactory','preserveOriginal':'0'},"; - ftdef += " {'class':'solr.StopFilterFactory','words':'stopwords.txt','ignoreCase':'true'},"; - ftdef += " {'class':'solr.LowerCaseFilterFactory'},"; - ftdef += " {'class':'solr.ASCIIFoldingFilterFactory'},"; - ftdef += " {'class':'solr.KStemFilterFactory'}"; - ftdef += " ]"; - ftdef += " }"; - ftdef += "}"; - - assertJPut("/schema/fieldtypes/myTextFieldType", json(ftdef), "/responseHeader/status==0"); - - expProps = getExpectedProps("myTextFieldType", "solr.TextField", true, true); - expProps.put("autoGeneratePhraseQueries", false); - expProps.put("omitNorms", false); - expProps.put("omitTermFreqAndPositions", false); - expProps.put("omitPositions", false); - expProps.put("storeOffsetsWithPositions", false); - expProps.put("tokenized", true); - - List analyzerTests = new ArrayList<>(); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='charFilters']/lst[1]/str[@name='class'] = 'solr.PatternReplaceCharFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.WhitespaceTokenizerFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[1]/str[@name='class'] = 'solr.WordDelimiterFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[2]/str[@name='class'] = 'solr.StopFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[3]/str[@name='class'] = 'solr.LowerCaseFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[4]/str[@name='class'] = 'solr.ASCIIFoldingFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[5]/str[@name='class'] = 'solr.KStemFilterFactory'"); - checkFieldTypeProps(expProps, 19, analyzerTests); - - // now add a field type that uses managed resources and a field that uses that type - - String piglatinStopWordEndpoint = "/schema/analysis/stopwords/piglatin"; - String piglatinSynonymEndpoint = "/schema/analysis/synonyms/piglatin"; - - // now define a new FieldType that uses the managed piglatin endpoints - // the managed endpoints will be autovivified as needed - ftdef = "{"; - ftdef += " 'class':'solr.TextField',"; - ftdef += " 'analyzer':{"; - ftdef += " 'tokenizer':{'class':'solr.StandardTokenizerFactory'},"; - ftdef += " 'filters':["; - ftdef += " {'class':'solr.ManagedStopFilterFactory','managed':'piglatin'},"; - ftdef += " {'class':'solr.ManagedSynonymFilterFactory','managed':'piglatin'}"; - ftdef += " ]"; - ftdef += " }"; - ftdef += "}"; - assertJPut("/schema/fieldtypes/piglatinFieldType", json(ftdef), "/responseHeader/status==0"); - - expProps = getExpectedProps("piglatinFieldType", "solr.TextField", true, true); - expProps.put("autoGeneratePhraseQueries", false); - expProps.put("omitNorms", false); - expProps.put("omitTermFreqAndPositions", false); - expProps.put("omitPositions", false); - expProps.put("storeOffsetsWithPositions", false); - expProps.put("tokenized", true); - - analyzerTests = new ArrayList<>(); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.StandardTokenizerFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[1]/str[@name='class'] = 'solr.ManagedStopFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[2]/str[@name='class'] = 'solr.ManagedSynonymFilterFactory'"); - checkFieldTypeProps(expProps, 18, analyzerTests); - - assertJQ(piglatinSynonymEndpoint, - "/synonymMappings/initArgs/ignoreCase==false", - "/synonymMappings/managedMap=={}"); - - // add some piglatin synonyms - Map> syns = new HashMap<>(); - syns.put("appyhay", Arrays.asList("ladgay","oyfuljay")); - assertJPut(piglatinSynonymEndpoint, - JSONUtil.toJSON(syns), - "/responseHeader/status==0"); - assertJQ(piglatinSynonymEndpoint, - "/synonymMappings/managedMap/appyhay==['ladgay','oyfuljay']"); - - // add some piglatin stopwords - assertJPut(piglatinStopWordEndpoint, - JSONUtil.toJSON(Arrays.asList("hetay")), - "/responseHeader/status==0"); - - assertJQ(piglatinStopWordEndpoint + "/hetay", "/hetay=='hetay'"); - - // add a field that uses our new type - assertJPut("/schema/fields/newManagedField", - json("{'type':'piglatinFieldType','stored':false}"), - "/responseHeader/status==0"); - - assertQ("/schema/fields/newManagedField?indent=on&wt=xml", - "count(/response/lst[@name='field']) = 1", - "/response/lst[@name='responseHeader']/int[@name='status'] = '0'"); - - // try to delete the managed synonyms endpoint, which should fail because it is being used - assertJDelete(piglatinSynonymEndpoint, "/responseHeader/status==403"); - - // test adding multiple field types at once - ftdef = "["; - ftdef += "{"; - ftdef += " 'name':'textFieldType1',"; - ftdef += " 'class':'solr.TextField','positionIncrementGap':'100',"; - ftdef += " 'analyzer':{"; - ftdef += " 'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'},"; - ftdef += " 'filters':["; - ftdef += " {'class':'solr.WordDelimiterFilterFactory','preserveOriginal':'0'},"; - ftdef += " {'class':'solr.StopFilterFactory','words':'stopwords.txt','ignoreCase':'true'},"; - ftdef += " {'class':'solr.LowerCaseFilterFactory'}"; - ftdef += " ]"; - ftdef += " }"; - ftdef += "},{"; - ftdef += " 'name':'textFieldType2',"; - ftdef += " 'class':'solr.TextField','positionIncrementGap':'100',"; - ftdef += " 'analyzer':{"; - ftdef += " 'tokenizer':{'class':'solr.WhitespaceTokenizerFactory'},"; - ftdef += " 'filters':["; - ftdef += " {'class':'solr.WordDelimiterFilterFactory','preserveOriginal':'0'},"; - ftdef += " {'class':'solr.StopFilterFactory','words':'stopwords.txt','ignoreCase':'true'},"; - ftdef += " {'class':'solr.LowerCaseFilterFactory'},"; - ftdef += " {'class':'solr.ASCIIFoldingFilterFactory'}"; - ftdef += " ]"; - ftdef += " }"; - ftdef += "}"; - ftdef += "]"; - - assertJPost("/schema/fieldtypes", json(ftdef), "/responseHeader/status==0"); - - expProps = getExpectedProps("textFieldType1", "solr.TextField", true, true); - expProps.put("autoGeneratePhraseQueries", false); - expProps.put("omitNorms", false); - expProps.put("omitTermFreqAndPositions", false); - expProps.put("omitPositions", false); - expProps.put("storeOffsetsWithPositions", false); - expProps.put("tokenized", true); - - analyzerTests = new ArrayList<>(); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.WhitespaceTokenizerFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[1]/str[@name='class'] = 'solr.WordDelimiterFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[2]/str[@name='class'] = 'solr.StopFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[3]/str[@name='class'] = 'solr.LowerCaseFilterFactory'"); - checkFieldTypeProps(expProps, 19, analyzerTests); - - expProps = getExpectedProps("textFieldType2", "solr.TextField", true, true); - expProps.put("autoGeneratePhraseQueries", false); - expProps.put("omitNorms", false); - expProps.put("omitTermFreqAndPositions", false); - expProps.put("omitPositions", false); - expProps.put("storeOffsetsWithPositions", false); - expProps.put("tokenized", true); - - analyzerTests = new ArrayList<>(); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.WhitespaceTokenizerFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[1]/str[@name='class'] = 'solr.WordDelimiterFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[2]/str[@name='class'] = 'solr.StopFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[3]/str[@name='class'] = 'solr.LowerCaseFilterFactory'"); - analyzerTests.add("/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst[4]/str[@name='class'] = 'solr.ASCIIFoldingFilterFactory'"); - checkFieldTypeProps(expProps, 19, analyzerTests); - } - - /** - * Helper function to check fieldType settings against a set of expected values. - */ - protected void checkFieldTypeProps(Map expected, int expectedChildCount) { - checkFieldTypeProps(expected, expectedChildCount, null); - } - - protected void checkFieldTypeProps(Map expected, int expectedChildCount, List addlTests) { - String fieldTypeName = (String)expected.get("name"); - - List tests = new ArrayList<>(); - tests.add("count(/response/lst[@name='fieldType']) = 1"); - tests.add("count(/response/lst[@name='fieldType']/*) = "+expectedChildCount); - tests.add("count(/response/lst[@name='fieldType']/arr[@name='fields']/*) = 0"); - tests.add("count(/response/lst[@name='fieldType']/arr[@name='dynamicFields']/*) = 0"); - for (Map.Entry next : expected.entrySet()) { - Object val = next.getValue(); - String pathType = null; - if (val instanceof Boolean) - pathType = "bool"; - else if (val instanceof String) - pathType = "str"; - else - fail("Unexpected value type "+val.getClass().getName()); - // NOTE: it seems like the fieldtypes endpoint only returns strings or booleans - - String xpath = - "/response/lst[@name='fieldType']/"+pathType+"[@name='"+next.getKey()+"']"; - tests.add(xpath+" = '"+val+"'"); - } - - if (addlTests != null) - tests.addAll(addlTests); - - assertQ("/schema/fieldtypes/"+fieldTypeName+"?indent=on&wt=xml&showDefaults=true", - tests.toArray(new String[0])); - } - - /** - * Builds a map containing expected values for a field type created by this test. - */ - protected Map getExpectedProps(String name, String className, boolean indexed, boolean stored) { - Map map = new HashMap<>(); - map.put("name", name); - map.put("class", className); - map.put("indexed", indexed); - map.put("stored", stored); - map.put("docValues", false); - map.put("termVectors", false); - map.put("termPositions", false); - map.put("termOffsets", false); - map.put("omitNorms", true); - map.put("omitTermFreqAndPositions", true); - map.put("omitPositions", false); - map.put("storeOffsetsWithPositions", false); - map.put("multiValued", false); - map.put("tokenized", false); - return map; - } -} diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestRemoveLastDynamicCopyField.java b/solr/core/src/test/org/apache/solr/rest/schema/TestRemoveLastDynamicCopyField.java deleted file mode 100644 index 4b4ddd34bacd..000000000000 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestRemoveLastDynamicCopyField.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.rest.schema; - -import java.io.File; -import java.io.StringReader; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FileUtils; -import org.apache.solr.util.RestTestBase; -import org.junit.After; -import org.junit.Before; -import org.noggit.JSONParser; -import org.noggit.ObjectBuilder; - -public class TestRemoveLastDynamicCopyField extends RestTestBase { - private static File tmpSolrHome; - - @Before - public void before() throws Exception { - tmpSolrHome = createTempDir().toFile(); - FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile()); - - System.setProperty("managed.schema.mutable", "true"); - System.setProperty("enable.update.log", "false"); - - createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-single-dynamic-copy-field.xml", - "/solr", true, null); - } - - @After - public void after() throws Exception { - if (jetty != null) { - jetty.stop(); - jetty = null; - } - client = null; - if (restTestHarness != null) { - restTestHarness.close(); - } - restTestHarness = null; - } - - public void test() throws Exception { - List copyFields = getCopyFields(); - assertEquals("There is more than one copyField directive", 1, copyFields.size()); - assertEquals("The copyField source is not '*'", "*", ((Map)copyFields.get(0)).get("source")); - assertEquals("The copyField dest is not 'text'", "text", ((Map)copyFields.get(0)).get("dest")); - - String payload = "{ 'delete-copy-field': { 'source': '*', 'dest': 'text' } }"; - - String response = restTestHarness.post("/schema?wt=json", json(payload)); - Map map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response))); - assertNull(response, map.get("errors")); - - assertEquals(0, getCopyFields().size()); - } - - private List getCopyFields() throws Exception { - String response = restTestHarness.query("/schema?wt=json"); - System.err.println(response); - Map map = (Map)ObjectBuilder.getVal(new JSONParser(new StringReader(response))); - return (List)((Map)map.get("schema")).get("copyFields"); - } -} diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaSimilarityResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaSimilarityResource.java index 6b7be719e666..20166770b71e 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaSimilarityResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaSimilarityResource.java @@ -24,7 +24,6 @@ public class TestSchemaSimilarityResource extends SolrRestletTestBase { * NOTE: schema used by parent class doesn't define a global sim, so we get the implicit default * which causes the FQN of the class to be returned * - * @see TestClassNameShortening#testShortenedGlobalSimilarityStaysShortened */ @Test public void testGetSchemaSimilarity() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedStopFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedStopFilterFactory.java index 242e5b5f4192..be8c39480e17 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedStopFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedStopFilterFactory.java @@ -140,7 +140,7 @@ public void testManagedStopwords() throws Exception { "/response/lst[@name='error']/int[@name='code'] = '404'"); // add the new field - assertJPut("/schema/fields/" + newFieldName, json("{'type':'managed_en'}"), + assertJPost("/schema/fields", "{add-field : { name :managed_en_field, type : managed_en}}", "/responseHeader/status==0"); // make sure the new field exists now diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java index 9afaf6fb0ca6..26fcde1e44c7 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java @@ -90,8 +90,8 @@ public void testManagedSynonyms() throws Exception { JSONUtil.toJSON(syns), "/responseHeader/status==0"); - assertJQ(endpoint, - "/synonymMappings/managedMap/happy==['cheerful','glad','joyful']"); + assertJQ(endpoint, + "/synonymMappings/managedMap/happy==['cheerful','glad','joyful']"); // request to a specific mapping assertJQ(endpoint+"/happy", @@ -146,7 +146,7 @@ public void testManagedSynonyms() throws Exception { "/response/lst[@name='error']/int[@name='code'] = '404'"); // add the new field - assertJPut("/schema/fields/" + newFieldName, json("{'type':'managed_en'}"), + assertJPost("/schema", "{ add-field : { name: managed_en_field, type : managed_en}}", "/responseHeader/status==0"); // make sure the new field exists now diff --git a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java index 3d0d120c0ad5..0993ba113db9 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java @@ -37,11 +37,12 @@ import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.restlet.ext.servlet.ServerServlet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +@Ignore public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String SUCCESS_XPATH = "/response/lst[@name='responseHeader']/int[@name='status'][.='0']"; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java index b723abf467d2..72051b123aad 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java @@ -575,10 +575,6 @@ public void testAddFieldTypeAccuracy() throws Exception { assertThat("solr.TextField", is(equalTo(newFieldTypeRepresentation.getAttributes().get("class")))); assertThat(analyzerDefinition.getTokenizer().get("class"), is(equalTo(newFieldTypeRepresentation.getAnalyzer().getTokenizer().get("class")))); - assertTrue(newFieldTypeRepresentation.getFields().size() == 1); - assertTrue(newFieldTypeRepresentation.getFields().contains(fieldName)); - assertTrue(newFieldTypeRepresentation.getDynamicFields().size() == 1); - assertTrue(newFieldTypeRepresentation.getDynamicFields().contains(dynamicFieldName)); } @Test From 549e6d7c497b1da0368f012ef0a2521cf0548582 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 7 Mar 2016 18:12:10 -0500 Subject: [PATCH 0051/1113] LUCENE-7071: reduce byte copying costs of OfflineSorter --- lucene/CHANGES.txt | 5 +++ .../org/apache/lucene/util/ByteBlockPool.java | 22 ++++++++++ .../org/apache/lucene/util/BytesRefArray.java | 41 +++++++++++++++---- .../org/apache/lucene/util/OfflineSorter.java | 3 +- .../search/suggest/SortedInputIterator.java | 16 ++++---- 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3647d5d191cb..290421a65cc6 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -6,6 +6,11 @@ http://s.apache.org/luceneversions ======================= Lucene 6.1.0 ======================= (No Changes) +Optimizations + +* LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% + speedup on merging 2D LatLonPoint values (Mike McCandless) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java index 5f8fd4180d6b..6bb12bdfea66 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java +++ b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java @@ -280,6 +280,28 @@ public int allocSlice(final byte[] slice, final int upto) { return newUpto+3; } + /** Fill the provided {@link BytesRef} with the bytes at the specified offset/length slice. + * This will avoid copying the bytes, if the slice fits into a single block; otherwise, it uses + * the provided {@linkl BytesRefBuilder} to copy bytes over. */ + void setBytesRef(BytesRefBuilder builder, BytesRef result, long offset, int length) { + result.length = length; + + int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT); + byte[] buffer = buffers[bufferIndex]; + int pos = (int) (offset & BYTE_BLOCK_MASK); + if (pos + length <= BYTE_BLOCK_SIZE) { + // common case where the slice lives in a single block: just reference the buffer directly without copying + result.bytes = buffer; + result.offset = pos; + } else { + // uncommon case: the slice spans at least 2 blocks, so we must copy the bytes: + builder.grow(length); + result.bytes = builder.get().bytes; + result.offset = 0; + readBytes(offset, result.bytes, 0, length); + } + } + // Fill in a BytesRef from term's length & bytes encoded in // byte block public void setBytesRef(BytesRef term, int textStart) { diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java b/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java index 47ca52b7145e..a19b7da5abed 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java +++ b/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java @@ -108,7 +108,23 @@ public BytesRef get(BytesRefBuilder spare, int index) { } throw new IndexOutOfBoundsException("index " + index + " must be less than the size: " + lastElement); - + } + + /** Used only by sort below, to set a {@link BytesRef} with the specified slice, avoiding copying bytes in the common case when the slice + * is contained in a single block in the byte block pool. */ + private void setBytesRef(BytesRefBuilder spare, BytesRef result, int index) { + if (index < lastElement) { + int offset = offsets[index]; + int length; + if (index == lastElement - 1) { + length = currentOffset - offset; + } else { + length = offsets[index + 1] - offset; + } + pool.setBytesRef(spare, result, offset, length); + } else { + throw new IndexOutOfBoundsException("index " + index + " must be less than the size: " + lastElement); + } } private int[] sort(final Comparator comp) { @@ -127,25 +143,30 @@ protected void swap(int i, int j) { @Override protected int compare(int i, int j) { final int idx1 = orderedEntries[i], idx2 = orderedEntries[j]; - return comp.compare(get(scratch1, idx1), get(scratch2, idx2)); + setBytesRef(scratch1, scratchBytes1, idx1); + setBytesRef(scratch2, scratchBytes2, idx2); + return comp.compare(scratchBytes1, scratchBytes2); } @Override protected void setPivot(int i) { final int index = orderedEntries[i]; - pivot = get(pivotBuilder, index); + setBytesRef(pivotBuilder, pivot, index); } @Override protected int comparePivot(int j) { final int index = orderedEntries[j]; - return comp.compare(pivot, get(scratch2, index)); + setBytesRef(scratch2, scratchBytes2, index); + return comp.compare(pivot, scratchBytes2); } - private BytesRef pivot; - private final BytesRefBuilder pivotBuilder = new BytesRefBuilder(), - scratch1 = new BytesRefBuilder(), - scratch2 = new BytesRefBuilder(); + private final BytesRef pivot = new BytesRef(); + private final BytesRef scratchBytes1 = new BytesRef(); + private final BytesRef scratchBytes2 = new BytesRef(); + private final BytesRefBuilder pivotBuilder = new BytesRefBuilder(); + private final BytesRefBuilder scratch1 = new BytesRefBuilder(); + private final BytesRefBuilder scratch2 = new BytesRefBuilder(); }.sort(0, size()); return orderedEntries; } @@ -173,6 +194,7 @@ public BytesRefIterator iterator() { */ public BytesRefIterator iterator(final Comparator comp) { final BytesRefBuilder spare = new BytesRefBuilder(); + final BytesRef result = new BytesRef(); final int size = size(); final int[] indices = comp == null ? null : sort(comp); return new BytesRefIterator() { @@ -181,7 +203,8 @@ public BytesRefIterator iterator(final Comparator comp) { @Override public BytesRef next() { if (pos < size) { - return get(spare, indices == null ? pos++ : indices[pos++]); + setBytesRef(spare, result, indices == null ? pos++ : indices[pos++]); + return result; } return null; } diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java index 283dc1f90cb6..18e421b7bedf 100644 --- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java @@ -282,7 +282,6 @@ public String sort(String inputFileName) throws IOException { /** Sort a single partition in-memory. */ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOException { - BytesRefArray data = this.buffer; try (IndexOutput tempFile = trackingDir.createTempOutput(tempFileNamePrefix, "sort", IOContext.DEFAULT); ByteSequencesWriter out = getWriter(tempFile);) { @@ -299,7 +298,7 @@ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOEx } // Clean up the buffer for the next partition. - data.clear(); + buffer.clear(); return tempFile.getName(); } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java index 2be175943653..bc71e45ffd53 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java @@ -146,6 +146,7 @@ public boolean hasContexts() { @Override public int compare(BytesRef left, BytesRef right) { // Make shallow copy in case decode changes the BytesRef: + assert left != right; leftScratch.bytes = left.bytes; leftScratch.offset = left.offset; leftScratch.length = left.length; @@ -245,24 +246,24 @@ protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, by /** decodes the weight at the current position */ protected long decode(BytesRef scratch, ByteArrayDataInput tmpInput) { - tmpInput.reset(scratch.bytes); + tmpInput.reset(scratch.bytes, scratch.offset, scratch.length); tmpInput.skipBytes(scratch.length - 8); // suggestion - scratch.length -= 8; // long + scratch.length -= Long.BYTES; // long return tmpInput.readLong(); } /** decodes the contexts at the current position */ protected Set decodeContexts(BytesRef scratch, ByteArrayDataInput tmpInput) { - tmpInput.reset(scratch.bytes); + tmpInput.reset(scratch.bytes, scratch.offset, scratch.length); tmpInput.skipBytes(scratch.length - 2); //skip to context set size short ctxSetSize = tmpInput.readShort(); scratch.length -= 2; final Set contextSet = new HashSet<>(); for (short i = 0; i < ctxSetSize; i++) { - tmpInput.setPosition(scratch.length - 2); + tmpInput.setPosition(scratch.offset + scratch.length - 2); short curContextLength = tmpInput.readShort(); scratch.length -= 2; - tmpInput.setPosition(scratch.length - curContextLength); + tmpInput.setPosition(scratch.offset + scratch.length - curContextLength); BytesRef contextSpare = new BytesRef(curContextLength); tmpInput.readBytes(contextSpare.bytes, 0, curContextLength); contextSpare.length = curContextLength; @@ -274,10 +275,11 @@ protected Set decodeContexts(BytesRef scratch, ByteArrayDataInput tmpI /** decodes the payload at the current position */ protected BytesRef decodePayload(BytesRef scratch, ByteArrayDataInput tmpInput) { - tmpInput.reset(scratch.bytes); + tmpInput.reset(scratch.bytes, scratch.offset, scratch.length); tmpInput.skipBytes(scratch.length - 2); // skip to payload size short payloadLength = tmpInput.readShort(); // read payload size - tmpInput.setPosition(scratch.length - 2 - payloadLength); // setPosition to start of payload + assert payloadLength >= 0: payloadLength; + tmpInput.setPosition(scratch.offset + scratch.length - 2 - payloadLength); // setPosition to start of payload BytesRef payloadScratch = new BytesRef(payloadLength); tmpInput.readBytes(payloadScratch.bytes, 0, payloadLength); // read payload payloadScratch.length = payloadLength; From d7ee7c661529b2a8c81c3cc52d581b4a5f19b5b8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 7 Mar 2016 16:12:15 -0500 Subject: [PATCH 0052/1113] LUCENE-7073: fix FieldType issues with Points --- .../benchmark/byTask/feeds/DocMaker.java | 44 ++++++++----------- .../byTask/tasks/ReadTokensTask.java | 13 ++---- .../lucene/codecs/lucene60/package-info.java | 7 ++- .../org/apache/lucene/document/FieldType.java | 24 +++++++--- .../document/SortedNumericDocValuesField.java | 4 +- .../apache/lucene/document/TestFieldType.java | 15 +++++-- .../apache/lucene/index/TestPointValues.java | 3 +- .../lucene/facet/range/DoubleRange.java | 3 +- 8 files changed, 59 insertions(+), 54 deletions(-) diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java index f2c863cf7985..4afafc321c77 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java @@ -35,13 +35,12 @@ import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyLongField; -import org.apache.lucene.document.LegacyFloatField; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -119,8 +118,8 @@ public DocState(boolean reuseFields, FieldType ft, FieldType bodyFt) { fields.put(ID_FIELD, new StringField(ID_FIELD, "", Field.Store.YES)); fields.put(NAME_FIELD, new Field(NAME_FIELD, "", ft)); - numericFields.put(DATE_MSEC_FIELD, new LegacyLongField(DATE_MSEC_FIELD, 0L, Field.Store.NO)); - numericFields.put(TIME_SEC_FIELD, new LegacyIntField(TIME_SEC_FIELD, 0, Field.Store.NO)); + numericFields.put(DATE_MSEC_FIELD, new LongPoint(DATE_MSEC_FIELD, 0L)); + numericFields.put(TIME_SEC_FIELD, new IntPoint(TIME_SEC_FIELD, 0)); doc = new Document(); } else { @@ -148,7 +147,7 @@ Field getField(String name, FieldType ft) { return f; } - Field getNumericField(String name, LegacyNumericType type) { + Field getNumericField(String name, Class numericType) { Field f; if (reuseFields) { f = numericFields.get(name); @@ -157,21 +156,16 @@ Field getNumericField(String name, LegacyNumericType type) { } if (f == null) { - switch(type) { - case INT: - f = new LegacyIntField(name, 0, Field.Store.NO); - break; - case LONG: - f = new LegacyLongField(name, 0L, Field.Store.NO); - break; - case FLOAT: - f = new LegacyFloatField(name, 0.0F, Field.Store.NO); - break; - case DOUBLE: - f = new LegacyDoubleField(name, 0.0, Field.Store.NO); - break; - default: - throw new AssertionError("Cannot get here"); + if (numericType.equals(Integer.class)) { + f = new IntPoint(name, 0); + } else if (numericType.equals(Long.class)) { + f = new LongPoint(name, 0L); + } else if (numericType.equals(Float.class)) { + f = new FloatPoint(name, 0.0F); + } else if (numericType.equals(Double.class)) { + f = new DoublePoint(name, 0.0); + } else { + throw new UnsupportedOperationException("Unsupported numeric type: " + numericType); } if (reuseFields) { numericFields.put(name, f); @@ -278,14 +272,14 @@ private Document createDocument(DocData docData, int size, int cnt) throws Unsup date = new Date(); } - Field dateField = ds.getNumericField(DATE_MSEC_FIELD, FieldType.LegacyNumericType.LONG); + Field dateField = ds.getNumericField(DATE_MSEC_FIELD, Long.class); dateField.setLongValue(date.getTime()); doc.add(dateField); util.cal.setTime(date); final int sec = util.cal.get(Calendar.HOUR_OF_DAY)*3600 + util.cal.get(Calendar.MINUTE)*60 + util.cal.get(Calendar.SECOND); - Field timeSecField = ds.getNumericField(TIME_SEC_FIELD, LegacyNumericType.INT); + Field timeSecField = ds.getNumericField(TIME_SEC_FIELD, Integer.class); timeSecField.setIntValue(sec); doc.add(timeSecField); diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java index 4950d418e713..2e44b99fe3eb 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java @@ -26,11 +26,7 @@ import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.benchmark.byTask.feeds.DocMaker; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; /** @@ -73,11 +69,8 @@ public int doLogic() throws Exception { Analyzer analyzer = getRunData().getAnalyzer(); int tokenCount = 0; for(final IndexableField field : fields) { - if (!field.fieldType().tokenized() || - field instanceof LegacyIntField || - field instanceof LegacyLongField || - field instanceof LegacyFloatField || - field instanceof LegacyDoubleField) { + if (field.fieldType().indexOptions() == IndexOptions.NONE || + field.fieldType().tokenized() == false) { continue; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/package-info.java index 64531f5c34f9..03a17ba2e38c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/package-info.java @@ -194,9 +194,9 @@ * *
  • * {@link org.apache.lucene.codecs.lucene60.Lucene60PointsFormat Point values}. - * Optional pair of files, recording dimesionally indexed fields, to enable fast + * Optional pair of files, recording dimensionally indexed fields, to enable fast * numeric range filtering and large numeric values like BigInteger and BigDecimal (1D) - * and geo shape intersection (2D, 3D). + * and geographic shape intersection (2D, 3D). *
    • * *

    Details on each of these are provided in their linked pages.

    @@ -396,6 +396,9 @@ * contain the zlib-crc32 checksum of the file. *
  • In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) * that is suitable for faceting/sorting/analytics. + *
  • In version 5.4, DocValues have been improved to store more information on disk: + * addresses for binary fields and ord indexes for multi-valued fields. + *
  • In version 6.0, Points were added, for multi-dimensional range/distance search. *
    • * * diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java index 1dfa8792a752..ae84016a327e 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java +++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.index.PointValues; import org.apache.lucene.util.LegacyNumericUtils; /** @@ -75,8 +76,8 @@ public FieldType(FieldType ref) { this.numericType = ref.numericType(); this.numericPrecisionStep = ref.numericPrecisionStep(); this.docValuesType = ref.docValuesType(); - this.dimensionCount = dimensionCount; - this.dimensionNumBytes = dimensionNumBytes; + this.dimensionCount = ref.dimensionCount; + this.dimensionNumBytes = ref.dimensionNumBytes; // Do not copy frozen! } @@ -365,18 +366,24 @@ public int numericPrecisionStep() { */ public void setDimensions(int dimensionCount, int dimensionNumBytes) { if (dimensionCount < 0) { - throw new IllegalArgumentException("pointDimensionCount must be >= 0; got " + dimensionCount); + throw new IllegalArgumentException("dimensionCount must be >= 0; got " + dimensionCount); + } + if (dimensionCount > PointValues.MAX_DIMENSIONS) { + throw new IllegalArgumentException("dimensionCount must be <= " + PointValues.MAX_DIMENSIONS + "; got " + dimensionCount); } if (dimensionNumBytes < 0) { - throw new IllegalArgumentException("pointNumBytes must be >= 0; got " + dimensionNumBytes); + throw new IllegalArgumentException("dimensionNumBytes must be >= 0; got " + dimensionNumBytes); + } + if (dimensionCount > PointValues.MAX_NUM_BYTES) { + throw new IllegalArgumentException("dimensionNumBytes must be <= " + PointValues.MAX_NUM_BYTES + "; got " + dimensionNumBytes); } if (dimensionCount == 0) { if (dimensionNumBytes != 0) { - throw new IllegalArgumentException("when pointDimensionCount is 0 pointNumBytes must 0; got " + dimensionNumBytes); + throw new IllegalArgumentException("when dimensionCount is 0, dimensionNumBytes must 0; got " + dimensionNumBytes); } } else if (dimensionNumBytes == 0) { if (dimensionCount != 0) { - throw new IllegalArgumentException("when pointNumBytes is 0 pointDimensionCount must 0; got " + dimensionCount); + throw new IllegalArgumentException("when dimensionNumBytes is 0, dimensionCount must 0; got " + dimensionCount); } } @@ -484,6 +491,8 @@ public void setDocValuesType(DocValuesType type) { public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + dimensionCount; + result = prime * result + dimensionNumBytes; result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); result = prime * result + indexOptions.hashCode(); result = prime * result + numericPrecisionStep; @@ -504,6 +513,8 @@ public boolean equals(Object obj) { if (obj == null) return false; if (getClass() != obj.getClass()) return false; FieldType other = (FieldType) obj; + if (dimensionCount != other.dimensionCount) return false; + if (dimensionNumBytes != other.dimensionNumBytes) return false; if (docValuesType != other.docValuesType) return false; if (indexOptions != other.indexOptions) return false; if (numericPrecisionStep != other.numericPrecisionStep) return false; @@ -517,5 +528,4 @@ public boolean equals(Object obj) { if (tokenized != other.tokenized) return false; return true; } - } diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java index 40ceb2595c15..cbba218f7de6 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java @@ -31,10 +31,10 @@ * *

    * Note that if you want to encode doubles or floats with proper sort order, - * you will need to encode them with {@link org.apache.lucene.util.LegacyNumericUtils}: + * you will need to encode them with {@link org.apache.lucene.util.NumericUtils}: * * 

    - *   document.add(new SortedNumericDocValuesField(name, LegacyNumericUtils.floatToSortableInt(-5.3f)));
+ *   document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(-5.3f)));
  *
    * *

    diff --git a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java index 65f32d843e60..c49d4e013bee 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType.LegacyNumericType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.PointValues; import org.apache.lucene.util.LuceneTestCase; import com.carrotsearch.randomizedtesting.generators.RandomPicks; @@ -70,6 +71,10 @@ public void testEquals() throws Exception { FieldType ft10 = new FieldType(); ft10.setStoreTermVectors(true); assertFalse(ft10.equals(ft)); + + FieldType ft11 = new FieldType(); + ft11.setDimensions(1, 4); + assertFalse(ft11.equals(ft)); } public void testPointsToString() { @@ -90,14 +95,16 @@ private static Object randomValue(Class clazz) { } private static FieldType randomFieldType() throws Exception { + // setDimensions handled special as values must be in-bounds. + Method setDimensionsMethod = FieldType.class.getMethod("setDimensions", int.class, int.class); FieldType ft = new FieldType(); for (Method method : FieldType.class.getMethods()) { - if ((method.getModifiers() & Modifier.PUBLIC) != 0 && method.getName().startsWith("set")) { + if (method.getName().startsWith("set")) { final Class[] parameterTypes = method.getParameterTypes(); final Object[] args = new Object[parameterTypes.length]; - if (method.getName().equals("setPointDimensions")) { - args[0] = 1 + random().nextInt(15); - args[1] = 1 + random().nextInt(100); + if (method.equals(setDimensionsMethod)) { + args[0] = 1 + random().nextInt(PointValues.MAX_DIMENSIONS); + args[1] = 1 + random().nextInt(PointValues.MAX_NUM_BYTES); } else { for (int i = 0; i < args.length; ++i) { args[i] = randomValue(parameterTypes[i]); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 9faa0bcfcce9..7231b1afc6fa 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -385,9 +385,8 @@ public void testIllegalTooManyDimensions() throws Exception { for(int i=0;i { - w.addDocument(doc); + doc.add(new BinaryPoint("dim", values)); }); Document doc2 = new Document(); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java index 7585708dde07..6f005ed4a1ae 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; /** Represents a range over double values. @@ -50,7 +49,7 @@ public DoubleRange(String label, double minIn, boolean minInclusive, double maxI super(label); // TODO: if DoubleDocValuesField used - // LegacyNumericUtils.doubleToSortableLong format (instead of + // NumericUtils.doubleToSortableLong format (instead of // Double.doubleToRawLongBits) we could do comparisons // in long space From 3a31a8c7686b27e90f9683c4d3a0cdd2e89d3e91 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 7 Mar 2016 17:29:46 -0500 Subject: [PATCH 0053/1113] LUCENE-7056: Geo3D package re-org (cherry picked from commit 0093e26) --- lucene/CHANGES.txt | 2 ++ .../lucene/spatial/spatial4j/Geo3dShape.java | 12 ++++----- .../spatial/spatial4j/Geo3dRptTest.java | 14 +++++----- .../Geo3dShapeRectRelationTestCase.java | 18 ++++++------- ...Geo3dShapeSphereModelRectRelationTest.java | 16 +++++------ .../Geo3dShapeWGS84ModelRectRelationTest.java | 16 +++++------ .../{geo3d => spatial3d}/Geo3DPoint.java | 6 +++-- .../{geo3d => spatial3d}/Geo3DUtil.java | 2 +- .../PointInGeo3DShapeQuery.java | 7 ++++- .../geom}/ArcDistance.java | 2 +- .../geom}/BasePlanetObject.java | 2 +- .../geom}/BaseXYZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/Bounds.java | 2 +- .../geom}/DistanceStyle.java | 2 +- .../{geo3d => spatial3d/geom}/GeoArea.java | 2 +- .../geom}/GeoAreaFactory.java | 4 +-- .../{geo3d => spatial3d/geom}/GeoBBox.java | 2 +- .../geom}/GeoBBoxFactory.java | 4 +-- .../geom}/GeoBaseBBox.java | 2 +- .../geom}/GeoBaseCircle.java | 2 +- .../geom}/GeoBaseDistanceShape.java | 2 +- .../geom}/GeoBaseMembershipShape.java | 2 +- .../geom}/GeoBasePolygon.java | 2 +- .../geom}/GeoBaseShape.java | 2 +- .../{geo3d => spatial3d/geom}/GeoCircle.java | 2 +- .../geom}/GeoCircleFactory.java | 2 +- .../geom}/GeoCompositeMembershipShape.java | 2 +- .../geom}/GeoCompositePolygon.java | 2 +- .../geom}/GeoConvexPolygon.java | 2 +- .../geom}/GeoDegenerateHorizontalLine.java | 2 +- .../geom}/GeoDegenerateLatitudeZone.java | 2 +- .../geom}/GeoDegenerateLongitudeSlice.java | 2 +- .../geom}/GeoDegeneratePoint.java | 2 +- .../geom}/GeoDegenerateVerticalLine.java | 2 +- .../geom}/GeoDistance.java | 2 +- .../geom}/GeoDistanceShape.java | 2 +- .../geom}/GeoLatitudeZone.java | 2 +- .../geom}/GeoLongitudeSlice.java | 2 +- .../geom}/GeoMembershipShape.java | 2 +- .../geom}/GeoNorthLatitudeZone.java | 2 +- .../geom}/GeoNorthRectangle.java | 2 +- .../geom}/GeoOutsideDistance.java | 2 +- .../{geo3d => spatial3d/geom}/GeoPath.java | 2 +- .../{geo3d => spatial3d/geom}/GeoPoint.java | 4 +-- .../{geo3d => spatial3d/geom}/GeoPolygon.java | 2 +- .../geom}/GeoPolygonFactory.java | 2 +- .../geom}/GeoRectangle.java | 2 +- .../{geo3d => spatial3d/geom}/GeoShape.java | 2 +- .../geom}/GeoSizeable.java | 2 +- .../geom}/GeoSouthLatitudeZone.java | 2 +- .../geom}/GeoSouthRectangle.java | 2 +- .../geom}/GeoStandardCircle.java | 2 +- .../GeoWideDegenerateHorizontalLine.java | 2 +- .../geom}/GeoWideLongitudeSlice.java | 2 +- .../geom}/GeoWideNorthRectangle.java | 2 +- .../geom}/GeoWideRectangle.java | 2 +- .../geom}/GeoWideSouthRectangle.java | 2 +- .../{geo3d => spatial3d/geom}/GeoWorld.java | 2 +- .../geom}/LatLonBounds.java | 2 +- .../geom}/LinearDistance.java | 2 +- .../geom}/LinearSquaredDistance.java | 2 +- .../{geo3d => spatial3d/geom}/Membership.java | 2 +- .../geom}/NormalDistance.java | 2 +- .../geom}/NormalSquaredDistance.java | 2 +- .../{geo3d => spatial3d/geom}/Plane.java | 2 +- .../geom}/PlanetModel.java | 4 +-- .../{geo3d => spatial3d/geom}/SidedPlane.java | 2 +- .../geom}/StandardXYZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/Tools.java | 2 +- .../{geo3d => spatial3d/geom}/Vector.java | 2 +- .../{geo3d => spatial3d/geom}/XYZBounds.java | 2 +- .../{geo3d => spatial3d/geom}/XYZSolid.java | 2 +- .../geom}/XYZSolidFactory.java | 4 +-- .../{geo3d => spatial3d/geom}/XYdZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/XdYZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/XdYdZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/dXYZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/dXYdZSolid.java | 2 +- .../{geo3d => spatial3d/geom}/dXdYZSolid.java | 2 +- .../geom}/dXdYdZSolid.java | 2 +- .../lucene/spatial3d/geom/package-info.java | 22 +++++++++++++++ .../{geo3d => spatial3d}/package-info.java | 4 +-- lucene/spatial3d/src/java/overview.html | 3 ++- .../{geo3d => spatial3d}/TestGeo3DPoint.java | 13 +++++++-- .../geom}/GeoBBoxTest.java | 2 +- .../geom}/GeoCircleTest.java | 9 ++----- .../geom}/GeoConvexPolygonTest.java | 2 +- .../geom}/GeoModelTest.java | 2 +- .../geom}/GeoPathTest.java | 2 +- .../lucene/spatial3d/geom}/GeoPointTest.java | 27 +++++++++---------- .../geom}/GeoPolygonTest.java | 2 +- .../{geo3d => spatial3d/geom}/PlaneTest.java | 2 +- .../geom}/XYZSolidTest.java | 2 +- 93 files changed, 185 insertions(+), 152 deletions(-) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d}/Geo3DPoint.java (95%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d}/Geo3DUtil.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d}/PointInGeo3DShapeQuery.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/ArcDistance.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/BasePlanetObject.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/BaseXYZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/Bounds.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/DistanceStyle.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoArea.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoAreaFactory.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBBox.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBBoxFactory.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBaseBBox.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBaseCircle.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBaseDistanceShape.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBaseMembershipShape.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBasePolygon.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBaseShape.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoCircle.java (95%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoCircleFactory.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoCompositeMembershipShape.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoCompositePolygon.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoConvexPolygon.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDegenerateHorizontalLine.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDegenerateLatitudeZone.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDegenerateLongitudeSlice.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDegeneratePoint.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDegenerateVerticalLine.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDistance.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoDistanceShape.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoLatitudeZone.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoLongitudeSlice.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoMembershipShape.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoNorthLatitudeZone.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoNorthRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoOutsideDistance.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPath.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPoint.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPolygon.java (95%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPolygonFactory.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoShape.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoSizeable.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoSouthLatitudeZone.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoSouthRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoStandardCircle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWideDegenerateHorizontalLine.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWideLongitudeSlice.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWideNorthRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWideRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWideSouthRectangle.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/GeoWorld.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/LatLonBounds.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/LinearDistance.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/LinearSquaredDistance.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/Membership.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/NormalDistance.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/NormalSquaredDistance.java (97%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/Plane.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/PlanetModel.java (98%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/SidedPlane.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/StandardXYZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/Tools.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/Vector.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XYZBounds.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XYZSolid.java (95%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XYZSolidFactory.java (96%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XYdZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XdYZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/XdYdZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/dXYZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/dXYdZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/dXdYZSolid.java (99%) rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d/geom}/dXdYdZSolid.java (99%) create mode 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/package-info.java rename lucene/spatial3d/src/java/org/apache/lucene/{geo3d => spatial3d}/package-info.java (83%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d}/TestGeo3DPoint.java (98%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoBBoxTest.java (99%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoCircleTest.java (99%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoConvexPolygonTest.java (98%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoModelTest.java (99%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPathTest.java (99%) rename lucene/{spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d => spatial3d/src/test/org/apache/lucene/spatial3d/geom}/GeoPointTest.java (69%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/GeoPolygonTest.java (99%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/PlaneTest.java (98%) rename lucene/spatial3d/src/test/org/apache/lucene/{geo3d => spatial3d/geom}/XYZSolidTest.java (99%) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 290421a65cc6..ca59e6b8f1e1 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -126,6 +126,8 @@ API Changes * LUCENE-7072: Geo3DPoint always uses WGS84 planet model. (Robert Muir, Mike McCandless) +* LUCENE-7056: Geo3D classes are in different packages now. (David Smiley) + Optimizations * LUCENE-6891: Use prefix coding when writing points in diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java index 518fb32b4f17..9fa6d8e5d5fc 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/spatial4j/Geo3dShape.java @@ -23,12 +23,12 @@ import org.locationtech.spatial4j.shape.Shape; import org.locationtech.spatial4j.shape.SpatialRelation; import org.locationtech.spatial4j.shape.impl.RectangleImpl; -import org.apache.lucene.geo3d.LatLonBounds; -import org.apache.lucene.geo3d.GeoArea; -import org.apache.lucene.geo3d.GeoAreaFactory; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.GeoShape; -import org.apache.lucene.geo3d.PlanetModel; +import org.apache.lucene.spatial3d.geom.LatLonBounds; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; /** * A Spatial4j Shape wrapping a {@link GeoShape} ("Geo3D") -- a 3D planar geometry based Spatial4j Shape implementation. diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java index d26bb29a5979..e62b857faeb7 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java @@ -32,13 +32,13 @@ import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.serialized.SerializedDVStrategy; -import org.apache.lucene.geo3d.GeoBBoxFactory; -import org.apache.lucene.geo3d.GeoStandardCircle; -import org.apache.lucene.geo3d.GeoPath; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.GeoPolygonFactory; -import org.apache.lucene.geo3d.GeoShape; -import org.apache.lucene.geo3d.PlanetModel; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Test; import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java index 134b8c753666..d58985f4c558 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java @@ -25,15 +25,15 @@ import org.locationtech.spatial4j.shape.Circle; import org.locationtech.spatial4j.shape.Point; import org.locationtech.spatial4j.shape.RectIntersectionTestHelper; -import org.apache.lucene.geo3d.LatLonBounds; -import org.apache.lucene.geo3d.GeoBBox; -import org.apache.lucene.geo3d.GeoBBoxFactory; -import org.apache.lucene.geo3d.GeoStandardCircle; -import org.apache.lucene.geo3d.GeoPath; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.GeoPolygonFactory; -import org.apache.lucene.geo3d.GeoShape; -import org.apache.lucene.geo3d.PlanetModel; +import org.apache.lucene.spatial3d.geom.LatLonBounds; +import org.apache.lucene.spatial3d.geom.GeoBBox; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Rule; import org.junit.Test; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java index 2d958231e6d2..3bce480fe525 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java @@ -20,14 +20,14 @@ import java.util.List; import org.locationtech.spatial4j.shape.Rectangle; -import org.apache.lucene.geo3d.GeoArea; -import org.apache.lucene.geo3d.GeoBBox; -import org.apache.lucene.geo3d.GeoBBoxFactory; -import org.apache.lucene.geo3d.GeoStandardCircle; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.GeoPolygonFactory; -import org.apache.lucene.geo3d.GeoShape; -import org.apache.lucene.geo3d.PlanetModel; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoBBox; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Test; public class Geo3dShapeSphereModelRectRelationTest extends Geo3dShapeRectRelationTestCase { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java index 3b026c36a745..b59d7df50a9a 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java @@ -16,14 +16,14 @@ */ package org.apache.lucene.spatial.spatial4j; -import org.apache.lucene.geo3d.GeoArea; -import org.apache.lucene.geo3d.GeoBBox; -import org.apache.lucene.geo3d.GeoBBoxFactory; -import org.apache.lucene.geo3d.GeoCircle; -import org.apache.lucene.geo3d.GeoStandardCircle; -import org.apache.lucene.geo3d.GeoPath; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.PlanetModel; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoBBox; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoCircle; +import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Test; public class Geo3dShapeWGS84ModelRectRelationTest extends Geo3dShapeRectRelationTestCase { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java similarity index 95% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index cde87f3c77ba..cd2c79a15d0f 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.RamUsageEstimator; /** * Add this to a document to index lat/lon or x/y/z point, indexed as a 3D point. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DUtil.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DUtil.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java index 34880a139ee3..0a0bf30039c8 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Geo3DUtil.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d; class Geo3DUtil { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 9e2132d680db..9df8752b1353 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -14,10 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d; import java.io.IOException; +import org.apache.lucene.spatial3d.geom.BasePlanetObject; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.PointValues.Relation; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/ArcDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/ArcDistance.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/ArcDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/ArcDistance.java index c49fd1fe1ca7..bb60be07b719 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/ArcDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/ArcDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Arc distance computation style. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BasePlanetObject.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BasePlanetObject.java index c64b974fd1fb..5cd5acca8ae9 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BasePlanetObject.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BasePlanetObject.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * All Geo3D shapes can derive from this base class, which furnishes diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BaseXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/BaseXYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java index 52bd5da62093..16b52ccee75d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/BaseXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Base class of a family of 3D rectangles, bounded on six sides by X,Y,Z limits diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Bounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Bounds.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java index 67172205f9b7..4f7c66309885 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Bounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * An interface for accumulating bounds information. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/DistanceStyle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/DistanceStyle.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/DistanceStyle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/DistanceStyle.java index 28056cb1841f..8c8658dda669 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/DistanceStyle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/DistanceStyle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Distance computation styles, supporting various ways of computing diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoArea.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoArea.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoArea.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoArea.java index 424e494ec147..5a6db0da6761 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoArea.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoArea.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * A GeoArea represents a standard 2-D breakdown of a part of sphere. It can diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoAreaFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoAreaFactory.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoAreaFactory.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoAreaFactory.java index 24dd211f1f2a..0c3caa994398 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoAreaFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoAreaFactory.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** - * Factory for {@link org.apache.lucene.geo3d.GeoArea}. + * Factory for {@link GeoArea}. * * @lucene.experimental */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBox.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBox.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBox.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBox.java index 10a2388e9557..0ae242592d61 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBox.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBox.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * All bounding box shapes have this interface in common. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBoxFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBoxFactory.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java index 9a02ab9d28e8..de7493e5c50e 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBBoxFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** - * Factory for {@link org.apache.lucene.geo3d.GeoBBox}. + * Factory for {@link GeoBBox}. * * @lucene.experimental */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseBBox.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseBBox.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java index f40a0a14215e..7190cdccff1b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseBBox.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * All bounding box shapes can derive from this base class, which furnishes diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseCircle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java index 8c306d7ac4cf..75219fd3d32a 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * GeoCircles have all the characteristics of GeoBaseDistanceShapes, plus GeoSizeable. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseDistanceShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseDistanceShape.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseDistanceShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseDistanceShape.java index 1c8306adb4a7..39dcf9616adf 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseDistanceShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseDistanceShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Distance shapes have capabilities of both geohashing and distance diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseMembershipShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseMembershipShape.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseMembershipShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseMembershipShape.java index a6bba8f4063e..831a7c6af9e3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseMembershipShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseMembershipShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Membership shapes have capabilities of both geohashing and membership diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBasePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBasePolygon.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java index 50ad0dcc9ca9..ba221ae88530 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBasePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * GeoBasePolygon objects are the base class of most GeoPolygon objects. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseShape.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseShape.java index 146cfe857148..54896fc67f37 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoBaseShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Base extended shape object. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircle.java similarity index 95% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircle.java index 154cdc4c084c..b05dff6af6c2 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Interface describing circular area with a center and radius. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircleFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircleFactory.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircleFactory.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircleFactory.java index 2bb8ffc97fa6..ee75179eb997 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCircleFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCircleFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Class which constructs a GeoCircle representing an arbitrary circle. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositeMembershipShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositeMembershipShape.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositeMembershipShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositeMembershipShape.java index 25bdda096a1c..9747edad6fa6 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositeMembershipShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositeMembershipShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.List; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositePolygon.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositePolygon.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositePolygon.java index b537590658b7..920d3fb9db40 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoCompositePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoCompositePolygon.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * GeoCompositePolygon is a specific implementation of GeoMembershipShape, which implements GeoPolygon explicitly. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoConvexPolygon.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index fc07c4bb4ad2..fb024b6efee2 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.BitSet; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateHorizontalLine.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateHorizontalLine.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java index 6644f0db2b84..b7de0c2106ae 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateHorizontalLine.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Degenerate bounding box limited on two sides (left lon, right lon). diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLatitudeZone.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java index 489073389145..e79412312b50 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This GeoBBox represents an area rectangle of one specific latitude with diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLongitudeSlice.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java index b5eb90237de0..0bb7b90dfc2f 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Degenerate longitude slice. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegeneratePoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegeneratePoint.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java index 63670d7f83bf..fcd20371004b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegeneratePoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This class represents a degenerate point bounding box. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateVerticalLine.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateVerticalLine.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateVerticalLine.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateVerticalLine.java index f21f7747d04c..dff53b4aa1d2 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDegenerateVerticalLine.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateVerticalLine.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Degenerate bounding box limited on two sides (top lat, bottom lat). diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistance.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistance.java index 899a687b431d..d41dd5146471 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * An implementer of this interface is capable of computing the described "distance" values, diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistanceShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistanceShape.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistanceShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistanceShape.java index 1e82f4828257..e7b0348c185d 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoDistanceShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDistanceShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Distance shapes have capabilities of both geohashing and distance diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLatitudeZone.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java index 3fc4423ec24d..912ca3205f84 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This GeoBBox represents an area rectangle limited only in latitude. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLongitudeSlice.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java index f5de7e766c8a..458cf8bef5bb 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box limited on left and right. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoMembershipShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoMembershipShape.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoMembershipShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoMembershipShape.java index 54b25513c558..2c479714ec34 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoMembershipShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoMembershipShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Membership shapes have capabilities of both geohashing and membership diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthLatitudeZone.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java index 43338bbd6882..2c940618f3ff 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This GeoBBox represents an area rectangle limited only in south latitude. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java index 66b9480d681f..a2b6f1b9c8cb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoNorthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box limited on three sides (bottom lat, left lon, right lon), including diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoOutsideDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoOutsideDistance.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoOutsideDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoOutsideDistance.java index c1d784d387f5..717854c542fc 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoOutsideDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoOutsideDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Implemented by Geo3D shapes that can compute the distance from a point to the closest outside edge. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPath.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java index bc5b9cf3f5ad..a5b8b9b06608 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPath.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.Collections; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPoint.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java index e8a265d40a1a..31ab0aa2f0af 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This class represents a point on the surface of a sphere or ellipsoid. @@ -117,7 +117,7 @@ public GeoPoint(final double x, final double y, final double z) { /** Compute an arc distance between two points. * Note: this is an angular distance, and not a surface distance, and is therefore independent of planet model. - * For surface distance, see {@link org.apache.lucene.geo3d.PlanetModel#surfaceDistance(GeoPoint, GeoPoint)} + * For surface distance, see {@link PlanetModel#surfaceDistance(GeoPoint, GeoPoint)} * @param v is the second point. * @return the angle, in radians, between the two points. */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygon.java similarity index 95% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygon.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygon.java index 634406d785ca..742bdf808d77 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygon.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * GeoPolygon interface description. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygonFactory.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 0dc70a5ebf61..8ee4290df3f6 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.BitSet; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java index fc2a531f2524..1420c1116d20 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box limited on four sides (top lat, bottom lat, left lon, right lon). diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoShape.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoShape.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoShape.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoShape.java index 21cdba3b9c70..a2d3947585ef 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoShape.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoShape.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Generic shape. This describes methods that help GeoAreas figure out diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSizeable.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSizeable.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSizeable.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSizeable.java index e8c0ebb2e5ad..3c7e2efce0ea 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSizeable.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSizeable.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Some shapes can compute radii of a geocircle in which they are inscribed. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthLatitudeZone.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java index 439dc1b96b83..a1d896740a72 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * This GeoBBox represents an area rectangle limited only in north latitude. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java index eb6526be9836..806535e72783 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoSouthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box limited on three sides (top lat, left lon, right lon). The diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoStandardCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoStandardCircle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java index 0304d532f8a1..bbf50466b8a6 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoStandardCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Circular area with a center and radius. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideDegenerateHorizontalLine.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideDegenerateHorizontalLine.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java index a9af5b2f60ba..48a73afec72c 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideDegenerateHorizontalLine.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Degenerate bounding box wider than PI and limited on two sides (left lon, right lon). diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideLongitudeSlice.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java index 64e4fa8b091a..1d6187649f3c 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box wider than PI but limited on left and right sides ( diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideNorthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideNorthRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java index 86de58420f9f..9f9dd49bab86 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideNorthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box wider than PI but limited on three sides ( diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java index 68397bba4737..c561747c57d1 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box wider than PI but limited on four sides (top lat, diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideSouthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideSouthRectangle.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java index 8bd72206551a..da9799ab0c34 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWideSouthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box wider than PI but limited on three sides (top lat, diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWorld.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWorld.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java index 35ec4ae2d9f2..25bdc9641659 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/GeoWorld.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Bounding box including the entire world. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LatLonBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/LatLonBounds.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java index 6478e0cc79b2..627fdaec996f 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LatLonBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * An object for accumulating latitude/longitude bounds information. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearDistance.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearDistance.java index 9cbedba4b417..0c89a166b6c6 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Linear distance computation style. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearSquaredDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearSquaredDistance.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearSquaredDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearSquaredDistance.java index 028d3c4222e6..3fc37da12899 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/LinearSquaredDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LinearSquaredDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Linear squared distance computation style. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Membership.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Membership.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Membership.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Membership.java index 3ca6b095ac67..0cf6ff0edd7c 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Membership.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Membership.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Implemented by Geo3D shapes that can calculate if a point is within it or not. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalDistance.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalDistance.java index cdac0d253bb3..50b2c7fc0b4d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Normal distance computation style. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalSquaredDistance.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalSquaredDistance.java similarity index 97% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalSquaredDistance.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalSquaredDistance.java index 035fd40e58c4..a355d09a67a2 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/NormalSquaredDistance.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/NormalSquaredDistance.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Normal squared distance computation style. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Plane.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java index 07d0c5b5f879..1f2c054791ba 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Plane.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * We know about three kinds of planes. First kind: general plain through two points and origin diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PlanetModel.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java similarity index 98% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/PlanetModel.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java index 395fa1512b9d..d45d7761a8c3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/PlanetModel.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Holds mathematical constants associated with the model of a planet. @@ -192,7 +192,7 @@ public boolean pointOutside(final double x, final double y, final double z) { * @param p1 is the first point. * @param p2 is the second point. * @return the adjusted angle, when multiplied by the mean earth radius, yields a surface distance. This will differ - * from GeoPoint.arcDistance() only when the planet model is not a sphere. @see {@link org.apache.lucene.geo3d.GeoPoint#arcDistance(GeoPoint)} + * from GeoPoint.arcDistance() only when the planet model is not a sphere. @see {@link GeoPoint#arcDistance(GeoPoint)} */ public double surfaceDistance(final GeoPoint p1, final GeoPoint p2) { final double latA = p1.getLatitude(); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/SidedPlane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/SidedPlane.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java index 7fc543d24e56..e080bc04f40e 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/SidedPlane.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Combination of a plane, and a sign value indicating what evaluation values are on the correct diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/StandardXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/StandardXYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java index cd542254cbd2..492f7b46a50c 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/StandardXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Tools.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Tools.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Tools.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Tools.java index 89d37aab844a..e8ee29e530e0 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Tools.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Tools.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Static methods globally useful for 3d geometric work. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Vector.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Vector.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/Vector.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Vector.java index 1a3972d3b0cd..3a1b233529a8 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/Vector.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Vector.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * A 3d vector in space, not necessarily diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZBounds.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java index 22e324bb457f..c3ee53d4166c 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * An object for accumulating XYZ bounds information. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolid.java similarity index 95% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolid.java index ab4640295973..9298079b7b64 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * Interface for a family of 3D rectangles, bounded on six sides by X,Y,Z limits diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolidFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolidFactory.java similarity index 96% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolidFactory.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolidFactory.java index 409ba86a0ff3..25ea4005044a 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYZSolidFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZSolidFactory.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** - * Factory for {@link org.apache.lucene.geo3d.XYZSolid}. + * Factory for {@link XYZSolid}. * * @lucene.experimental */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYdZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java index e7cbe25e1316..66aac84d9161 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in Z diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java index f6a2fa3931bd..d9e11b89fe6b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in Y diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYdZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java index 562e5a6aed95..33d0beafc656 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/XdYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in Y and Z. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java index 5a2a0063fe93..48fe714c28ad 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in X. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYdZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java index 96b3004bc5ca..d824f2629750 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in X and Z. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java index b58cd9261c23..b9942b5e6c9b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in X and Y. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java similarity index 99% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYdZSolid.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java index b26cf6345953..66dcab8536e6 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/dXdYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; /** * 3D rectangle, bounded on six sides by X,Y,Z limits, degenerate in all dimensions diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/package-info.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/package-info.java new file mode 100644 index 000000000000..446365cbd24a --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shapes implemented using 3D planar geometry. This package has no dependencies aside from Java. + * This code was contributed under the name "Geo3D". + */ +package org.apache.lucene.spatial3d.geom; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/package-info.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/package-info.java similarity index 83% rename from lucene/spatial3d/src/java/org/apache/lucene/geo3d/package-info.java rename to lucene/spatial3d/src/java/org/apache/lucene/spatial3d/package-info.java index 2b6af740b51f..032d26f73934 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/geo3d/package-info.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/package-info.java @@ -16,6 +16,6 @@ */ /** - * Shapes implemented using 3D planar geometry. + * Lucene field & query support for the spatial geometry implemented in {@link org.apache.lucene.spatial3d.geom}. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d; diff --git a/lucene/spatial3d/src/java/overview.html b/lucene/spatial3d/src/java/overview.html index 152d06e61101..293cc65aa60c 100644 --- a/lucene/spatial3d/src/java/overview.html +++ b/lucene/spatial3d/src/java/overview.html @@ -23,7 +23,8 @@

    The Spatial3D Module for Apache Lucene

    - APIs for planar spatial3d math. + APIs for planar spatial3d math. It is mostly comprised of computational geometry code in the + "org.apache.lucene.spatial3d.geom" package (AKA "Geo3D").

    diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java similarity index 98% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 17a40755d2fc..a4d8ed136316 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d; import java.io.IOException; import java.io.PrintWriter; @@ -36,6 +36,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; +import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; +import org.apache.lucene.spatial3d.geom.XYZBounds; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -50,7 +60,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoBBoxTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoBBoxTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java index b76134e43b31..f5a148fca9b7 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoBBoxTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.List; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoCircleTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoCircleTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java index aa5c2e3325bb..186bf4c650ee 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoCircleTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java @@ -14,15 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +package org.apache.lucene.spatial3d.geom; import org.apache.lucene.util.LuceneTestCase; +import org.junit.Test; public class GeoCircleTest extends LuceneTestCase { diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoConvexPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoConvexPolygonTest.java similarity index 98% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoConvexPolygonTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoConvexPolygonTest.java index d6ca7ba621a6..a6ca404ceaae 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoConvexPolygonTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoConvexPolygonTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import org.junit.Test; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoModelTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoModelTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoModelTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoModelTest.java index b3001d4df832..d5fcbdd5b89c 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoModelTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoModelTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import org.junit.Test; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPathTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPathTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java index fea7ed48ac44..37460699359e 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPathTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import org.junit.Test; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java similarity index 69% rename from lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java index 444647404e5c..ed1792862be2 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/geo3d/GeoPointTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java @@ -14,33 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.spatial.spatial4j.geo3d; +package org.apache.lucene.spatial3d.geom; -import org.apache.lucene.geo3d.GeoPoint; -import org.apache.lucene.geo3d.PlanetModel; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; -import org.locationtech.spatial4j.distance.DistanceUtils; - import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat; /** * Test basic GeoPoint functionality. */ public class GeoPointTest extends LuceneTestCase { + static final double DEGREES_TO_RADIANS = Math.PI / 180; @Test public void testConversion() { - testPointRoundTrip(PlanetModel.SPHERE, 90 * DistanceUtils.DEGREES_TO_RADIANS, 0, 1e-6); - testPointRoundTrip(PlanetModel.SPHERE, -90 * DistanceUtils.DEGREES_TO_RADIANS, 0, 1e-6); - testPointRoundTrip(PlanetModel.WGS84, 90 * DistanceUtils.DEGREES_TO_RADIANS, 0, 1e-6); - testPointRoundTrip(PlanetModel.WGS84, -90 * DistanceUtils.DEGREES_TO_RADIANS, 0, 1e-6); + testPointRoundTrip(PlanetModel.SPHERE, 90 * DEGREES_TO_RADIANS, 0, 1e-6); + testPointRoundTrip(PlanetModel.SPHERE, -90 * DEGREES_TO_RADIANS, 0, 1e-6); + testPointRoundTrip(PlanetModel.WGS84, 90 * DEGREES_TO_RADIANS, 0, 1e-6); + testPointRoundTrip(PlanetModel.WGS84, -90 * DEGREES_TO_RADIANS, 0, 1e-6); final int times = atLeast(100); for (int i = 0; i < times; i++) { - final double pLat = (randomFloat() * 180.0 - 90.0) * DistanceUtils.DEGREES_TO_RADIANS; - final double pLon = (randomFloat() * 360.0 - 180.0) * DistanceUtils.DEGREES_TO_RADIANS; + final double pLat = (randomFloat() * 180.0 - 90.0) * DEGREES_TO_RADIANS; + final double pLon = (randomFloat() * 360.0 - 180.0) * DEGREES_TO_RADIANS; testPointRoundTrip(PlanetModel.SPHERE, pLat, pLon, 1e-6);//1e-6 since there's a square root in there (Karl says) testPointRoundTrip(PlanetModel.WGS84, pLat, pLon, 1e-6); } @@ -60,10 +57,10 @@ protected void testPointRoundTrip(PlanetModel planetModel, double pLat, double p public void testSurfaceDistance() { final int times = atLeast(100); for (int i = 0; i < times; i++) { - final double p1Lat = (randomFloat() * 180.0 - 90.0) * DistanceUtils.DEGREES_TO_RADIANS; - final double p1Lon = (randomFloat() * 360.0 - 180.0) * DistanceUtils.DEGREES_TO_RADIANS; - final double p2Lat = (randomFloat() * 180.0 - 90.0) * DistanceUtils.DEGREES_TO_RADIANS; - final double p2Lon = (randomFloat() * 360.0 - 180.0) * DistanceUtils.DEGREES_TO_RADIANS; + final double p1Lat = (randomFloat() * 180.0 - 90.0) * DEGREES_TO_RADIANS; + final double p1Lon = (randomFloat() * 360.0 - 180.0) * DEGREES_TO_RADIANS; + final double p2Lat = (randomFloat() * 180.0 - 90.0) * DEGREES_TO_RADIANS; + final double p2Lon = (randomFloat() * 360.0 - 180.0) * DEGREES_TO_RADIANS; final GeoPoint p1 = new GeoPoint(PlanetModel.SPHERE, p1Lat, p1Lon); final GeoPoint p2 = new GeoPoint(PlanetModel.SPHERE, p2Lat, p2Lon); final double arcDistance = p1.arcDistance(p2); diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPolygonTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java index f1511b96942d..d9b220de9252 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/GeoPolygonTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import java.util.ArrayList; import java.util.List; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/PlaneTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/PlaneTest.java similarity index 98% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/PlaneTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/PlaneTest.java index 2ac3856bf7ec..91bd0c3f928e 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/PlaneTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/PlaneTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import org.junit.Test; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/XYZSolidTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/XYZSolidTest.java similarity index 99% rename from lucene/spatial3d/src/test/org/apache/lucene/geo3d/XYZSolidTest.java rename to lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/XYZSolidTest.java index 876a5256b116..98c616ebbff2 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/XYZSolidTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/XYZSolidTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.geo3d; +package org.apache.lucene.spatial3d.geom; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; From 16d98894ed41992eb5cfd5de11ae1e2ee72b97c9 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 7 Mar 2016 23:48:31 -0500 Subject: [PATCH 0054/1113] SOLR-6926: Remove deprecated "ant example"; replaced by "ant server" a year ago (cherry picked from commit a0a571c) --- solr/build.xml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/solr/build.xml b/solr/build.xml index 218bf8ce6d39..61503dfb9a14 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -809,18 +809,4 @@ - - - - - ! ! ! NOTICE NOTICE NOTICE ! ! ! - - 'ant example' is no longer recomended - - Use 'ant server' instead - - 'ant example' is going to be removed at some point - - - From 761618727d7604b09ec4ecc68d65689c888311f1 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 7 Mar 2016 16:00:40 +0000 Subject: [PATCH 0055/1113] SOLR-8782: Improve async collections API --- solr/CHANGES.txt | 4 + .../configsets/cloud-minimal/conf/schema.xml | 32 + .../cloud-minimal/conf/solrconfig.xml | 48 ++ .../CollectionsAPIAsyncDistributedZkTest.java | 174 ++--- .../security/BasicAuthIntegrationTest.java | 2 +- .../solrj/request/CollectionAdminRequest.java | 691 ++++++++++++------ 6 files changed, 622 insertions(+), 329 deletions(-) create mode 100644 solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml create mode 100644 solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0ebb488c2f35..ba2e737519cf 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -19,6 +19,10 @@ See the Quick Start guide at http://lucene.apache.org/solr/quickstart.html ================== 6.1.0 ================== Detailed Change List ---------------------- +* SOLR-8782: Add asynchronous sugar methods to the SolrJ Collections API. You + can now call .processAsync() to run a method asynchronously, or + .processAndWait() to wait for a call to finish without holding HTTP + collections open. (Alan Woodward) New Features ---------------------- diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml new file mode 100644 index 000000000000..2a276af206a4 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml new file mode 100644 index 000000000000..059e58f447c2 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml @@ -0,0 +1,48 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIAsyncDistributedZkTest.java index 493b2988853b..dcb115a1269b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIAsyncDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIAsyncDistributedZkTest.java @@ -21,90 +21,80 @@ import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util.TestUtil; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create; import org.apache.solr.client.solrj.request.CollectionAdminRequest.SplitShard; import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Slice; +import org.junit.BeforeClass; import org.junit.Test; /** * Tests the Cloud Collections API. */ @Slow -public class CollectionsAPIAsyncDistributedZkTest extends AbstractFullDistribZkTestBase { +public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase { + private static final int MAX_TIMEOUT_SECONDS = 60; - public CollectionsAPIAsyncDistributedZkTest() { - sliceCount = 1; + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); } @Test - @ShardsFixed(num = 1) public void testSolrJAPICalls() throws Exception { - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - Create createCollectionRequest = new Create() - .setCollectionName("testasynccollectioncreation") - .setNumShards(1) - .setConfigName("conf1") - .setAsyncId("1001"); - createCollectionRequest.process(client); - - RequestStatusState state = getRequestStateAfterCompletion("1001", MAX_TIMEOUT_SECONDS, client); - - assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state); - - createCollectionRequest = new Create() - .setCollectionName("testasynccollectioncreation") - .setNumShards(1) - .setConfigName("conf1") - .setAsyncId("1002"); - createCollectionRequest.process(client); - - state = getRequestStateAfterCompletion("1002", MAX_TIMEOUT_SECONDS, client); - - assertSame("Recreating a collection with the same should have failed.", RequestStatusState.FAILED, state); - - CollectionAdminRequest.AddReplica addReplica = new CollectionAdminRequest.AddReplica() - .setCollectionName("testasynccollectioncreation") - .setShardName("shard1") - .setAsyncId("1003"); - client.request(addReplica); - state = getRequestStateAfterCompletion("1003", MAX_TIMEOUT_SECONDS, client); - assertSame("Add replica did not complete", RequestStatusState.COMPLETED, state); - - SplitShard splitShardRequest = new SplitShard() - .setCollectionName("testasynccollectioncreation") - .setShardName("shard1") - .setAsyncId("1004"); - splitShardRequest.process(client); - - state = getRequestStateAfterCompletion("1004", MAX_TIMEOUT_SECONDS * 2, client); - - assertEquals("Shard split did not complete. Last recorded state: " + state, RequestStatusState.COMPLETED, state); - } + + final CloudSolrClient client = cluster.getSolrClient(); + + RequestStatusState state = new Create() + .setCollectionName("testasynccollectioncreation") + .setNumShards(1) + .setReplicationFactor(1) + .setConfigName("conf1") + .processAndWait(client, MAX_TIMEOUT_SECONDS); + assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state); + + state = new Create() + .setCollectionName("testasynccollectioncreation") + .setNumShards(1) + .setConfigName("conf1") + .processAndWait(client, MAX_TIMEOUT_SECONDS); + assertSame("Recreating a collection with the same should have failed.", RequestStatusState.FAILED, state); + + state = new CollectionAdminRequest.AddReplica() + .setCollectionName("testasynccollectioncreation") + .setShardName("shard1") + .processAndWait(client, MAX_TIMEOUT_SECONDS); + assertSame("Add replica did not complete", RequestStatusState.COMPLETED, state); + + state = new SplitShard() + .setCollectionName("testasynccollectioncreation") + .setShardName("shard1") + .processAndWait(client, MAX_TIMEOUT_SECONDS * 2); + assertEquals("Shard split did not complete. Last recorded state: " + state, RequestStatusState.COMPLETED, state); + } @Test public void testAsyncRequests() throws Exception { - String collection = "testAsyncOperations"; - Create createCollectionRequest = new Create() + final String collection = "testAsyncOperations"; + final CloudSolrClient client = cluster.getSolrClient(); + + RequestStatusState state = new Create() .setCollectionName(collection) .setNumShards(1) .setRouterName("implicit") .setShards("shard1") .setConfigName("conf1") - .setAsyncId("42"); - CollectionAdminResponse response = createCollectionRequest.process(cloudClient); - assertEquals("42", response.getResponse().get("requestid")); - RequestStatusState state = getRequestStateAfterCompletion("42", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state); //Add a few documents to shard1 @@ -116,59 +106,48 @@ public void testAsyncRequests() throws Exception { doc.addField("_route_", "shard1"); docs.add(doc); } - cloudClient.add(collection, docs); - cloudClient.commit(collection); + client.add(collection, docs); + client.commit(collection); SolrQuery query = new SolrQuery("*:*"); query.set("shards", "shard1"); - assertEquals(numDocs, cloudClient.query(collection, query).getResults().getNumFound()); + assertEquals(numDocs, client.query(collection, query).getResults().getNumFound()); - CollectionAdminRequest.Reload reloadCollection = new CollectionAdminRequest.Reload(); - reloadCollection.setCollectionName(collection).setAsyncId("43"); - response = reloadCollection.process(cloudClient); - assertEquals("43", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("43", MAX_TIMEOUT_SECONDS, cloudClient); + state = new CollectionAdminRequest.Reload() + .setCollectionName(collection) + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("ReloadCollection did not complete", RequestStatusState.COMPLETED, state); - CollectionAdminRequest.CreateShard createShard = new CollectionAdminRequest.CreateShard() + state = new CollectionAdminRequest.CreateShard() .setCollectionName(collection) .setShardName("shard2") - .setAsyncId("44"); - response = createShard.process(cloudClient); - assertEquals("44", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("44", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("CreateShard did not complete", RequestStatusState.COMPLETED, state); //Add a doc to shard2 to make sure shard2 was created properly SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", numDocs + 1); doc.addField("_route_", "shard2"); - cloudClient.add(collection, doc); - cloudClient.commit(collection); + client.add(collection, doc); + client.commit(collection); query = new SolrQuery("*:*"); query.set("shards", "shard2"); - assertEquals(1, cloudClient.query(collection, query).getResults().getNumFound()); + assertEquals(1, client.query(collection, query).getResults().getNumFound()); - CollectionAdminRequest.DeleteShard deleteShard = new CollectionAdminRequest.DeleteShard() + state = new CollectionAdminRequest.DeleteShard() .setCollectionName(collection) .setShardName("shard2") - .setAsyncId("45"); - response = deleteShard.process(cloudClient); - assertEquals("45", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("45", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("DeleteShard did not complete", RequestStatusState.COMPLETED, state); - CollectionAdminRequest.AddReplica addReplica = new CollectionAdminRequest.AddReplica() + state = new CollectionAdminRequest.AddReplica() .setCollectionName(collection) .setShardName("shard1") - .setAsyncId("46"); - response = addReplica.process(cloudClient); - assertEquals("46", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("46", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("AddReplica did not complete", RequestStatusState.COMPLETED, state); //cloudClient watch might take a couple of seconds to reflect it - Slice shard1 = cloudClient.getZkStateReader().getClusterState().getSlice(collection, "shard1"); + Slice shard1 = client.getZkStateReader().getClusterState().getSlice(collection, "shard1"); int count = 0; while (shard1.getReplicas().size() != 2) { if (count++ > 1000) { @@ -177,51 +156,40 @@ public void testAsyncRequests() throws Exception { Thread.sleep(100); } - CollectionAdminRequest.CreateAlias createAlias = new CollectionAdminRequest.CreateAlias() + state = new CollectionAdminRequest.CreateAlias() .setAliasName("myalias") .setAliasedCollections(collection) - .setAsyncId("47"); - response = createAlias.process(cloudClient); - assertEquals("47", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("47", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("CreateAlias did not complete", RequestStatusState.COMPLETED, state); query = new SolrQuery("*:*"); query.set("shards", "shard1"); - assertEquals(numDocs, cloudClient.query("myalias", query).getResults().getNumFound()); + assertEquals(numDocs, client.query("myalias", query).getResults().getNumFound()); - CollectionAdminRequest.DeleteAlias deleteAlias = new CollectionAdminRequest.DeleteAlias() + state = new CollectionAdminRequest.DeleteAlias() .setAliasName("myalias") - .setAsyncId("48"); - response = deleteAlias.process(cloudClient); - assertEquals("48", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("48", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("DeleteAlias did not complete", RequestStatusState.COMPLETED, state); try { - cloudClient.query("myalias", query); + client.query("myalias", query); fail("Alias should not exist"); } catch (SolrException e) { //expected } String replica = shard1.getReplicas().iterator().next().getName(); - CollectionAdminRequest.DeleteReplica deleteReplica = new CollectionAdminRequest.DeleteReplica() + state = new CollectionAdminRequest.DeleteReplica() .setCollectionName(collection) .setShardName("shard1") .setReplica(replica) - .setAsyncId("47"); - response = deleteReplica.process(cloudClient); - assertEquals("47", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("47", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("DeleteReplica did not complete", RequestStatusState.COMPLETED, state); - CollectionAdminRequest.Delete deleteCollection = new CollectionAdminRequest.Delete() + state = new CollectionAdminRequest.Delete() .setCollectionName(collection) - .setAsyncId("48"); - response = deleteCollection.process(cloudClient); - assertEquals("48", response.getResponse().get("requestid")); - state = getRequestStateAfterCompletion("48", MAX_TIMEOUT_SECONDS, cloudClient); + .processAndWait(client, MAX_TIMEOUT_SECONDS); assertSame("DeleteCollection did not complete", RequestStatusState.COMPLETED, state); } + } diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index c5d27a9a544e..ab02a3e1eacd 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -161,7 +161,7 @@ protected void doExtraTests(MiniSolrCloudCluster miniCluster, SolrZkClient zkCli verifySecurityStatus(cl, baseUrl + authzPrefix, "authorization/permissions[2]/name", "collection-admin-edit", 20); CollectionAdminRequest.Reload reload = new CollectionAdminRequest.Reload(); - reload.setCollectionName(cloudSolrClient.getDefaultCollection()); + reload.setCollectionName(defaultCollName); HttpSolrClient solrClient = new HttpSolrClient(baseUrl); try { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index a7d71ca5b93e..c9c8c3989dfb 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -21,10 +21,15 @@ import java.util.Iterator; import java.util.Map; import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.DocCollection; @@ -37,33 +42,28 @@ import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.NamedList; /** * This class is experimental and subject to change. * * @since solr 4.5 */ -public abstract class CollectionAdminRequest > extends SolrRequest { +public abstract class CollectionAdminRequest extends SolrRequest { - protected CollectionAction action = null; + protected final CollectionAction action; private static String PROPERTY_PREFIX = "property."; - protected CollectionAdminRequest setAction(CollectionAction action) { - this.action = action; - return this; - } - - public CollectionAdminRequest() { - super(METHOD.GET, "/admin/collections"); + public CollectionAdminRequest(CollectionAction action) { + this("/admin/collections", action); } - public CollectionAdminRequest(String path) { + public CollectionAdminRequest(String path, CollectionAction action) { super(METHOD.GET, path); + this.action = action; } - protected abstract Q getThis(); - @Override public SolrParams getParams() { if (action == null) { @@ -79,11 +79,6 @@ public Collection getContentStreams() throws IOException { return null; } - @Override - protected CollectionAdminResponse createResponse(SolrClient client) { - return new CollectionAdminResponse(); - } - protected void addProperties(ModifiableSolrParams params, Properties props) { Iterator> iter = props.entrySet().iterator(); while(iter.hasNext()) { @@ -94,18 +89,84 @@ protected void addProperties(ModifiableSolrParams params, Properties props) { } } - protected abstract static class AsyncCollectionAdminRequest > extends CollectionAdminRequest { - protected String asyncId = null; + protected abstract static class AsyncCollectionAdminRequest extends CollectionAdminRequest { - public final T setAsyncId(String asyncId) { - this.asyncId = asyncId; - return getThis(); + public AsyncCollectionAdminRequest(CollectionAction action) { + super(action); } + @Override + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); + } + + private static String generateAsyncId() { + return UUID.randomUUID().toString(); + } + + protected String asyncId = null; + public String getAsyncId() { return asyncId; } + /** + * @deprecated Use {@link #processAsync(String, SolrClient)} or {@link #processAsync(SolrClient)} + */ + @Deprecated + public abstract AsyncCollectionAdminRequest setAsyncId(String id); + + /** + * Process this request asynchronously, generating and returning a request id + * @param client a Solr client + * @return the request id + * @see CollectionAdminRequest.RequestStatus + */ + public String processAsync(SolrClient client) throws IOException, SolrServerException { + return processAsync(generateAsyncId(), client); + } + + /** + * Process this request asynchronously, using a specified request id + * @param asyncId the request id + * @param client a Solr client + * @return the request id + */ + public String processAsync(String asyncId, SolrClient client) throws IOException, SolrServerException { + this.asyncId = asyncId; + NamedList resp = client.request(this); + if (resp.get("error") != null) { + throw new SolrServerException((String)resp.get("error")); + } + return (String) resp.get("requestid"); + } + + /** + * Send this request to a Solr server, and wait (up to a timeout) for the request to + * complete or fail + * @param client a Solr client + * @param timeoutSeconds the maximum time to wait + * @return the status of the request on completion or timeout + */ + public RequestStatusState processAndWait(SolrClient client, long timeoutSeconds) + throws SolrServerException, InterruptedException, IOException { + return processAndWait(generateAsyncId(), client, timeoutSeconds); + } + + /** + * Send this request to a Solr server, and wait (up to a timeout) for the request to + * complete or fail + * @param asyncId an id for the request + * @param client a Solr client + * @param timeoutSeconds the maximum time to wait + * @return the status of the request on completion or timeout + */ + public RequestStatusState processAndWait(String asyncId, SolrClient client, long timeoutSeconds) + throws IOException, SolrServerException, InterruptedException { + processAsync(asyncId, client); + return new RequestStatus().setRequestId(asyncId).waitFor(client, timeoutSeconds); + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -116,121 +177,110 @@ public SolrParams getParams() { } } - //--------------------------------------------------------------------------------------- - // - //--------------------------------------------------------------------------------------- + protected abstract static class AsyncCollectionSpecificAdminRequest extends AsyncCollectionAdminRequest { - protected abstract static class CollectionSpecificAdminRequest > extends CollectionAdminRequest { - protected String collection = null; + protected String collection; - public T setCollectionName(String collectionName) { - this.collection = collectionName; - return getThis(); + public AsyncCollectionSpecificAdminRequest(CollectionAction action) { + super(action); } - public final String getCollectionName() { - return collection; - } + public abstract AsyncCollectionSpecificAdminRequest setCollectionName(String collection); @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + if (collection == null) + throw new IllegalArgumentException("You must call setCollectionName() on this request"); params.set(CoreAdminParams.NAME, collection); return params; } } - protected abstract static class CollectionSpecificAsyncAdminRequest> extends CollectionSpecificAdminRequest { - protected String asyncId = null; + protected abstract static class AsyncShardSpecificAdminRequest extends AsyncCollectionAdminRequest { - public final T setAsyncId(String asyncId) { - this.asyncId = asyncId; - return getThis(); - } + protected String collection; + protected String shard; - public String getAsyncId() { - return asyncId; + public AsyncShardSpecificAdminRequest(CollectionAction action) { + super(action); } + public abstract AsyncShardSpecificAdminRequest setCollectionName(String collection); + + public abstract AsyncShardSpecificAdminRequest setShardName(String shard); + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - if (asyncId != null) { - params.set(CommonAdminParams.ASYNC, asyncId); - } + if (collection == null) + throw new IllegalArgumentException("You must call setCollectionName() on this request"); + if (shard == null) + throw new IllegalArgumentException("You must call setShardName() on this request"); + params.set(CoreAdminParams.COLLECTION, collection); + params.set(CoreAdminParams.SHARD, shard); return params; } } - protected abstract static class CollectionShardAdminRequest > extends CollectionAdminRequest { - protected String shardName = null; - protected String collection = null; + protected abstract static class ShardSpecificAdminRequest extends CollectionAdminRequest { - public T setCollectionName(String collectionName) { - this.collection = collectionName; - return getThis(); - } + protected String collection; + protected String shard; - public String getCollectionName() { - return collection; + public ShardSpecificAdminRequest(CollectionAction action) { + super(action); } - public T setShardName(String shard) { - this.shardName = shard; - return getThis(); - } + public abstract ShardSpecificAdminRequest setCollectionName(String collection); - public String getShardName() { - return this.shardName; - } + public abstract ShardSpecificAdminRequest setShardName(String shard); @Override public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + if (collection == null) + throw new IllegalArgumentException("You must call setCollectionName() on this request"); + if (shard == null) + throw new IllegalArgumentException("You must call setShardName() on this request"); params.set(CoreAdminParams.COLLECTION, collection); - params.set(CoreAdminParams.SHARD, shardName); + params.set(CoreAdminParams.SHARD, shard); return params; } + + @Override + protected SolrResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); + } } - protected abstract static class CollectionShardAsyncAdminRequest> extends CollectionShardAdminRequest { - protected String asyncId = null; + //--------------------------------------------------------------------------------------- + // + //--------------------------------------------------------------------------------------- - public final T setAsyncId(String asyncId) { - this.asyncId = asyncId; - return getThis(); - } - public String getAsyncId() { - return asyncId; + protected abstract static class CollectionAdminRoleRequest extends AsyncCollectionAdminRequest { + + protected String node; + protected String role; + + public CollectionAdminRoleRequest(CollectionAction action) { + super(action); } @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - if (asyncId != null) { - params.set(CommonAdminParams.ASYNC, asyncId); - } - return params; + public CollectionAdminRoleRequest setAsyncId(String id) { + this.asyncId = id; + return this; } - } - protected abstract static class CollectionAdminRoleRequest > extends AsyncCollectionAdminRequest { - protected String node; - protected String role; - public T setNode(String node) { - this.node = node; - return getThis(); - } + public abstract CollectionAdminRoleRequest setNode(String node); public String getNode() { return this.node; } - public T setRole(String role) { - this.role = role; - return getThis(); - } + public abstract CollectionAdminRoleRequest setRole(String role); public String getRole() { return this.role; @@ -249,7 +299,8 @@ public SolrParams getParams() { /** Specific Collection API call implementations **/ // CREATE request - public static class Create extends CollectionSpecificAsyncAdminRequest { + public static class Create extends AsyncCollectionSpecificAdminRequest { + protected String configName = null; protected String createNodeSet = null; protected String routerName; @@ -263,8 +314,9 @@ public static class Create extends CollectionSpecificAsyncAdminRequest { protected Boolean autoAddReplicas; protected Integer stateFormat; private String[] rule , snitch; + public Create() { - action = CollectionAction.CREATE; + super(CollectionAction.CREATE); } public Create setConfigName(String config) { this.configName = config; return this; } @@ -314,7 +366,6 @@ public Create setShards(String shards) { * * @throws IllegalArgumentException if the collection name contains invalid characters. */ - @Override public Create setCollectionName(String collectionName) throws SolrException { if (!SolrIdentifierValidator.validateCollectionName(collectionName)) { throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.COLLECTION, @@ -324,6 +375,12 @@ public Create setCollectionName(String collectionName) throws SolrException { return this; } + @Override + public Create setAsyncId(String id) { + this.asyncId = id; + return this; + } + public Properties getProperties() { return properties; } @@ -337,8 +394,8 @@ public Create setProperties(Properties properties) { public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set( "collection.configName", configName); - params.set( "createNodeSet", createNodeSet); + params.set("collection.configName", configName); + params.set("createNodeSet", createNodeSet); if (numShards != null) { params.set( ZkStateReader.NUM_SHARDS_PROP, numShards); } @@ -367,51 +424,51 @@ public SolrParams getParams() { return params; } - @Override - protected Create getThis() { - return this; - } } // RELOAD request - public static class Reload extends CollectionSpecificAsyncAdminRequest { + public static class Reload extends AsyncCollectionSpecificAdminRequest { + public Reload() { - action = CollectionAction.RELOAD; + super(CollectionAction.RELOAD); } @Override - public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - return params; + public Reload setCollectionName(String collection) { + this.collection = collection; + return this; } @Override - protected Reload getThis() { + public Reload setAsyncId(String id) { + this.asyncId = id; return this; } } // DELETE request - public static class Delete extends CollectionSpecificAsyncAdminRequest { + public static class Delete extends AsyncCollectionSpecificAdminRequest { public Delete() { - action = CollectionAction.DELETE; + super(CollectionAction.DELETE); } @Override - public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - return params; + public Delete setCollectionName(String collection) { + this.collection = collection; + return this; } @Override - protected Delete getThis() { + public Delete setAsyncId(String id) { + this.asyncId = id; return this; } } // CREATESHARD request - public static class CreateShard extends CollectionShardAsyncAdminRequest { + public static class CreateShard extends AsyncShardSpecificAdminRequest { + protected String nodeSet; protected Properties properties; @@ -434,9 +491,15 @@ public CreateShard setProperties(Properties properties) { } public CreateShard() { - action = CollectionAction.CREATESHARD; + super(CollectionAction.CREATESHARD); } - + + @Override + public CreateShard setCollectionName(String collection) { + this.collection = collection; + return this; + } + /** * Provide the name of the shard to be created. * @@ -450,7 +513,13 @@ public CreateShard setShardName(String shardName) { throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.SHARD, shardName)); } - this.shardName = shardName; + this.shard = shardName; + return this; + } + + @Override + public CreateShard setAsyncId(String id) { + this.asyncId = id; return this; } @@ -466,21 +535,18 @@ public SolrParams getParams() { return params; } - @Override - protected CreateShard getThis() { - return this; - } + } // SPLITSHARD request - public static class SplitShard extends CollectionShardAsyncAdminRequest { + public static class SplitShard extends AsyncShardSpecificAdminRequest { protected String ranges; protected String splitKey; private Properties properties; public SplitShard() { - action = CollectionAction.SPLITSHARD; + super(CollectionAction.SPLITSHARD); } public SplitShard setRanges(String ranges) { this.ranges = ranges; return this; } @@ -504,6 +570,24 @@ public SplitShard setProperties(Properties properties) { return this; } + @Override + public SplitShard setCollectionName(String collection) { + this.collection = collection; + return this; + } + + @Override + public SplitShard setShardName(String shard) { + this.shard = shard; + return this; + } + + @Override + public SplitShard setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); @@ -518,25 +602,16 @@ public SolrParams getParams() { return params; } - @Override - protected SplitShard getThis() { - return this; - } } // DELETESHARD request - public static class DeleteShard extends CollectionShardAsyncAdminRequest { + public static class DeleteShard extends AsyncShardSpecificAdminRequest { private Boolean deleteInstanceDir; private Boolean deleteDataDir; public DeleteShard() { - action = CollectionAction.DELETESHARD; - } - - @Override - protected DeleteShard getThis() { - return this; + super(CollectionAction.DELETESHARD); } public Boolean getDeleteInstanceDir() { @@ -557,6 +632,24 @@ public DeleteShard setDeleteDataDir(Boolean deleteDataDir) { return this; } + @Override + public DeleteShard setCollectionName(String collection) { + this.collection = collection; + return this; + } + + @Override + public DeleteShard setShardName(String shard) { + this.shard = shard; + return this; + } + + @Override + public DeleteShard setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -571,24 +664,43 @@ public SolrParams getParams() { } // FORCELEADER request - public static class ForceLeader extends CollectionShardAdminRequest { + public static class ForceLeader extends ShardSpecificAdminRequest { public ForceLeader() { - action = CollectionAction.FORCELEADER; + super(CollectionAction.FORCELEADER); + } + + + @Override + public ForceLeader setCollectionName(String collection) { + this.collection = collection; + return this; } @Override - protected ForceLeader getThis() { + public ForceLeader setShardName(String shard) { + this.shard = shard; return this; } + + } + + public static class RequestStatusResponse extends CollectionAdminResponse { + + public RequestStatusState getRequestStatus() { + NamedList innerResponse = (NamedList) getResponse().get("status"); + return RequestStatusState.fromKey((String) innerResponse.get("state")); + } + } // REQUESTSTATUS request - public static class RequestStatus extends CollectionAdminRequest { - protected String requestId = null; + public static class RequestStatus extends CollectionAdminRequest { + + protected String requestId = null; public RequestStatus() { - action = CollectionAction.REQUESTSTATUS; + super(CollectionAction.REQUESTSTATUS); } public RequestStatus setRequestId(String requestId) { @@ -603,23 +715,41 @@ public String getRequestId() { @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + if (requestId == null) + throw new IllegalArgumentException("You must call setRequestId() on this request"); params.set(CoreAdminParams.REQUESTID, requestId); return params; } @Override - protected RequestStatus getThis() { - return this; + protected RequestStatusResponse createResponse(SolrClient client) { + return new RequestStatusResponse(); + } + + public RequestStatusState waitFor(SolrClient client, long timeoutSeconds) + throws IOException, SolrServerException, InterruptedException { + long finishTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + RequestStatusState state = RequestStatusState.NOT_FOUND; + while (System.nanoTime() < finishTime) { + state = this.process(client).getRequestStatus(); + if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) { + new DeleteStatus().setRequestId(requestId).process(client); + return state; + } + TimeUnit.SECONDS.sleep(1); + } + return state; } } // DELETESTATUS request - public static class DeleteStatus extends CollectionAdminRequest { + public static class DeleteStatus extends CollectionAdminRequest { + protected String requestId = null; protected Boolean flush = null; public DeleteStatus() { - action = CollectionAction.DELETESTATUS; + super(CollectionAction.DELETESTATUS); } public DeleteStatus setRequestId(String requestId) { @@ -652,18 +782,20 @@ public SolrParams getParams() { } @Override - protected DeleteStatus getThis() { - return this; + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); } + } // CREATEALIAS request - public static class CreateAlias extends AsyncCollectionAdminRequest { + public static class CreateAlias extends AsyncCollectionAdminRequest { + protected String aliasName; protected String aliasedCollections; public CreateAlias() { - action = CollectionAction.CREATEALIAS; + super(CollectionAction.CREATEALIAS); } /** @@ -695,6 +827,12 @@ public String getAliasedCollections() { return this.aliasedCollections; } + @Override + public CreateAlias setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); @@ -703,18 +841,15 @@ public SolrParams getParams() { return params; } - @Override - protected CreateAlias getThis() { - return this; - } } // DELETEALIAS request - public static class DeleteAlias extends AsyncCollectionAdminRequest { + public static class DeleteAlias extends AsyncCollectionAdminRequest { + protected String aliasName; public DeleteAlias() { - action = CollectionAction.DELETEALIAS; + super(CollectionAction.DELETEALIAS); } public DeleteAlias setAliasName(String aliasName) { @@ -722,6 +857,12 @@ public DeleteAlias setAliasName(String aliasName) { return this; } + @Override + public DeleteAlias setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -729,14 +870,14 @@ public SolrParams getParams() { return params; } - @Override - protected DeleteAlias getThis() { - return this; - } + } // ADDREPLICA request - public static class AddReplica extends CollectionShardAsyncAdminRequest { + public static class AddReplica extends AsyncCollectionAdminRequest { + + protected String collection; + protected String shard; protected String node; protected String routeKey; protected String instanceDir; @@ -744,7 +885,7 @@ public static class AddReplica extends CollectionShardAsyncAdminRequest { + public static class DeleteReplica extends AsyncShardSpecificAdminRequest { + protected String replica; protected Boolean onlyIfDown; private Boolean deleteDataDir; @@ -832,7 +992,7 @@ public static class DeleteReplica extends CollectionShardAsyncAdminRequest { + public static class ClusterProp extends CollectionAdminRequest { + private String propertyName; private String propertyValue; public ClusterProp() { - this.action = CollectionAction.CLUSTERPROP; + super(CollectionAction.CLUSTERPROP); } public ClusterProp setPropertyName(String propertyName) { @@ -934,13 +1108,16 @@ public SolrParams getParams() { } @Override - protected ClusterProp getThis() { - return this; + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); } + + } // MIGRATE request - public static class Migrate extends AsyncCollectionAdminRequest { + public static class Migrate extends AsyncCollectionAdminRequest { + private String collection; private String targetCollection; private String splitKey; @@ -948,7 +1125,7 @@ public static class Migrate extends AsyncCollectionAdminRequest { private Properties properties; public Migrate() { - action = CollectionAction.MIGRATE; + super(CollectionAction.MIGRATE); } public Migrate setCollectionName(String collection) { @@ -996,6 +1173,12 @@ public Properties getProperties() { return this.properties; } + @Override + public Migrate setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -1012,58 +1195,72 @@ public SolrParams getParams() { return params; } - @Override - protected Migrate getThis() { - return this; - } + } // ADDROLE request - public static class AddRole extends CollectionAdminRoleRequest { + public static class AddRole extends CollectionAdminRoleRequest { + public AddRole() { - action = CollectionAction.ADDROLE; + super(CollectionAction.ADDROLE); } @Override - protected AddRole getThis() { + public AddRole setNode(String node) { + this.node = node; + return this; + } + + @Override + public AddRole setRole(String role) { + this.role = role; return this; } } // REMOVEROLE request - public static class RemoveRole extends CollectionAdminRoleRequest { + public static class RemoveRole extends CollectionAdminRoleRequest { + public RemoveRole() { - action = CollectionAction.REMOVEROLE; + super(CollectionAction.REMOVEROLE); } @Override - protected RemoveRole getThis() { + public RemoveRole setNode(String node) { + this.node = node; + return this; + } + + @Override + public RemoveRole setRole(String role) { + this.role = role; return this; } } // OVERSEERSTATUS request - public static class OverseerStatus extends AsyncCollectionAdminRequest { + public static class OverseerStatus extends AsyncCollectionAdminRequest { public OverseerStatus () { - action = CollectionAction.OVERSEERSTATUS; + super(CollectionAction.OVERSEERSTATUS); } @Override - protected OverseerStatus getThis() { + public OverseerStatus setAsyncId(String id) { + this.asyncId = id; return this; } } // CLUSTERSTATUS request - public static class ClusterStatus extends CollectionAdminRequest { + public static class ClusterStatus extends CollectionAdminRequest { protected String shardName = null; protected String collection = null; protected String routeKey = null; public ClusterStatus () { - action = CollectionAction.CLUSTERSTATUS; + super(CollectionAction.CLUSTERSTATUS); } public ClusterStatus setCollectionName(String collectionName) { @@ -1109,32 +1306,35 @@ public SolrParams getParams() { } @Override - protected ClusterStatus getThis() { - return this; + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); } + + } // LIST request - public static class List extends CollectionAdminRequest { + public static class List extends CollectionAdminRequest { public List () { - action = CollectionAction.LIST; + super(CollectionAction.LIST); } @Override - protected List getThis() { - return this; + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); } } // ADDREPLICAPROP request - public static class AddReplicaProp extends CollectionShardAsyncAdminRequest { + public static class AddReplicaProp extends AsyncShardSpecificAdminRequest { + private String replica; private String propertyName; private String propertyValue; private Boolean shardUnique; public AddReplicaProp() { - action = CollectionAction.ADDREPLICAPROP; + super(CollectionAction.ADDREPLICAPROP); } public String getReplica() { @@ -1173,6 +1373,24 @@ public AddReplicaProp setShardUnique(Boolean shardUnique) { return this; } + @Override + public AddReplicaProp setCollectionName(String collection) { + this.collection = collection; + return this; + } + + @Override + public AddReplicaProp setShardName(String shard) { + this.shard = shard; + return this; + } + + @Override + public AddReplicaProp setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -1187,19 +1405,16 @@ public SolrParams getParams() { return params; } - @Override - protected AddReplicaProp getThis() { - return this; - } } // DELETEREPLICAPROP request - public static class DeleteReplicaProp extends CollectionShardAsyncAdminRequest { + public static class DeleteReplicaProp extends AsyncShardSpecificAdminRequest { + private String replica; private String propertyName; public DeleteReplicaProp() { - this.action = CollectionAction.DELETEREPLICAPROP; + super(CollectionAction.DELETEREPLICAPROP); } public String getReplica() { @@ -1220,6 +1435,24 @@ public DeleteReplicaProp setPropertyName(String propertyName) { return this; } + @Override + public DeleteReplicaProp setCollectionName(String collection) { + this.collection = collection; + return this; + } + + @Override + public DeleteReplicaProp setShardName(String shard) { + this.shard = shard; + return this; + } + + @Override + public DeleteReplicaProp setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -1228,44 +1461,49 @@ public SolrParams getParams() { return params; } - @Override - protected DeleteReplicaProp getThis() { - return this; - } + } // MIGRATECLUSTERSTATE request - public static class MigrateClusterState extends CollectionShardAsyncAdminRequest { + public static class MigrateClusterState extends AsyncCollectionAdminRequest { + + protected String collection; public MigrateClusterState() { - this.action = CollectionAction.MIGRATESTATEFORMAT; + super(CollectionAction.MIGRATESTATEFORMAT); } - @Override - public MigrateClusterState setShardName(String shard) { - throw new UnsupportedOperationException(); + public MigrateClusterState setCollectionName(String collection) { + this.collection = collection; + return this; } @Override - public String getShardName() { - throw new UnsupportedOperationException(); + public MigrateClusterState setAsyncId(String id) { + this.asyncId = id; + return this; } @Override - protected MigrateClusterState getThis() { - return this; + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + if (collection == null) + throw new IllegalArgumentException("You must call setCollection() on this request"); + params.set(CoreAdminParams.COLLECTION, collection); + return params; } } // BALANCESHARDUNIQUE request - public static class BalanceShardUnique extends AsyncCollectionAdminRequest { + public static class BalanceShardUnique extends AsyncCollectionAdminRequest { + protected String collection; protected String propertyName; protected Boolean onlyActiveNodes; protected Boolean shardUnique; public BalanceShardUnique() { - this.action = CollectionAction.BALANCESHARDUNIQUE; + super(CollectionAction.BALANCESHARDUNIQUE); } public String getPropertyName() { @@ -1304,21 +1542,24 @@ public String getCollection() { return collection; } + @Override + public BalanceShardUnique setAsyncId(String id) { + this.asyncId = id; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); params.set(CoreAdminParams.COLLECTION, collection); params.set("property", propertyName); - if(onlyActiveNodes != null) + if (onlyActiveNodes != null) params.set("onlyactivenodes", onlyActiveNodes); - if(shardUnique != null) + if (shardUnique != null) params.set("shardUnique", shardUnique); return params; } - @Override - protected BalanceShardUnique getThis() { - return this; - } } + } From b97e17096e770a4a7c4edc0a353755673ef359ad Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 06:20:25 -0500 Subject: [PATCH 0056/1113] LUCENE-7076: Improve MIGRATE.txt/Point javadocs --- lucene/MIGRATE.txt | 14 ++--- .../apache/lucene/document/BinaryPoint.java | 2 + .../apache/lucene/document/DoublePoint.java | 2 + .../apache/lucene/document/FloatPoint.java | 2 + .../org/apache/lucene/document/IntPoint.java | 2 + .../org/apache/lucene/document/LongPoint.java | 2 + .../org/apache/lucene/index/PointValues.java | 54 +++++++++++++++++-- .../apache/lucene/search/PointInSetQuery.java | 13 +---- .../apache/lucene/search/PointRangeQuery.java | 11 +--- .../lucene/document/BigIntegerPoint.java | 2 + .../lucene/document/InetAddressPoint.java | 2 + .../apache/lucene/document/LatLonPoint.java | 2 + .../apache/lucene/spatial3d/Geo3DPoint.java | 3 +- 13 files changed, 79 insertions(+), 32 deletions(-) diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt index b48226de8677..b94c12d24e62 100644 --- a/lucene/MIGRATE.txt +++ b/lucene/MIGRATE.txt @@ -74,10 +74,12 @@ would be equivalent to the following code with the old setBoost API: float boost = ...; q.setBoost(q.getBoost() * boost); -## DimensionalValues replaces NumericField (LUCENE-6917) +# PointValues replaces NumericField (LUCENE-6917) -DimensionalValues provides faster indexing and searching, a smaller -index size, and less heap used at search time. The numeric fields -(IntField, FloatField, LongField, DoubleField) and NumericRangeQuery -have been moved to the backward-codecs module and prefixed with -Legacy. +PointValues provides faster indexing and searching, a smaller +index size, and less heap used at search time. See org.apache.lucene.index.PointValues +for an introduction. + +Legacy numeric encodings from previous versions of Lucene are +deprecated as LegacyIntField, LegacyFloatField, LegacyLongField, and LegacyDoubleField, +and can be searched with LegacyNumericRangeQuery. diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java index e139a877264a..4e27d8127c42 100644 --- a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Comparator; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; @@ -40,6 +41,7 @@ *
  • {@link #newRangeQuery(String, byte[], byte[])} for matching a 1D range. *
  • {@link #newRangeQuery(String, byte[][], byte[][])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public final class BinaryPoint extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java index 52b039f818ea..26ac0ced6f10 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java @@ -18,6 +18,7 @@ import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -38,6 +39,7 @@ *
  • {@link #newRangeQuery(String, double, double)} for matching a 1D range. *
  • {@link #newRangeQuery(String, double[], double[])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public final class DoublePoint extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java index cad666c4528d..c58881ec8800 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java @@ -18,6 +18,7 @@ import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -38,6 +39,7 @@ *
  • {@link #newRangeQuery(String, float, float)} for matching a 1D range. *
  • {@link #newRangeQuery(String, float[], float[])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public final class FloatPoint extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java index b6f5ae764d49..cb8315f2f205 100644 --- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java @@ -18,6 +18,7 @@ import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -38,6 +39,7 @@ *
  • {@link #newRangeQuery(String, int, int)} for matching a 1D range. *
  • {@link #newRangeQuery(String, int[], int[])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public final class IntPoint extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java index c0672aeacc68..ff78132d7b66 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java @@ -18,6 +18,7 @@ import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -38,6 +39,7 @@ *
  • {@link #newRangeQuery(String, long, long)} for matching a 1D range. *
  • {@link #newRangeQuery(String, long[], long[])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public final class LongPoint extends Field { diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java index 230a14f4be86..1fb265443028 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java @@ -17,17 +17,65 @@ package org.apache.lucene.index; import java.io.IOException; +import java.math.BigInteger; +import java.net.InetAddress; import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.util.bkd.BKDWriter; -/** Allows recursively visiting point values indexed with {@link IntPoint}, - * {@link FloatPoint}, {@link LongPoint}, {@link DoublePoint} - * or {@link BinaryPoint}. +/** + * Access to indexed numeric values. + *

    + * Points represent numeric values and are indexed differently than ordinary text. Instead of an inverted index, + * points are indexed with datastructures such as KD-trees. + * These structures are optimized for operations such as range, distance, nearest-neighbor, + * and point-in-polygon queries. + *

    Basic Point Types

    + * + * + * + * + * + * + * + * + * + *
    Java typeLucene class
    {@code int}{@link IntPoint}
    {@code long}{@link LongPoint}
    {@code float}{@link FloatPoint}
    {@code double}{@link DoublePoint}
    {@code byte[]}{@link BinaryPoint}
    {@link BigInteger}BigIntegerPoint*
    {@link InetAddress}InetAddressPoint*
    + * * in the lucene-sandbox jar
    + *

    + * Basic Lucene point types behave like their java peers: for example {@link IntPoint} represents a signed 32-bit + * {@link Integer}, supporting values ranging from {@link Integer#MIN_VALUE} to {@link Integer#MAX_VALUE}, ordered + * consistent with {@link Integer#compareTo(Integer)}. In addition to indexing support, point classes also contain + * static methods (such as {@link IntPoint#newRangeQuery(String, int, int)}) for creating common queries. For example: + * 

    + *   // add year 1970 to document
+ *   document.add(new IntPoint("year", 1970));
+ *   // index document
+ *   writer.addDocument(document);
+ *   ...
+ *   // issue range query of 1960-1980
+ *   Query query = IntPoint.newRangeQuery("year", 1960, 1980);
+ *   TopDocs docs = searcher.search(query, ...);
+ *
    + *

    Geospatial Point Types

    + * Although basic point types such as {@link DoublePoint} support points in multi-dimensional space too, Lucene has + * specialized classes for location data. These classes are optimized for location data: they are more space-efficient and + * support special operations such as distance and polygon queries. There are currently two implementations: + *
    + *
      + *
    1. LatLonPoint in lucene-sandbox: indexes {@code (latitude,longitude)} as {@code (x,y)} in two-dimensional space. + *
    2. Geo3DPoint* in lucene-spatial3d: indexes {@code (latitude,longitude)} as {@code (x,y,z)} in three-dimensional space. + *
    + * * does not support altitude, 3D here means "uses three dimensions under-the-hood"
    + *

    Advanced usage

    + * Custom structures can be created on top of single- or multi- dimensional basic types, on top of + * {@link BinaryPoint} for more flexibility, or via custom {@link Field} subclasses. * * @lucene.experimental */ public abstract class PointValues { diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index f5ba12dc0db6..944fadfb2bb2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -16,15 +16,10 @@ */ package org.apache.lucene.search; - import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.document.BinaryPoint; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -48,13 +43,7 @@ * create range queries for lucene's standard {@code Point} types, refer to factory * methods on those classes, e.g. {@link IntPoint#newSetQuery IntPoint.newSetQuery()} for * fields indexed with {@link IntPoint}. - - * @see IntPoint - * @see LongPoint - * @see FloatPoint - * @see DoublePoint - * @see BinaryPoint - * + * @see PointValues * @lucene.experimental */ public abstract class PointInSetQuery extends Query { diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 777c13388130..ebbe7e2fbd6a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -23,11 +23,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; -import org.apache.lucene.document.BinaryPoint; // javadocs -import org.apache.lucene.document.DoublePoint; // javadocs -import org.apache.lucene.document.FloatPoint; // javadocs import org.apache.lucene.document.IntPoint; // javadocs -import org.apache.lucene.document.LongPoint; // javadocs import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -44,12 +40,7 @@ * fields indexed with {@link IntPoint}. *

    * For a single-dimensional field this query is a simple range query; in a multi-dimensional field it's a box shape. - * @see IntPoint - * @see LongPoint - * @see FloatPoint - * @see DoublePoint - * @see BinaryPoint - * + * @see PointValues * @lucene.experimental */ public abstract class PointRangeQuery extends Query { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java index f1758581d2dc..70445d6d2fd7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java @@ -19,6 +19,7 @@ import java.math.BigInteger; import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -39,6 +40,7 @@ *

  • {@link #newRangeQuery(String, BigInteger, BigInteger)} for matching a 1D range. *
  • {@link #newRangeQuery(String, BigInteger[], BigInteger[])} for matching points/ranges in n-dimensional space. * + * @see PointValues */ public class BigIntegerPoint extends Field { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java index a0623b367927..f0df6ff29564 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java @@ -20,6 +20,7 @@ import java.net.UnknownHostException; import java.util.Arrays; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -43,6 +44,7 @@ * This field supports both IPv4 and IPv6 addresses: IPv4 addresses are converted * to IPv4-Mapped IPv6 Addresses: * indexing {@code 1.2.3.4} is the same as indexing {@code ::FFFF:1.2.3.4}. + * @see PointValues */ public class InetAddressPoint extends Field { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 5f45cb5996a9..fd3284b6e29b 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.NumericUtils; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.PointValues; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; @@ -43,6 +44,7 @@ *

    * WARNING: Values are indexed with some loss of precision, incurring up to 1E-7 error from the * original {@code double} values. + * @see PointValues */ // TODO ^^^ that is very sandy and hurts the API, usage, and tests tremendously, because what the user passes // to the field is not actually what gets indexed. Float would be 1E-5 error vs 1E-7, but it might be diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index cd2c79a15d0f..955a2bc0f36a 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -18,6 +18,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.PointValues; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; @@ -34,7 +35,7 @@ *

      *
    • {@link #newShapeQuery newShapeQuery()} for matching all points inside a specified shape *
    - * + * @see PointValues * @lucene.experimental */ public final class Geo3DPoint extends Field { From 914003d19a7c44a8d9aa5fd0a7a6e59882eb1c69 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 06:42:35 -0500 Subject: [PATCH 0057/1113] LUCENE-7077: fail precommit on useless assignment --- .../java/org/apache/lucene/codecs/memory/FSTTermsWriter.java | 2 +- lucene/tools/javadoc/ecj.javadocs.prefs | 2 +- .../test/org/apache/solr/update/DirectUpdateHandlerTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 50044f1c1eeb..8284d7444f26 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -167,7 +167,7 @@ public void write(Fields fields) throws IOException { FieldInfo fieldInfo = fieldInfos.fieldInfo(field); boolean hasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; TermsEnum termsEnum = terms.iterator(); - TermsWriter termsWriter = termsWriter = new TermsWriter(fieldInfo); + TermsWriter termsWriter = new TermsWriter(fieldInfo); long sumTotalTermFreq = 0; long sumDocFreq = 0; diff --git a/lucene/tools/javadoc/ecj.javadocs.prefs b/lucene/tools/javadoc/ecj.javadocs.prefs index 63f22e6c1eca..d01148c602fa 100644 --- a/lucene/tools/javadoc/ecj.javadocs.prefs +++ b/lucene/tools/javadoc/ecj.javadocs.prefs @@ -51,7 +51,7 @@ org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled org.eclipse.jdt.core.compiler.problem.missingSerialVersion=ignore org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=ignore org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore org.eclipse.jdt.core.compiler.problem.nullReference=ignore diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java index d35614d02303..2bde1186ad67 100644 --- a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java @@ -299,7 +299,7 @@ public void testExpungeDeletes() throws Exception { assertU(commit("expungeDeletes","true")); sr = req("q","foo"); - r = r = sr.getSearcher().getIndexReader(); + r = sr.getSearcher().getIndexReader(); assertEquals(r.maxDoc(), r.numDocs()); // no deletions assertEquals(4,r.maxDoc()); // no dups sr.close(); From 644d534f6c5d68e149f5ff36c477e82ecec0f3d2 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 8 Mar 2016 18:00:12 +0530 Subject: [PATCH 0058/1113] SOLR-8736: schema GET operations on fields, dynamicFields, fieldTypes, copyField have less details --- solr/CHANGES.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ba2e737519cf..1bba686419ac 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -106,7 +106,8 @@ Upgrading from Solr 5.x * When requesting stats in date fields, "sum" is now a double value instead of a date. See SOLR-8671 -* SOLR-8736: The deprecated GET methods for schema are now accessible and implemented differently +* SOLR-8736: The deprecated GET methods for schema are now accessible through the bulk API. The output + has less details and is not backward compatible. Detailed Change List ---------------------- From 2ede76b3871befdcf0eca9a6950101253d8e7521 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 07:36:09 -0500 Subject: [PATCH 0059/1113] LUCENE-7075: convert test class to use points --- .../index/BaseStoredFieldsFormatTestCase.java | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java index 840fdf5987e6..c58d56aa11e3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java @@ -33,20 +33,15 @@ import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.simpletext.SimpleTextCodec; import org.apache.lucene.document.Document; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -244,51 +239,44 @@ public void testNumericField() throws Exception { RandomIndexWriter w = new RandomIndexWriter(random(), dir); final int numDocs = atLeast(500); final Number[] answers = new Number[numDocs]; - final LegacyNumericType[] typeAnswers = new LegacyNumericType[numDocs]; + final Class[] typeAnswers = new Class[numDocs]; for(int id=0;id typeAnswer; if (random().nextBoolean()) { // float/double if (random().nextBoolean()) { final float f = random().nextFloat(); answer = Float.valueOf(f); - nf = new LegacyFloatField("nf", f, Field.Store.NO); - sf = new StoredField("nf", f); - typeAnswer = LegacyNumericType.FLOAT; + nf = new StoredField("nf", f); + typeAnswer = Float.class; } else { final double d = random().nextDouble(); answer = Double.valueOf(d); - nf = new LegacyDoubleField("nf", d, Field.Store.NO); - sf = new StoredField("nf", d); - typeAnswer = LegacyNumericType.DOUBLE; + nf = new StoredField("nf", d); + typeAnswer = Double.class; } } else { // int/long if (random().nextBoolean()) { final int i = random().nextInt(); answer = Integer.valueOf(i); - nf = new LegacyIntField("nf", i, Field.Store.NO); - sf = new StoredField("nf", i); - typeAnswer = LegacyNumericType.INT; + nf = new StoredField("nf", i); + typeAnswer = Integer.class; } else { final long l = random().nextLong(); answer = Long.valueOf(l); - nf = new LegacyLongField("nf", l, Field.Store.NO); - sf = new StoredField("nf", l); - typeAnswer = LegacyNumericType.LONG; + nf = new StoredField("nf", l); + typeAnswer = Long.class; } } doc.add(nf); - doc.add(sf); answers[id] = answer; typeAnswers[id] = typeAnswer; - FieldType ft = new FieldType(LegacyIntField.TYPE_STORED); - ft.setNumericPrecisionStep(Integer.MAX_VALUE); - doc.add(new LegacyIntField("id", id, ft)); + doc.add(new StoredField("id", id)); + doc.add(new IntPoint("id", id)); doc.add(new NumericDocValuesField("id", id)); w.addDocument(doc); } @@ -348,10 +336,10 @@ public void testReadSkip() throws IOException { List fields = Arrays.asList( new Field("bytes", bytes, ft), new Field("string", string, ft), - new LegacyLongField("long", l, Store.YES), - new LegacyIntField("int", i, Store.YES), - new LegacyFloatField("float", f, Store.YES), - new LegacyDoubleField("double", d, Store.YES) + new StoredField("long", l), + new StoredField("int", i), + new StoredField("float", f), + new StoredField("double", d) ); for (int k = 0; k < 100; ++k) { @@ -519,11 +507,14 @@ public void testWriteReadMerge() throws IOException { final FieldType type = new FieldType(StringField.TYPE_STORED); type.setIndexOptions(IndexOptions.NONE); type.freeze(); - LegacyIntField id = new LegacyIntField("id", 0, Store.YES); + IntPoint id = new IntPoint("id", 0); + StoredField idStored = new StoredField("id", 0); for (int i = 0; i < data.length; ++i) { Document doc = new Document(); doc.add(id); + doc.add(idStored); id.setIntValue(i); + idStored.setIntValue(i); for (int j = 0; j < data[i].length; ++j) { Field f = new Field("bytes" + j, data[i][j], type); doc.add(f); @@ -546,7 +537,7 @@ public void testWriteReadMerge() throws IOException { for (int i = 0; i < 10; ++i) { final int min = random().nextInt(data.length); final int max = min + random().nextInt(20); - iw.deleteDocuments(LegacyNumericRangeQuery.newIntRange("id", min, max, true, false)); + iw.deleteDocuments(IntPoint.newRangeQuery("id", min, max-1)); } iw.forceMerge(2); // force merges with deletions From 058ddffcafd215039194c84ac6b6f315f9dd83cd Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 8 Mar 2016 18:49:44 +0530 Subject: [PATCH 0060/1113] SOLR-8766: deprecated tag in solrconfig.xml is removed --- solr/CHANGES.txt | 2 ++ .../src/test-files/solr/collection1/conf/solrconfig.xml | 6 ------ .../src/test-files/solr/minimr/conf/solrconfig.xml | 6 ------ .../src/test-files/solr/mrunit/conf/solrconfig.xml | 7 ------- .../solr/solrcelltest/collection1/conf/solrconfig.xml | 6 ------ .../src/test-files/solr/solrcloud/conf/solrconfig.xml | 6 ------ solr/example/example-DIH/solr/db/conf/solrconfig.xml | 6 ------ solr/example/example-DIH/solr/mail/conf/solrconfig.xml | 6 ------ solr/example/example-DIH/solr/rss/conf/solrconfig.xml | 6 ------ solr/example/example-DIH/solr/solr/conf/solrconfig.xml | 6 ------ solr/example/example-DIH/solr/tika/conf/solrconfig.xml | 7 ------- solr/example/files/conf/solrconfig.xml | 6 ------ .../solr/configsets/basic_configs/conf/solrconfig.xml | 5 ----- .../data_driven_schema_configs/conf/solrconfig.xml | 7 ------- .../sample_techproducts_configs/conf/solrconfig.xml | 6 ------ 15 files changed, 2 insertions(+), 86 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1bba686419ac..fa43ecec71cf 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -415,6 +415,8 @@ Other Changes * SOLR-8736: schema GET operations on fields, dynamicFields, fieldTypes, copyField are reimplemented as a part of the bulk API with less details (noble) +* SOLR-8766 : deprecated tag in solrconfig.xml is removed (noble) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml index b8d1ca66e22f..ab1acd9ec556 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -1487,11 +1487,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml b/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml index 65637c19e514..f5ce41b4a710 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml @@ -1508,10 +1508,4 @@ --> - - - - *:* - - diff --git a/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml b/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml index 691643fc90b6..d40b16de41d9 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml @@ -1511,11 +1511,4 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - - diff --git a/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml b/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml index b8d1ca66e22f..ab1acd9ec556 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml @@ -1487,11 +1487,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml b/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml index 336c789ea131..9d33201ba331 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml @@ -1510,11 +1510,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/example/example-DIH/solr/db/conf/solrconfig.xml b/solr/example/example-DIH/solr/db/conf/solrconfig.xml index 6050c1fba56c..0a4c6ea5ae65 100644 --- a/solr/example/example-DIH/solr/db/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/db/conf/solrconfig.xml @@ -1486,11 +1486,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/example/example-DIH/solr/mail/conf/solrconfig.xml b/solr/example/example-DIH/solr/mail/conf/solrconfig.xml index 59861b23643f..1cf7d8975bf4 100644 --- a/solr/example/example-DIH/solr/mail/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/mail/conf/solrconfig.xml @@ -1489,11 +1489,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/example/example-DIH/solr/rss/conf/solrconfig.xml b/solr/example/example-DIH/solr/rss/conf/solrconfig.xml index 4a9f5e81c6e0..8db8e55fcdbe 100644 --- a/solr/example/example-DIH/solr/rss/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/rss/conf/solrconfig.xml @@ -1485,11 +1485,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/example/example-DIH/solr/solr/conf/solrconfig.xml b/solr/example/example-DIH/solr/solr/conf/solrconfig.xml index 836b1fd14aab..318fbf8bb022 100644 --- a/solr/example/example-DIH/solr/solr/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/solr/conf/solrconfig.xml @@ -1485,11 +1485,5 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - diff --git a/solr/example/example-DIH/solr/tika/conf/solrconfig.xml b/solr/example/example-DIH/solr/tika/conf/solrconfig.xml index 98b2a8ec2cf9..689429897be3 100644 --- a/solr/example/example-DIH/solr/tika/conf/solrconfig.xml +++ b/solr/example/example-DIH/solr/tika/conf/solrconfig.xml @@ -1463,11 +1463,4 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - - diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml index 7e488a895dc9..7aa682a00176 100644 --- a/solr/example/files/conf/solrconfig.xml +++ b/solr/example/files/conf/solrconfig.xml @@ -1505,10 +1505,4 @@ --> - - - - *:* - - diff --git a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml index 6e76b306eea2..a4c00b2af6f0 100644 --- a/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml @@ -559,9 +559,4 @@ - - - *:* - - diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml index 65a3cb8ddd4a..bd58d3634217 100644 --- a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml @@ -1479,11 +1479,4 @@ EditorialMarkerFactory will do exactly that: --> - - - - - *:* - - diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml index 70dc965d0e5b..be3c16fdbfdd 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml @@ -1698,10 +1698,4 @@ --> - - - - *:* - - From 51278ff4c038c3baf96868a6fa2258392f45a98c Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 08:40:39 -0500 Subject: [PATCH 0061/1113] LUCENE-7075: remove legacy numericutils usage from test. --- .../lucene/search/join/TestBlockJoin.java | 72 ++++++++++--------- .../java/org/apache/lucene/util/TestUtil.java | 23 ------ 2 files changed, 38 insertions(+), 57 deletions(-) diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index a1987744cae4..b5f2038a0dcf 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StoredField; @@ -52,10 +52,8 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; @@ -73,8 +71,6 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -93,7 +89,7 @@ private Document makeResume(String name, String country) { private Document makeJob(String skill, int year) { Document job = new Document(); job.add(newStringField("skill", skill, Field.Store.YES)); - job.add(new LegacyIntField("year", year, Field.Store.NO)); + job.add(new IntPoint("year", year)); job.add(new StoredField("year", year)); return job; } @@ -102,7 +98,7 @@ private Document makeJob(String skill, int year) { private Document makeQualification(String qualification, int year) { Document job = new Document(); job.add(newStringField("qualification", qualification, Field.Store.YES)); - job.add(new LegacyIntField("year", year, Field.Store.NO)); + job.add(new IntPoint("year", year)); return job; } @@ -135,7 +131,7 @@ public void testEmptyChildFilter() throws Exception { BooleanQuery.Builder childQuery = new BooleanQuery.Builder(); childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST)); - childQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST)); + childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST)); ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg); @@ -189,7 +185,7 @@ public void testSimple() throws Exception { // Define child document criteria (finds an example of relevant work experience) BooleanQuery.Builder childQuery = new BooleanQuery.Builder(); childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST)); - childQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST)); + childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST)); // Define parent document criteria (find a resident in the UK) Query parentQuery = new TermQuery(new Term("country", "United Kingdom")); @@ -269,23 +265,30 @@ public void testBugCausedByRewritingTwice() throws IOException { w.close(); IndexSearcher s = newSearcher(r); - MultiTermQuery qc = LegacyNumericRangeQuery.newIntRange("year", 2007, 2007, true, true); // Hacky: this causes the query to need 2 rewrite // iterations: - qc.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(IntPoint.newExactQuery("year", 2007), BooleanClause.Occur.MUST); + Query qc = new Query() { + @Override + public Query rewrite(IndexReader reader) throws IOException { + return builder.build(); + } + + @Override + public String toString(String field) { + return "hack!"; + } + }; BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); CheckJoinIndex.check(r, parentsFilter); - int h1 = qc.hashCode(); Query qw1 = qc.rewrite(r); - int h2 = qw1.hashCode(); Query qw2 = qw1.rewrite(r); - int h3 = qw2.hashCode(); - assertTrue(h1 != h2); - assertTrue(h2 != h3); - assertTrue(h3 != h1); + assertNotSame(qc, qw1); + assertNotSame(qw1, qw2); ToParentBlockJoinQuery qp = new ToParentBlockJoinQuery(qc, parentsFilter, ScoreMode.Max); ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 10, true, true); @@ -342,7 +345,7 @@ public void testSimpleFilter() throws Exception { // Define child document criteria (finds an example of relevant work experience) BooleanQuery.Builder childQuery = new BooleanQuery.Builder(); childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST)); - childQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST)); + childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST)); // Define parent document criteria (find a resident in the UK) Query parentQuery = new TermQuery(new Term("country", "United Kingdom")); @@ -516,7 +519,7 @@ public void testRandom() throws Exception { for(int parentDocID=0;parentDocID joinDocs = new ArrayList<>(); @@ -563,7 +566,7 @@ public void testRandom() throws Exception { Document joinChildDoc = new Document(); joinDocs.add(joinChildDoc); - Field childID = new LegacyIntField("childID", childDocID, Field.Store.YES); + Field childID = new StoredField("childID", childDocID); childDoc.add(childID); joinChildDoc.add(childID); childID = new NumericDocValuesField("childID", childDocID); @@ -596,7 +599,7 @@ public void testRandom() throws Exception { } if (doDeletes) { - joinChildDoc.add(new LegacyIntField("blockID", parentDocID, Field.Store.NO)); + joinChildDoc.add(new IntPoint("blockID", parentDocID)); } w.addDocument(childDoc); @@ -611,14 +614,15 @@ public void testRandom() throws Exception { } } - BytesRefBuilder term = new BytesRefBuilder(); - for(int deleteID : toDelete) { - if (VERBOSE) { - System.out.println("DELETE parentID=" + deleteID); + if (!toDelete.isEmpty()) { + // TODO: we should add newSetQuery(String, Collection) ? this is awkward. + int[] array = new int[toDelete.size()]; + for (int i = 0; i < toDelete.size(); i++) { + array[i] = toDelete.get(i); } - LegacyNumericUtils.intToPrefixCoded(deleteID, 0, term); - w.deleteDocuments(new Term("blockID", term.toBytesRef())); - joinW.deleteDocuments(new Term("blockID", term.toBytesRef())); + Query query = IntPoint.newSetQuery("blockID", array); + w.deleteDocuments(query); + joinW.deleteDocuments(query); } final IndexReader r = w.getReader(); @@ -1061,11 +1065,11 @@ public void testMultiChildTypes() throws Exception { // Define child document criteria (finds an example of relevant work experience) BooleanQuery.Builder childJobQuery = new BooleanQuery.Builder(); childJobQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST)); - childJobQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST)); + childJobQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST)); BooleanQuery.Builder childQualificationQuery = new BooleanQuery.Builder(); childQualificationQuery.add(new BooleanClause(new TermQuery(new Term("qualification", "maths")), Occur.MUST)); - childQualificationQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 1980, 2000, true, true), Occur.MUST)); + childQualificationQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 1980, 2000), Occur.MUST)); // Define parent document criteria (find a resident in the UK) @@ -1210,7 +1214,7 @@ public void testGetTopGroups() throws Exception { // Define child document criteria (finds an example of relevant work experience) BooleanQuery.Builder childQuery = new BooleanQuery.Builder(); childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST)); - childQuery.add(new BooleanClause(LegacyNumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST)); + childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST)); // Wrap the child document query to 'join' any matches // up to corresponding parent: @@ -1707,7 +1711,7 @@ public void testMultiChildQueriesOfDiffParentLevels() throws Exception { Query resumeQuery = new ToChildBlockJoinQuery(new TermQuery(new Term("country","rv" + qrv)), resumeFilter); - Query jobQuery = new ToChildBlockJoinQuery(LegacyNumericRangeQuery.newIntRange("year", qjv, qjv, true, true), + Query jobQuery = new ToChildBlockJoinQuery(IntPoint.newRangeQuery("year", qjv, qjv), jobFilter); BooleanQuery.Builder fullQuery = new BooleanQuery.Builder(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index 50692e81322a..5e328ba13661 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -61,11 +61,6 @@ import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType.LegacyNumericType; -import org.apache.lucene.document.LegacyDoubleField; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.CheckIndex; @@ -1072,7 +1067,6 @@ public static Document cloneDocument(Document doc1) { final Field field2; final DocValuesType dvType = field1.fieldType().docValuesType(); final int dimCount = field1.fieldType().pointDimensionCount(); - final LegacyNumericType numType = field1.fieldType().numericType(); if (dvType != DocValuesType.NONE) { switch(dvType) { case NUMERIC: @@ -1092,23 +1086,6 @@ public static Document cloneDocument(Document doc1) { byte[] bytes = new byte[br.length]; System.arraycopy(br.bytes, br.offset, bytes, 0, br.length); field2 = new BinaryPoint(field1.name(), bytes, field1.fieldType()); - } else if (numType != null) { - switch (numType) { - case INT: - field2 = new LegacyIntField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); - break; - case FLOAT: - field2 = new LegacyFloatField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); - break; - case LONG: - field2 = new LegacyLongField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); - break; - case DOUBLE: - field2 = new LegacyDoubleField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); - break; - default: - throw new IllegalStateException("unknown Type: " + numType); - } } else { field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType()); } From 981ad999d075266423b01ece3be31ada3f89913b Mon Sep 17 00:00:00 2001 From: Varun Thacker Date: Tue, 8 Mar 2016 19:11:00 +0530 Subject: [PATCH 0062/1113] SOLR-8766: Remove support for admin/gettableFiles as well --- solr/CHANGES.txt | 3 +- .../java/org/apache/solr/core/SolrCore.java | 51 ------------------- 2 files changed, 2 insertions(+), 52 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index fa43ecec71cf..a002efa8282e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -415,7 +415,8 @@ Other Changes * SOLR-8736: schema GET operations on fields, dynamicFields, fieldTypes, copyField are reimplemented as a part of the bulk API with less details (noble) -* SOLR-8766 : deprecated tag in solrconfig.xml is removed (noble) +* SOLR-8766: Remove deprecated tag in solrconfig.xml and support for admin/gettableFiles + (noble, Jason Gerlowski, Varun Thacker) ================== 5.5.1 ================== diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index f4c40f842f7d..cde878a5ebf3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -726,9 +726,6 @@ public SolrCore(String name, String dataDir, SolrConfig config, reqHandlers = new RequestHandlers(this); reqHandlers.initHandlersFromConfig(solrConfig); - // Handle things that should eventually go away - initDeprecatedSupport(); - statsCache = initStatsCache(); // cause the executor to stall so firstSearcher events won't fire @@ -2269,54 +2266,6 @@ public ValueSourceParser getValueSourceParser(String parserName) { return valueSourceParsers.get(parserName); } - /** - * Manage anything that should be taken care of in case configs change - */ - private void initDeprecatedSupport() - { - // TODO -- this should be removed in deprecation release... - String gettable = solrConfig.get("admin/gettableFiles", null ); - if( gettable != null ) { - log.warn( - "solrconfig.xml uses deprecated , Please "+ - "update your config to use the ShowFileRequestHandler." ); - if( getRequestHandler( "/admin/file" ) == null ) { - NamedList invariants = new NamedList<>(); - - // Hide everything... - Set hide = new HashSet<>(); - - for (String file : solrConfig.getResourceLoader().listConfigDir()) { - hide.add(file.toUpperCase(Locale.ROOT)); - } - - // except the "gettable" list - StringTokenizer st = new StringTokenizer( gettable ); - while( st.hasMoreTokens() ) { - hide.remove( st.nextToken().toUpperCase(Locale.ROOT) ); - } - for( String s : hide ) { - invariants.add( ShowFileRequestHandler.HIDDEN, s ); - } - - NamedList args = new NamedList<>(); - args.add( "invariants", invariants ); - ShowFileRequestHandler handler = new ShowFileRequestHandler(); - handler.init( args ); - reqHandlers.register("/admin/file", handler); - - log.warn( "adding ShowFileRequestHandler with hidden files: "+hide ); - } - } - - String facetSort = solrConfig.get("//bool[@name='facet.sort']", null); - if (facetSort != null) { - log.warn( - "solrconfig.xml uses deprecated . Please "+ - "update your config to use ."); - } - } - /** * Creates and initializes a RestManager based on configuration args in solrconfig.xml. * RestManager provides basic storage support for managed resource data, such as to From b6ffd27bf33adbb3ef92a41a36581c6f101535f2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 10:54:29 -0500 Subject: [PATCH 0063/1113] LUCENE-7075: clean up LegacyNumeric* in .document javadocs --- .../org/apache/lucene/document/Document.java | 6 ++---- .../org/apache/lucene/document/Field.java | 19 ++++++++++++++----- .../apache/lucene/document/TestDocument.java | 4 ++-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/Document.java b/lucene/core/src/java/org/apache/lucene/document/Document.java index cdba083942fc..2f44444babeb 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Document.java +++ b/lucene/core/src/java/org/apache/lucene/document/Document.java @@ -199,8 +199,7 @@ public final List getFields() { * Returns an array of values of the field specified as the method parameter. * This method returns an empty array when there are no * matching fields. It never returns null. - * For {@link LegacyIntField}, {@link LegacyLongField}, {@link - * LegacyFloatField} and {@link LegacyDoubleField} it returns the string value of the number. If you want + * For a numeric {@link StoredField} it returns the string value of the number. If you want * the actual numeric field instances back, use {@link #getFields}. * @param name the name of the field * @return a String[] of field values @@ -224,8 +223,7 @@ public final String[] getValues(String name) { * this document, or null. If multiple fields exist with this name, this * method returns the first value added. If only binary fields with this name * exist, returns null. - * For {@link LegacyIntField}, {@link LegacyLongField}, {@link - * LegacyFloatField} and {@link LegacyDoubleField} it returns the string value of the number. If you want + * For a numeric {@link StoredField} it returns the string value of the number. If you want * the actual numeric field instance back, use {@link #getField}. */ public final String get(String name) { diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java index dff2e58aa6ef..550d1fd1797f 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Field.java +++ b/lucene/core/src/java/org/apache/lucene/document/Field.java @@ -33,11 +33,20 @@ /** * Expert: directly create a field for a document. Most - * users should use one of the sugar subclasses: {@link - * LegacyIntField}, {@link LegacyLongField}, {@link LegacyFloatField}, {@link - * LegacyDoubleField}, {@link BinaryDocValuesField}, {@link - * NumericDocValuesField}, {@link SortedDocValuesField}, {@link - * StringField}, {@link TextField}, {@link StoredField}. + * users should use one of the sugar subclasses: + *
      + *
    • {@link TextField}: {@link Reader} or {@link String} indexed for full-text search + *
    • {@link StringField}: {@link String} indexed verbatim as a single token + *
    • {@link IntPoint}: {@code int} indexed for exact/range queries. + *
    • {@link LongPoint}: {@code long} indexed for exact/range queries. + *
    • {@link FloatPoint}: {@code float} indexed for exact/range queries. + *
    • {@link DoublePoint}: {@code double} indexed for exact/range queries. + *
    • {@link SortedDocValuesField}: {@code byte[]} indexed column-wise for sorting/faceting + *
    • {@link SortedSetDocValuesField}: {@code SortedSet} indexed column-wise for sorting/faceting + *
    • {@link NumericDocValuesField}: {@code long} indexed column-wise for sorting/faceting + *
    • {@link SortedNumericDocValuesField}: {@code SortedSet} indexed column-wise for sorting/faceting + *
    • {@link StoredField}: Stored-only value for retrieving in summary results + *
    * *

    A field is a section of a Document. Each field has three * parts: name, type and value. Values may be text diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java index bd873ec8a9cd..50c1ed0a919a 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java @@ -340,10 +340,10 @@ public void testInvalidFields() { public void testNumericFieldAsString() throws Exception { Document doc = new Document(); - doc.add(new LegacyIntField("int", 5, Field.Store.YES)); + doc.add(new StoredField("int", 5)); assertEquals("5", doc.get("int")); assertNull(doc.get("somethingElse")); - doc.add(new LegacyIntField("int", 4, Field.Store.YES)); + doc.add(new StoredField("int", 4)); assertArrayEquals(new String[] { "5", "4" }, doc.getValues("int")); Directory dir = newDirectory(); From f297e900c0d0e15d2b5f9a7bd7dbe1978315902b Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 8 Mar 2016 11:17:12 -0500 Subject: [PATCH 0064/1113] LUCENE-7075: clean up LegacyNumerics* usage in queries/ tests --- .../queries/function/FunctionTestSetup.java | 8 +++----- .../queries/function/TestFunctionQuerySort.java | 4 ++-- .../queries/function/TestValueSources.java | 16 ---------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java index d5a587df61ee..2764a8fc5b4a 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java @@ -21,12 +21,10 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; -import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; @@ -143,11 +141,11 @@ private static void addDoc(RandomIndexWriter iw, int i) throws Exception { f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search d.add(f); - f = new LegacyIntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring + f = new StoredField(INT_FIELD, scoreAndID); // for function scoring d.add(f); d.add(new NumericDocValuesField(INT_FIELD, scoreAndID)); - f = new LegacyFloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring + f = new StoredField(FLOAT_FIELD, scoreAndID); // for function scoring d.add(f); d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID))); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java index b9e1eb2cade2..67f67b2dc573 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java @@ -20,8 +20,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; @@ -102,7 +102,7 @@ public void testSearchAfterWhenSortingByFunctionValues() throws IOException { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); - Field field = new LegacyIntField("value", 0, Field.Store.YES); + Field field = new StoredField("value", 0); Field dvField = new NumericDocValuesField("value", 0); doc.add(field); doc.add(dvField); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java index 37a32da5db65..509e0ab3cb2f 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java @@ -24,11 +24,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyFloatField; -import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; @@ -120,20 +116,12 @@ public static void beforeClass() throws Exception { document.add(idField); Field idDVField = new SortedDocValuesField("id", new BytesRef()); document.add(idDVField); - Field doubleField = new LegacyDoubleField("double", 0d, Field.Store.NO); - document.add(doubleField); Field doubleDVField = new NumericDocValuesField("double", 0); document.add(doubleDVField); - Field floatField = new LegacyFloatField("float", 0f, Field.Store.NO); - document.add(floatField); Field floatDVField = new NumericDocValuesField("float", 0); document.add(floatDVField); - Field intField = new LegacyIntField("int", 0, Field.Store.NO); - document.add(intField); Field intDVField = new NumericDocValuesField("int", 0); document.add(intDVField); - Field longField = new LegacyLongField("long", 0L, Field.Store.NO); - document.add(longField); Field longDVField = new NumericDocValuesField("long", 0); document.add(longDVField); Field stringField = new StringField("string", "", Field.Store.NO); @@ -146,13 +134,9 @@ public static void beforeClass() throws Exception { for (String [] doc : documents) { idField.setStringValue(doc[0]); idDVField.setBytesValue(new BytesRef(doc[0])); - doubleField.setDoubleValue(Double.valueOf(doc[1])); doubleDVField.setLongValue(Double.doubleToRawLongBits(Double.valueOf(doc[1]))); - floatField.setFloatValue(Float.valueOf(doc[2])); floatDVField.setLongValue(Float.floatToRawIntBits(Float.valueOf(doc[2]))); - intField.setIntValue(Integer.valueOf(doc[3])); intDVField.setLongValue(Integer.valueOf(doc[3])); - longField.setLongValue(Long.valueOf(doc[4])); longDVField.setLongValue(Long.valueOf(doc[4])); stringField.setStringValue(doc[5]); stringDVField.setBytesValue(new BytesRef(doc[5])); From ae59bc0785b57ff5ce8cc4f88cd7728de3735e18 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 8 Mar 2016 12:36:46 -0500 Subject: [PATCH 0065/1113] LUCENE-6952: Make most Filter* classes abstract. (cherry picked from commit 9393a31) --- lucene/CHANGES.txt | 4 ++++ .../org/apache/lucene/index/FilterCodecReader.java | 2 +- .../java/org/apache/lucene/index/FilterLeafReader.java | 10 +++++----- .../java/org/apache/lucene/search/FilterCollector.java | 2 +- .../org/apache/lucene/search/FilterLeafCollector.java | 2 +- .../java/org/apache/lucene/store/FilterDirectory.java | 2 +- .../org/apache/lucene/index/TestFilterLeafReader.java | 2 +- .../org/apache/lucene/store/TestFilterDirectory.java | 4 ++-- .../org/apache/lucene/index/MockRandomMergePolicy.java | 2 +- .../org/apache/lucene/mockfile/FilterFileChannel.java | 2 +- .../org/apache/lucene/mockfile/FilterFileStore.java | 2 +- .../org/apache/lucene/mockfile/FilterFileSystem.java | 2 +- .../lucene/mockfile/FilterFileSystemProvider.java | 2 +- .../apache/lucene/mockfile/FilterOutputStream2.java | 2 +- 14 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index ca59e6b8f1e1..3abd4fb5e80a 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -128,6 +128,10 @@ API Changes * LUCENE-7056: Geo3D classes are in different packages now. (David Smiley) +* LUCENE-6952: These classes are now abstract: FilterCodecReader, FilterLeafReader, + FilterCollector, FilterDirectory. And some Filter* classes in + lucene-test-framework too. (David Smiley) + Optimizations * LUCENE-6891: Use prefix coding when writing points in diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java index 41f09843aed7..c35dc6719c91 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java @@ -32,7 +32,7 @@ * uses as its basic source of data, possibly transforming the data along the * way or providing additional functionality. */ -public class FilterCodecReader extends CodecReader { +public abstract class FilterCodecReader extends CodecReader { /** * The underlying CodecReader instance. */ diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java index 98365a6cf658..1d593c32a127 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java @@ -44,7 +44,7 @@ * overridden as well if the {@link #getLiveDocs() live docs} are not changed * either. */ -public class FilterLeafReader extends LeafReader { +public abstract class FilterLeafReader extends LeafReader { /** Get the wrapped instance by reader as long as this reader is * an instance of {@link FilterLeafReader}. */ @@ -57,7 +57,7 @@ public static LeafReader unwrap(LeafReader reader) { /** Base class for filtering {@link Fields} * implementations. */ - public static class FilterFields extends Fields { + public abstract static class FilterFields extends Fields { /** The underlying Fields instance. */ protected final Fields in; @@ -93,7 +93,7 @@ public int size() { * these terms are going to be intersected with automata, you could consider * overriding {@link #intersect} for better performance. */ - public static class FilterTerms extends Terms { + public abstract static class FilterTerms extends Terms { /** The underlying Terms instance. */ protected final Terms in; @@ -160,7 +160,7 @@ public Object getStats() throws IOException { } /** Base class for filtering {@link TermsEnum} implementations. */ - public static class FilterTermsEnum extends TermsEnum { + public abstract static class FilterTermsEnum extends TermsEnum { /** The underlying TermsEnum instance. */ protected final TermsEnum in; @@ -223,7 +223,7 @@ public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { } /** Base class for filtering {@link PostingsEnum} implementations. */ - public static class FilterPostingsEnum extends PostingsEnum { + public abstract static class FilterPostingsEnum extends PostingsEnum { /** The underlying PostingsEnum instance. */ protected final PostingsEnum in; diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterCollector.java b/lucene/core/src/java/org/apache/lucene/search/FilterCollector.java index d290330f5d69..d4ec91492675 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilterCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilterCollector.java @@ -26,7 +26,7 @@ * * @lucene.experimental */ -public class FilterCollector implements Collector { +public abstract class FilterCollector implements Collector { protected final Collector in; diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java index ab15babad0be..b55410c87c72 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java @@ -24,7 +24,7 @@ * * @lucene.experimental */ -public class FilterLeafCollector implements LeafCollector { +public abstract class FilterLeafCollector implements LeafCollector { protected final LeafCollector in; diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java index 5df571358aec..8148b5ac3a38 100644 --- a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java @@ -29,7 +29,7 @@ * {@link Directory} or {@link BaseDirectory} rather than try to reuse * functionality of existing {@link Directory}s by extending this class. * @lucene.internal */ -public class FilterDirectory extends Directory { +public abstract class FilterDirectory extends Directory { /** Get the wrapped instance by dir as long as this reader is * an instance of {@link FilterDirectory}. */ diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java index 82fb3bce2f67..cad47a4e1268 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java @@ -196,7 +196,7 @@ public void testUnwrap() throws IOException { w.addDocument(new Document()); DirectoryReader dr = w.getReader(); LeafReader r = dr.leaves().get(0).reader(); - FilterLeafReader r2 = new FilterLeafReader(r); + FilterLeafReader r2 = new FilterLeafReader(r) {}; assertEquals(r, r2.getDelegate()); assertEquals(r, FilterLeafReader.unwrap(r2)); w.close(); diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java index 7fe9bc2be0d4..6224140e3ca4 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java @@ -29,7 +29,7 @@ public class TestFilterDirectory extends BaseDirectoryTestCase { @Override protected Directory getDirectory(Path path) throws IOException { - return new FilterDirectory(new RAMDirectory()); + return new FilterDirectory(new RAMDirectory()) {}; } @Test @@ -48,7 +48,7 @@ public void testOverrides() throws Exception { public void testUnwrap() throws IOException { Directory dir = FSDirectory.open(createTempDir()); - FilterDirectory dir2 = new FilterDirectory(dir); + FilterDirectory dir2 = new FilterDirectory(dir) {}; assertEquals(dir, dir2.getDelegate()); assertEquals(dir, FilterDirectory.unwrap(dir2)); dir2.close(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java index bcee1b6c4e18..b40ac2685d51 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/MockRandomMergePolicy.java @@ -159,7 +159,7 @@ public List getMergeReaders() throws IOException { if (LuceneTestCase.VERBOSE) { System.out.println("NOTE: MockRandomMergePolicy now swaps in a SlowCodecReaderWrapper for merging reader=" + readers.get(i)); } - readers.set(i, SlowCodecReaderWrapper.wrap(new FilterLeafReader(readers.get(i)))); + readers.set(i, SlowCodecReaderWrapper.wrap(new FilterLeafReader(readers.get(i)) {})); } else if (thingToDo == 1) { // renumber fields // NOTE: currently this only "blocks" bulk merges just by diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileChannel.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileChannel.java index 0c95af0ee637..ccc6e7abc9eb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileChannel.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileChannel.java @@ -31,7 +31,7 @@ * source of data, possibly transforming the data along the * way or providing additional functionality. */ -public class FilterFileChannel extends FileChannel { +public abstract class FilterFileChannel extends FileChannel { /** * The underlying {@code FileChannel} instance. diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileStore.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileStore.java index dc90799d6ece..423b32d75ba1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileStore.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileStore.java @@ -28,7 +28,7 @@ * source of data, possibly transforming the data along the * way or providing additional functionality. */ -public class FilterFileStore extends FileStore { +public abstract class FilterFileStore extends FileStore { /** * The underlying {@code FileStore} instance. diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java index d79ed35e7f50..e24506d7cd52 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java @@ -131,7 +131,7 @@ public boolean hasNext() { @Override public FileStore next() { - return new FilterFileStore(iterator.next(), parent.getScheme()); + return new FilterFileStore(iterator.next(), parent.getScheme()) {}; } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystemProvider.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystemProvider.java index c9c016551c85..8a7ff754e014 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystemProvider.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystemProvider.java @@ -48,7 +48,7 @@ * source of data, possibly transforming the data along the * way or providing additional functionality. */ -public class FilterFileSystemProvider extends FileSystemProvider { +public abstract class FilterFileSystemProvider extends FileSystemProvider { /** * The underlying {@code FileSystemProvider}. diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterOutputStream2.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterOutputStream2.java index dbf7a95bb718..5413c87dd710 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterOutputStream2.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterOutputStream2.java @@ -34,7 +34,7 @@ * that just overrides {@code close} will not force bytes to be * written one-at-a-time. */ -public class FilterOutputStream2 extends OutputStream { +public abstract class FilterOutputStream2 extends OutputStream { /** * The underlying {@code OutputStream} instance. From 0ab23fd1f5e38cdd4fe742f70f66d97e666b9e80 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 8 Mar 2016 13:28:28 -0500 Subject: [PATCH 0066/1113] LUCENE-7083: default points merge logic should not ask a reader to merge points on a field that doesn't exist in that segment --- .../apache/lucene/codecs/PointsWriter.java | 6 ++++++ .../org/apache/lucene/index/IndexWriter.java | 1 + .../apache/lucene/index/TestPointValues.java | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java index 56689ecae7c7..53db281777af 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java @@ -54,6 +54,12 @@ public void intersect(String fieldName, IntersectVisitor mergedVisitor) throws I // This segment has no points continue; } + FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldName); + if (readerFieldInfo == null) { + // This segment never saw this field + continue; + } + MergeState.DocMap docMap = mergeState.docMaps[i]; int docBase = mergeState.docBase[i]; pointsReader.intersect(fieldInfo.name, diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 57ce3ddea6d1..66a0e73b601b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -4116,6 +4116,7 @@ private int mergeMiddle(MergePolicy.OneMerge merge, MergePolicy mergePolicy) thr (mergeState.mergeFieldInfos.hasDocValues() ? "docValues" : "no docValues") + "; " + (mergeState.mergeFieldInfos.hasProx() ? "prox" : "no prox") + "; " + (mergeState.mergeFieldInfos.hasProx() ? "freqs" : "no freqs") + "; " + + (mergeState.mergeFieldInfos.hasPointValues() ? "points" : "no points") + "; " + String.format(Locale.ROOT, "%.1f sec (%.1f sec stopped, %.1f sec paused) to merge segment [%.2f MB, %.2f MB/sec]", sec, diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 7231b1afc6fa..9ced11a07187 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.PointValues; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -562,4 +563,23 @@ public void testDeleteAllPointDocs() throws Exception { r.close(); dir.close(); } + + public void testPointsFieldMissingFromOneSegment() throws Exception { + Directory dir = FSDirectory.open(createTempDir()); + IndexWriterConfig iwc = new IndexWriterConfig(null); + IndexWriter w = new IndexWriter(dir, iwc); + Document doc = new Document(); + doc.add(new StringField("id", "0", Field.Store.NO)); + doc.add(new IntPoint("int0", 0)); + w.addDocument(doc); + w.commit(); + + doc = new Document(); + doc.add(new IntPoint("int1", 17)); + w.addDocument(doc); + w.forceMerge(1); + + w.close(); + dir.close(); + } } From 7c527cc346c0863c4b0289fc23d8870290b41233 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 8 Mar 2016 13:34:08 -0500 Subject: [PATCH 0067/1113] don't use slow composite reader in this test --- .../lucene/index/ThreadedIndexingAndSearchingTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index a3e049de3151..80c3903e4828 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -480,7 +480,7 @@ public void warm(LeafReader reader) throws IOException { } } - IndexSearcher searcher = newSearcher(reader); + IndexSearcher searcher = newSearcher(reader, false); sum += searcher.search(new TermQuery(new Term("body", "united")), 10).totalHits; if (VERBOSE) { From 7c69f272337fb16d7aea0995aed48d19ff62b1ec Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 8 Mar 2016 14:36:04 -0500 Subject: [PATCH 0068/1113] SOLR-6926: fix smokeTestRelease.py to stop calling ant example. (cherry picked from commit e490b32) --- dev-tools/scripts/smokeTestRelease.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index aa367f6d5313..c5b8d6e2f3cd 100644 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -700,7 +700,7 @@ def verifyUnpacked(java, project, artifact, unpackPath, gitRevision, version, te checkJavadocpathFull('%s/solr/build/docs' % unpackPath, False) print(' test solr example w/ Java 8...') - java.run_java8('ant clean example', '%s/antexample.log' % unpackPath) + java.run_java8('ant clean server', '%s/antexample.log' % unpackPath) testSolrExample(unpackPath, java.java8_home, True) os.chdir('..') @@ -782,6 +782,7 @@ def readSolrOutput(p, startupEvent, failureEvent, logFile): f.close() def testSolrExample(unpackPath, javaPath, isSrc): + # test solr using some examples it comes with logFile = '%s/solr-example.log' % unpackPath if isSrc: os.chdir(unpackPath+'/solr') From 122c24807196205f2bf65d37e6565caa7c6be987 Mon Sep 17 00:00:00 2001 From: Shai Erera Date: Tue, 8 Mar 2016 22:11:18 +0200 Subject: [PATCH 0069/1113] SOLR-8793: Fix stale commit files' size computation in LukeRequestHandler --- solr/CHANGES.txt | 5 +++++ .../solr/handler/admin/LukeRequestHandler.java | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a002efa8282e..818cf72c235a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -433,6 +433,11 @@ Bug Fixes * SOLR-8712: Variable solr.core.instanceDir was not being resolved (Kristine Jetzke, Shawn Heisey, Alan Woodward) +* SOLR-8793: Fix Core admin status API to not fail when computing the size of the segments_N + file if the file no longer exists (for example, if a commit happened and the IndexReader + hasn't refreshed yet). In this case the reported size of the file is -1. + (Shai Erera, Alexey Serba, Richard Coggins) + ======================= 5.5.0 ======================= Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 0ec6d79fecbb..450a5052d1a3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -582,7 +582,7 @@ public static SimpleOrderedMap getIndexInfo(DirectoryReader reader) thro IndexCommit indexCommit = reader.getIndexCommit(); String segmentsFileName = indexCommit.getSegmentsFileName(); indexInfo.add("segmentsFile", segmentsFileName); - indexInfo.add("segmentsFileSizeInBytes", indexCommit.getDirectory().fileLength(segmentsFileName)); + indexInfo.add("segmentsFileSizeInBytes", getFileLength(indexCommit.getDirectory(), segmentsFileName)); Map userData = indexCommit.getUserData(); indexInfo.add("userData", userData); String s = userData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY); @@ -592,6 +592,16 @@ public static SimpleOrderedMap getIndexInfo(DirectoryReader reader) thro return indexInfo; } + private static long getFileLength(Directory dir, String filename) { + try { + return dir.fileLength(filename); + } catch (IOException e) { + // Whatever the error is, only log it and return -1. + log.warn("Error getting file length for [{}]", filename, e); + return -1; + } + } + /** Returns the sum of RAM bytes used by each segment */ private static long getIndexHeapUsed(DirectoryReader reader) { long indexHeapRamBytesUsed = 0; From 6eea1f94c468d060bbac4f23de4e1b9d64c691f1 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 8 Mar 2016 15:21:37 -0500 Subject: [PATCH 0070/1113] improve testing for sparse points --- .../index/TestBackwardsCompatibility.java | 2 +- .../index/TestFlushByRamOrCountsPolicy.java | 2 +- .../lucene/index/TestForceMergeForever.java | 2 +- .../org/apache/lucene/index/TestNorms.java | 2 +- .../apache/lucene/index/TestPointValues.java | 46 +++++++++++++++ .../lucene/index/TestRollingUpdates.java | 2 +- .../apache/lucene/index/TestTermsEnum.java | 2 +- .../lucene/store/TestNRTCachingDirectory.java | 2 +- .../org/apache/lucene/util/fst/TestFSTs.java | 2 +- .../analyzing/TestFreeTextSuggester.java | 2 +- .../ThreadedIndexingAndSearchingTestCase.java | 2 +- .../lucene/search/ShardSearchingTestBase.java | 2 +- .../org/apache/lucene/util/LineFileDocs.java | 58 ++++++++++++------- 13 files changed, 95 insertions(+), 31 deletions(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index a2a749187913..6121bca6e5c9 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -148,7 +148,7 @@ public void testCreateMoreTermsIndex() throws Exception { IndexWriterConfig conf = new IndexWriterConfig(analyzer) .setMergePolicy(mp).setUseCompoundFile(false); IndexWriter writer = new IndexWriter(dir, conf); - LineFileDocs docs = new LineFileDocs(null, true); + LineFileDocs docs = new LineFileDocs(null); for(int i=0;i<50;i++) { writer.addDocument(docs.nextDoc()); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java index 562913e079ac..993a521a36e3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java @@ -39,7 +39,7 @@ public class TestFlushByRamOrCountsPolicy extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - lineDocFile = new LineFileDocs(random(), true); + lineDocFile = new LineFileDocs(random()); } @AfterClass diff --git a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java index 3edeef15f618..037939534f17 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestForceMergeForever.java @@ -62,7 +62,7 @@ public void test() throws Exception { // Try to make an index that requires merging: w.getConfig().setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 11)); final int numStartDocs = atLeast(20); - final LineFileDocs docs = new LineFileDocs(random(), true); + final LineFileDocs docs = new LineFileDocs(random()); for(int docIDX=0;docIDX Date: Tue, 8 Mar 2016 15:21:19 -0500 Subject: [PATCH 0071/1113] SOLR-8799: Improve error message when tuple can't be read by SolrJ JDBC --- .../solr/client/solrj/io/sql/ResultSetImpl.java | 2 +- .../apache/solr/client/solrj/io/sql/JdbcTest.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java index e2f8cf04f4be..0aa3a4bc5173 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java @@ -78,7 +78,7 @@ class ResultSetImpl implements ResultSet { this.firstTuple = this.solrStream.read(); this.solrStream.pushBack(firstTuple); } catch (IOException e) { - throw new SQLException("Couldn't read first tuple", e); + throw new SQLException(e); } this.resultSetMetaData = new ResultSetMetaDataImpl(this); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java index e1e9739511f3..572491e31e8f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java @@ -396,6 +396,21 @@ public void doTest() throws Exception { } } + + //Test error propagation + props = new Properties(); + props.put("aggregationMode", "facet"); + try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=collection1", props)) { + try (Statement stmt = con.createStatement()) { + try (ResultSet rs = stmt.executeQuery("select crap from collection1 group by a_s " + + "order by sum(a_f) desc")) { + } catch (Exception e) { + String errorMessage = e.getMessage(); + assertTrue(errorMessage.contains("Group by queries must include atleast one aggregate function")); + } + } + } + testDriverMetadata(); } From d0427d8617565cb4782faf1c93cf0ac0a6207680 Mon Sep 17 00:00:00 2001 From: jbernste Date: Tue, 8 Mar 2016 15:25:52 -0500 Subject: [PATCH 0072/1113] SOLR-8799: Update CHANGES.txt --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 818cf72c235a..5ee5b979aa03 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -418,6 +418,8 @@ Other Changes * SOLR-8766: Remove deprecated tag in solrconfig.xml and support for admin/gettableFiles (noble, Jason Gerlowski, Varun Thacker) +* SOLR-8799: Improve error message when tuple can't be read by SolrJ JDBC (Kevin Risden, Joel Bernstein) + ================== 5.5.1 ================== Bug Fixes From acbabac7b574c792daccddfecf29f1adb3da1ec5 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 8 Mar 2016 17:21:12 -0500 Subject: [PATCH 0073/1113] fix random float test to do the +/- 1 ulp in float space --- .../lucene/facet/range/TestRangeFacetCounts.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index 9fde6e3ed9f1..9f8b1096ac90 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -532,6 +532,8 @@ public void testRandomFloats() throws Exception { int[] expectedCounts = new int[numRange]; float minAcceptedValue = Float.POSITIVE_INFINITY; float maxAcceptedValue = Float.NEGATIVE_INFINITY; + boolean[] rangeMinIncl = new boolean[numRange]; + boolean[] rangeMaxIncl = new boolean[numRange]; if (VERBOSE) { System.out.println("TEST: " + numRange + " ranges"); } @@ -582,6 +584,8 @@ public void testRandomFloats() throws Exception { minIncl = random().nextBoolean(); maxIncl = random().nextBoolean(); } + rangeMinIncl[rangeID] = minIncl; + rangeMaxIncl[rangeID] = maxIncl; ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); if (VERBOSE) { @@ -642,7 +646,17 @@ public void testRandomFloats() throws Exception { // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (random().nextBoolean()) { - ddq.add("field", FloatPoint.newRangeQuery("field", (float) range.min, (float) range.max)); + // We must do the nextUp/down in float space, here, because the nextUp that DoubleRange did in double space, when cast back to float, + // in fact does nothing! + float minFloat = (float) range.min; + if (rangeMinIncl[rangeID] == false) { + minFloat = Math.nextUp(minFloat); + } + float maxFloat = (float) range.max; + if (rangeMaxIncl[rangeID] == false) { + maxFloat = Math.nextAfter(maxFloat, Float.NEGATIVE_INFINITY); + } + ddq.add("field", FloatPoint.newRangeQuery("field", minFloat, maxFloat)); } else { ddq.add("field", range.getQuery(fastMatchQuery, vs)); } From 885ad49105ca98cdf2d530005433902ea4e24ffb Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 8 Mar 2016 17:30:30 -0500 Subject: [PATCH 0074/1113] remove troublesome float tests since facets only actually expose doubles --- .../facet/range/TestRangeFacetCounts.java | 203 ------------------ 1 file changed, 203 deletions(-) diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index 9f8b1096ac90..626d772d6a9e 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -23,11 +23,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; -import org.apache.lucene.document.FloatDocValuesField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.facet.DrillDownQuery; import org.apache.lucene.facet.DrillSideways; @@ -52,7 +50,6 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; -import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; @@ -321,37 +318,6 @@ public void testBasicDouble() throws Exception { IOUtils.close(r, d); } - public void testBasicFloat() throws Exception { - Directory d = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), d); - Document doc = new Document(); - FloatDocValuesField field = new FloatDocValuesField("field", 0.0f); - doc.add(field); - for(long l=0;l<100;l++) { - field.setFloatValue(l); - w.addDocument(doc); - } - - IndexReader r = w.getReader(); - - FacetsCollector fc = new FacetsCollector(); - - IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); - - Facets facets = new DoubleRangeFacetCounts("field", new FloatFieldSource("field"), fc, - new DoubleRange("less than 10", 0.0f, true, 10.0f, false), - new DoubleRange("less than or equal to 10", 0.0f, true, 10.0f, true), - new DoubleRange("over 90", 90.0f, false, 100.0f, false), - new DoubleRange("90 or above", 90.0f, true, 100.0f, false), - new DoubleRange("over 1000", 1000.0f, false, Double.POSITIVE_INFINITY, false)); - - assertEquals("dim=field path=[] value=21 childCount=5\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", - facets.getTopChildren(10, "field").toString()); - w.close(); - IOUtils.close(r, d); - } - public void testRandomLongs() throws Exception { Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); @@ -499,175 +465,6 @@ public void testRandomLongs() throws Exception { IOUtils.close(r, dir); } - public void testRandomFloats() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), dir); - - int numDocs = atLeast(1000); - float[] values = new float[numDocs]; - float minValue = Float.POSITIVE_INFINITY; - float maxValue = Float.NEGATIVE_INFINITY; - for(int i=0;i 0 && random().nextInt(10) == 7) { - // Use an existing boundary: - DoubleRange prevRange = ranges[random().nextInt(rangeID)]; - if (random().nextBoolean()) { - min = prevRange.min; - } else { - min = prevRange.max; - } - } else { - min = random().nextDouble(); - } - double max; - if (rangeID > 0 && random().nextInt(10) == 7) { - // Use an existing boundary: - DoubleRange prevRange = ranges[random().nextInt(rangeID)]; - if (random().nextBoolean()) { - max = prevRange.min; - } else { - max = prevRange.max; - } - } else { - max = random().nextDouble(); - } - - if (min > max) { - double x = min; - min = max; - max = x; - } - - // Must truncate to float precision so that the - // drill-down counts (which use NRQ.newFloatRange) - // are correct: - min = (float) min; - max = (float) max; - - boolean minIncl; - boolean maxIncl; - if (min == max) { - minIncl = true; - maxIncl = true; - } else { - minIncl = random().nextBoolean(); - maxIncl = random().nextBoolean(); - } - rangeMinIncl[rangeID] = minIncl; - rangeMaxIncl[rangeID] = maxIncl; - ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); - - if (VERBOSE) { - System.out.println("TEST: range " + rangeID + ": " + ranges[rangeID]); - } - - // Do "slow but hopefully correct" computation of - // expected count: - for(int i=0;i= min; - } else { - accept &= values[i] > min; - } - if (maxIncl) { - accept &= values[i] <= max; - } else { - accept &= values[i] < max; - } - if (VERBOSE) { - System.out.println("TEST: check doc=" + i + " val=" + values[i] + " accept=" + accept); - } - if (accept) { - expectedCounts[rangeID]++; - minAcceptedValue = Math.min(minAcceptedValue, values[i]); - maxAcceptedValue = Math.max(maxAcceptedValue, values[i]); - } - } - } - - FacetsCollector sfc = new FacetsCollector(); - s.search(new MatchAllDocsQuery(), sfc); - Query fastMatchQuery; - if (random().nextBoolean()) { - if (random().nextBoolean()) { - fastMatchQuery = FloatPoint.newRangeQuery("field", minValue, maxValue); - } else { - fastMatchQuery = FloatPoint.newRangeQuery("field", minAcceptedValue, maxAcceptedValue); - } - } else { - fastMatchQuery = null; - } - ValueSource vs = new FloatFieldSource("field"); - Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchQuery, ranges); - FacetResult result = facets.getTopChildren(10, "field"); - assertEquals(numRange, result.labelValues.length); - for(int rangeID=0;rangeID Date: Wed, 9 Mar 2016 10:56:13 +0100 Subject: [PATCH 0075/1113] LUCENE-7080: Sort files to corrupt to prevent HashSet iteration order issues across JVMs --- .../java/org/apache/lucene/store/MockDirectoryWrapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 962062e0312b..7fe7c3b0d5a3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -45,6 +45,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -296,7 +297,11 @@ public synchronized void corruptUnknownFiles() throws IOException { public synchronized void corruptFiles(Collection files) throws IOException { // Must make a copy because we change the incoming unsyncedFiles // when we create temp files, delete, etc., below: - for(String name : new ArrayList<>(files)) { + final List filesToCorrupt = new ArrayList<>(files); + // sort the files otherwise we have reproducibility issues + // across JVMs if the incoming collection is a hashSet etc. + CollectionUtil.timSort(filesToCorrupt); + for(String name : filesToCorrupt) { int damage = randomState.nextInt(6); String action = null; From c1277cda1116a44b3371a3fa8364cc2032e14273 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 7 Mar 2016 19:53:09 +0000 Subject: [PATCH 0076/1113] SOLR-8765: Enforce required parameters in SolrJ Collections API --- solr/CHANGES.txt | 4 + .../org/apache/solr/core/CoreContainer.java | 9 +- .../handler/admin/CollectionsHandler.java | 101 ++-- .../apache/solr/cloud/DeleteStatusTest.java | 172 +++--- .../solrj/request/CollectionAdminRequest.java | 546 ++++++++++++++++-- .../solrj/request/CoreAdminRequest.java | 23 +- .../solrj/util/SolrIdentifierValidator.java | 26 +- 7 files changed, 674 insertions(+), 207 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5ee5b979aa03..feb0e10dc2ea 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -24,6 +24,10 @@ Detailed Change List .processAndWait() to wait for a call to finish without holding HTTP collections open. (Alan Woodward) +* SOLR-8765: Enforce required parameters at query construction time in the SolrJ + Collections API, add static factory methods, and deprecate old setter methods. + (Alan Woodward, Jason Gerlowski) + New Features ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 9ff45eaf1c28..1d614e343dca 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -804,9 +804,7 @@ private SolrCore create(CoreDescriptor dcore, boolean publishState) { SolrCore core = null; try { MDCLoggingContext.setCore(core); - if (!SolrIdentifierValidator.validateCoreName(dcore.getName())) { - throw new SolrException(ErrorCode.BAD_REQUEST, SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.CORE, dcore.getName())); - } + SolrIdentifierValidator.validateCoreName(dcore.getName()); if (zkSys.getZkController() != null) { zkSys.getZkController().preRegister(dcore); } @@ -1009,10 +1007,7 @@ public void unload(String name, boolean deleteIndexDir, boolean deleteDataDir, b } public void rename(String name, String toName) { - if (!SolrIdentifierValidator.validateCoreName(toName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.CORE, - toName)); - } + SolrIdentifierValidator.validateCoreName(toName); try (SolrCore core = getCore(name)) { if (core != null) { registerCore(toName, core, true); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 593dac81bcc6..06968c381484 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -16,45 +16,8 @@ */ package org.apache.solr.handler.admin; -import static org.apache.solr.client.solrj.response.RequestStatusState.*; -import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROP_PREFIX; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET_SHUFFLE; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_ACTIVE_NODES; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOWN; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.REQUESTID; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARD_UNIQUE; -import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER; -import static org.apache.solr.common.cloud.DocCollection.RULE; -import static org.apache.solr.common.cloud.DocCollection.SNITCH; -import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT; -import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; -import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; -import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.*; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; -import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.common.params.CommonParams.VALUE_LONG; -import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; -import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; -import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR; -import static org.apache.solr.common.params.ShardParams._ROUTE_; -import static org.apache.solr.common.util.StrUtils.formatString; - import java.io.IOException; import java.lang.invoke.MethodHandles; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -66,6 +29,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.SolrResponse; @@ -117,8 +82,45 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED; +import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED; +import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; +import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING; +import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED; +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROP_PREFIX; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET_SHUFFLE; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_ACTIVE_NODES; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOWN; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.REQUESTID; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARD_UNIQUE; +import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER; +import static org.apache.solr.common.cloud.DocCollection.RULE; +import static org.apache.solr.common.cloud.DocCollection.SNITCH; +import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT; +import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; +import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.*; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.params.CommonParams.VALUE_LONG; +import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR; +import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; +import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; +import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; +import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR; +import static org.apache.solr.common.params.ShardParams._ROUTE_; +import static org.apache.solr.common.util.StrUtils.formatString; public class CollectionsHandler extends RequestHandlerBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -348,11 +350,7 @@ Map call(SolrQueryRequest req, SolrQueryResponse rsp, Collection addMapObject(props, RULE); addMapObject(props, SNITCH); verifyRuleParams(h.coreContainer, props); - final String collectionName = (String) props.get(NAME); - if (!SolrIdentifierValidator.validateCollectionName(collectionName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, - SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.COLLECTION, collectionName)); - } + final String collectionName = SolrIdentifierValidator.validateCollectionName((String)props.get(NAME)); final String shardsParam = (String) props.get(SHARDS_PROP); if (StringUtils.isNotEmpty(shardsParam)) { verifyShardsParam(shardsParam); @@ -433,10 +431,7 @@ Map call(SolrQueryRequest req, SolrQueryResponse rsp, Collection @Override Map call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler handler) throws Exception { - final String aliasName = req.getParams().get(NAME); - if (!SolrIdentifierValidator.validateCollectionName(aliasName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.ALIAS, aliasName)); - } + final String aliasName = SolrIdentifierValidator.validateAliasName(req.getParams().get(NAME)); return req.getParams().required().getAll(null, NAME, "collections"); } }, @@ -505,11 +500,7 @@ Map call(SolrQueryRequest req, SolrQueryResponse rsp, Collection COLLECTION_PROP, SHARD_ID_PROP); ClusterState clusterState = handler.coreContainer.getZkController().getClusterState(); - final String newShardName = req.getParams().get(SHARD_ID_PROP); - if (!SolrIdentifierValidator.validateShardName(newShardName)) { - throw new SolrException(ErrorCode.BAD_REQUEST, SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.SHARD, - newShardName)); - } + final String newShardName = SolrIdentifierValidator.validateShardName(req.getParams().get(SHARD_ID_PROP)); if (!ImplicitDocRouter.NAME.equals(((Map) clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).get(DOC_ROUTER)).get(NAME))) throw new SolrException(ErrorCode.BAD_REQUEST, "shards can be added only to 'implicit' collections"); req.getParams().getAll(map, @@ -997,9 +988,7 @@ private static Map addMapObject(Map props, Strin private static void verifyShardsParam(String shardsParam) { for (String shard : shardsParam.split(",")) { - if (!SolrIdentifierValidator.validateShardName(shard)) - throw new SolrException(ErrorCode.BAD_REQUEST, SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.SHARD, - shard)); + SolrIdentifierValidator.validateShardName(shard); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteStatusTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteStatusTest.java index 16ca35a0865b..3b8e014c0a74 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteStatusTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteStatusTest.java @@ -17,101 +17,129 @@ package org.apache.solr.cloud; import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; +import org.junit.BeforeClass; import org.junit.Test; -public class DeleteStatusTest extends AbstractFullDistribZkTestBase { +public class DeleteStatusTest extends SolrCloudTestCase { + + public static final int MAX_WAIT_TIMEOUT = 30; + + @BeforeClass + public static void createCluster() throws Exception { + configureCluster(2) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + } + + // Basically equivalent to RequestStatus.waitFor(), but doesn't delete the id from the queue + private static RequestStatusState waitForRequestState(String id, SolrClient client, int timeout) + throws IOException, SolrServerException, InterruptedException { + RequestStatusState state = RequestStatusState.SUBMITTED; + long endTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(MAX_WAIT_TIMEOUT); + while (System.nanoTime() < endTime) { + state = CollectionAdminRequest.requestStatus(id).process(client).getRequestStatus(); + if (state == RequestStatusState.COMPLETED) + break; + assumeTrue("Error creating collection - skipping test", state != RequestStatusState.FAILED); + TimeUnit.SECONDS.sleep(1); + } + assumeTrue("Timed out creating collection - skipping test", state == RequestStatusState.COMPLETED); + return state; + } @Test - public void testDeleteStatus() throws IOException, SolrServerException { - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setCollectionName("requeststatus") - .setConfigName("conf1") - .setReplicationFactor(1) - .setNumShards(1) - .setAsyncId("collectioncreate") - .process(cloudClient); - - RequestStatusState state = getRequestStateAfterCompletion("collectioncreate", 30, cloudClient); - assertSame(RequestStatusState.COMPLETED, state); + public void testAsyncIdsMayBeDeleted() throws Exception { + + final CloudSolrClient client = cluster.getSolrClient(); + + final String collection = "deletestatus"; + final String asyncId = CollectionAdminRequest.createCollection(collection, "conf1", 1, 1).processAsync(client); + + waitForRequestState(asyncId, client, MAX_WAIT_TIMEOUT); - // Let's delete the stored response now - CollectionAdminRequest.DeleteStatus deleteStatus = new CollectionAdminRequest.DeleteStatus(); - CollectionAdminResponse rsp = deleteStatus - .setRequestId("collectioncreate") - .process(cloudClient); - assertEquals("successfully removed stored response for [collectioncreate]", rsp.getResponse().get("status")); - - // Make sure that the response was deleted from zk - state = getRequestState("collectioncreate", cloudClient); - assertSame(RequestStatusState.NOT_FOUND, state); - - // Try deleting the same requestid again - deleteStatus = new CollectionAdminRequest.DeleteStatus(); - rsp = deleteStatus - .setRequestId("collectioncreate") - .process(cloudClient); - assertEquals("[collectioncreate] not found in stored responses", rsp.getResponse().get("status")); - - // Let's try deleting a non-existent status - deleteStatus = new CollectionAdminRequest.DeleteStatus(); - rsp = deleteStatus - .setRequestId("foo") - .process(cloudClient); + assertEquals(RequestStatusState.COMPLETED, + CollectionAdminRequest.requestStatus(asyncId).process(client).getRequestStatus()); + + CollectionAdminResponse rsp = CollectionAdminRequest.deleteAsyncId(asyncId).process(client); + assertEquals("successfully removed stored response for [" + asyncId + "]", rsp.getResponse().get("status")); + + assertEquals(RequestStatusState.NOT_FOUND, + CollectionAdminRequest.requestStatus(asyncId).process(client).getRequestStatus()); + + } + + @Test + public void testDeletingNonExistentRequests() throws Exception { + + final CloudSolrClient client = cluster.getSolrClient(); + + CollectionAdminResponse rsp = CollectionAdminRequest.deleteAsyncId("foo").process(client); assertEquals("[foo] not found in stored responses", rsp.getResponse().get("status")); + + } + + @Test + public void testProcessAndWaitDeletesAsyncIds() throws IOException, SolrServerException, InterruptedException { + + final CloudSolrClient client = cluster.getSolrClient(); + + RequestStatusState state = CollectionAdminRequest.createCollection("requeststatus", "conf1", 1, 1) + .processAndWait("request1", client, MAX_WAIT_TIMEOUT); + assertSame(RequestStatusState.COMPLETED, state); + + // using processAndWait deletes the requestid + state = CollectionAdminRequest.requestStatus("request1").process(client).getRequestStatus(); + assertSame("Request id was not deleted by processAndWait call", RequestStatusState.NOT_FOUND, state); + } @Test public void testDeleteStatusFlush() throws Exception { - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setConfigName("conf1") - .setCollectionName("foo") - .setAsyncId("foo") - .setNumShards(1) - .setReplicationFactor(1) - .process(cloudClient); - - create = new CollectionAdminRequest.Create(); - create.setConfigName("conf1") - .setCollectionName("bar") - .setAsyncId("bar") - .setNumShards(1) - .setReplicationFactor(1) - .process(cloudClient); - - RequestStatusState state = getRequestStateAfterCompletion("foo", 30, cloudClient); - assertEquals(RequestStatusState.COMPLETED, state); - - state = getRequestStateAfterCompletion("bar", 30, cloudClient); - assertEquals(RequestStatusState.COMPLETED, state); - - CollectionAdminRequest.DeleteStatus deleteStatus = new CollectionAdminRequest.DeleteStatus(); - deleteStatus.setFlush(true) - .process(cloudClient); - - assertEquals(RequestStatusState.NOT_FOUND, getRequestState("foo", cloudClient)); - assertEquals(RequestStatusState.NOT_FOUND, getRequestState("bar", cloudClient)); - - deleteStatus = new CollectionAdminRequest.DeleteStatus(); + + final CloudSolrClient client = cluster.getSolrClient(); + + String id1 = CollectionAdminRequest.createCollection("flush1", "conf1", 1, 1).processAsync(client); + String id2 = CollectionAdminRequest.createCollection("flush2", "conf1", 1, 1).processAsync(client); + + assertEquals(RequestStatusState.COMPLETED, waitForRequestState(id1, client, MAX_WAIT_TIMEOUT)); + assertEquals(RequestStatusState.COMPLETED, waitForRequestState(id2, client, MAX_WAIT_TIMEOUT)); + + CollectionAdminRequest.deleteAllAsyncIds().process(client); + + assertEquals(RequestStatusState.NOT_FOUND, + CollectionAdminRequest.requestStatus(id1).process(client).getRequestStatus()); + assertEquals(RequestStatusState.NOT_FOUND, + CollectionAdminRequest.requestStatus(id2).process(client).getRequestStatus()); + + } + + @Test + @SuppressWarnings("deprecation") + public void testDeprecatedConstructorValidation() throws Exception { + + final CloudSolrClient client = cluster.getSolrClient(); + try { - deleteStatus.process(cloudClient); + new CollectionAdminRequest.DeleteStatus().process(client); fail("delete status should have failed"); - } catch (HttpSolrClient.RemoteSolrException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Either requestid or flush parameter must be specified.")); } - deleteStatus = new CollectionAdminRequest.DeleteStatus(); try { - deleteStatus.setFlush(true) + new CollectionAdminRequest.DeleteStatus().setFlush(true) .setRequestId("foo") - .process(cloudClient); + .process(client); fail("delete status should have failed"); - } catch (HttpSolrClient.RemoteSolrException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Both requestid and flush parameters can not be specified together.")); } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index c9c8c3989dfb..4f28408ce210 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -18,8 +18,6 @@ import java.io.IOException; import java.util.Collection; -import java.util.Iterator; -import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -80,16 +78,15 @@ public Collection getContentStreams() throws IOException { } protected void addProperties(ModifiableSolrParams params, Properties props) { - Iterator> iter = props.entrySet().iterator(); - while(iter.hasNext()) { - Map.Entry prop = iter.next(); - String key = (String) prop.getKey(); - String value = (String) prop.getValue(); - params.set(PROPERTY_PREFIX + key, value); + for (String propertyName : props.stringPropertyNames()) { + params.set(PROPERTY_PREFIX + propertyName, props.getProperty(propertyName)); } } - protected abstract static class AsyncCollectionAdminRequest extends CollectionAdminRequest { + /** + * Base class for asynchronous collection admin requests + */ + public abstract static class AsyncCollectionAdminRequest extends CollectionAdminRequest { public AsyncCollectionAdminRequest(CollectionAction action) { super(action); @@ -164,7 +161,7 @@ public RequestStatusState processAndWait(SolrClient client, long timeoutSeconds) public RequestStatusState processAndWait(String asyncId, SolrClient client, long timeoutSeconds) throws IOException, SolrServerException, InterruptedException { processAsync(asyncId, client); - return new RequestStatus().setRequestId(asyncId).waitFor(client, timeoutSeconds); + return requestStatus(asyncId).waitFor(client, timeoutSeconds); } @Override @@ -181,10 +178,12 @@ protected abstract static class AsyncCollectionSpecificAdminRequest extends Asyn protected String collection; - public AsyncCollectionSpecificAdminRequest(CollectionAction action) { + public AsyncCollectionSpecificAdminRequest(CollectionAction action, String collection) { super(action); + this.collection = collection; } + @Deprecated public abstract AsyncCollectionSpecificAdminRequest setCollectionName(String collection); @Override @@ -202,12 +201,14 @@ protected abstract static class AsyncShardSpecificAdminRequest extends AsyncColl protected String collection; protected String shard; - public AsyncShardSpecificAdminRequest(CollectionAction action) { + public AsyncShardSpecificAdminRequest(CollectionAction action, String collection, String shard) { super(action); } + @Deprecated public abstract AsyncShardSpecificAdminRequest setCollectionName(String collection); + @Deprecated public abstract AsyncShardSpecificAdminRequest setShardName(String shard); @Override @@ -228,12 +229,14 @@ protected abstract static class ShardSpecificAdminRequest extends CollectionAdmi protected String collection; protected String shard; - public ShardSpecificAdminRequest(CollectionAction action) { + public ShardSpecificAdminRequest(CollectionAction action, String collection, String shard) { super(action); } + @Deprecated public abstract ShardSpecificAdminRequest setCollectionName(String collection); + @Deprecated public abstract ShardSpecificAdminRequest setShardName(String shard); @Override @@ -264,7 +267,7 @@ protected abstract static class CollectionAdminRoleRequest extends AsyncCollecti protected String node; protected String role; - public CollectionAdminRoleRequest(CollectionAction action) { + public CollectionAdminRoleRequest(CollectionAction action, String node, String role) { super(action); } @@ -274,12 +277,14 @@ public CollectionAdminRoleRequest setAsyncId(String id) { return this; } + @Deprecated public abstract CollectionAdminRoleRequest setNode(String node); public String getNode() { return this.node; } + @Deprecated public abstract CollectionAdminRoleRequest setRole(String role); public String getRole() { @@ -298,6 +303,17 @@ public SolrParams getParams() { /** Specific Collection API call implementations **/ + /** + * Returns a SolrRequest for creating a collection + * @param collection the collection name + * @param config the collection config + * @param numShards the number of shards in the collection + * @param numReplicas the replication factor of the collection + */ + public static Create createCollection(String collection, String config, int numShards, int numReplicas) { + return new Create(collection, config, numShards, numReplicas); + } + // CREATE request public static class Create extends AsyncCollectionSpecificAdminRequest { @@ -315,17 +331,31 @@ public static class Create extends AsyncCollectionSpecificAdminRequest { protected Integer stateFormat; private String[] rule , snitch; + /** + * @deprecated Use {@link #createCollection(String, String, int, int)} + */ + @Deprecated public Create() { - super(CollectionAction.CREATE); + super(CollectionAction.CREATE, null); } + private Create(String collection, String config, int numShards, int numReplicas) { + super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection)); + this.configName = config; + this.numShards = numShards; + this.replicationFactor = numReplicas; + } + + @Deprecated public Create setConfigName(String config) { this.configName = config; return this; } public Create setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; return this; } public Create setRouterName(String routerName) { this.routerName = routerName; return this; } public Create setRouterField(String routerField) { this.routerField = routerField; return this; } + @Deprecated public Create setNumShards(Integer numShards) {this.numShards = numShards; return this; } public Create setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; return this; } public Create setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; return this; } + @Deprecated public Create setReplicationFactor(Integer repl) { this.replicationFactor = repl; return this; } public Create setStateFormat(Integer stateFormat) { this.stateFormat = stateFormat; return this; } public Create setRule(String... s){ this.rule = s; return this; } @@ -350,10 +380,7 @@ public Create() { */ public Create setShards(String shards) { for (String shard : shards.split(",")) { - if (!SolrIdentifierValidator.validateShardName(shard)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.SHARD, - shard)); - } + SolrIdentifierValidator.validateShardName(shard); } this.shards = shards; return this; @@ -366,16 +393,14 @@ public Create setShards(String shards) { * * @throws IllegalArgumentException if the collection name contains invalid characters. */ + @Deprecated public Create setCollectionName(String collectionName) throws SolrException { - if (!SolrIdentifierValidator.validateCollectionName(collectionName)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.COLLECTION, - collectionName)); - } - this.collection = collectionName; + this.collection = SolrIdentifierValidator.validateCollectionName(collectionName); return this; } @Override + @Deprecated public Create setAsyncId(String id) { this.asyncId = id; return this; @@ -426,46 +451,87 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to reload a collection + */ + public static Reload reloadCollection(String collection) { + return new Reload(collection); + } + // RELOAD request public static class Reload extends AsyncCollectionSpecificAdminRequest { + /** + * @deprecated use {@link #reloadCollection(String)} + */ + @Deprecated public Reload() { - super(CollectionAction.RELOAD); + super(CollectionAction.RELOAD, null); + } + + private Reload(String collection) { + super(CollectionAction.RELOAD, collection); } @Override + @Deprecated public Reload setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public Reload setAsyncId(String id) { this.asyncId = id; return this; } } + /** + * Returns a SolrRequest to delete a collection + */ + public static Delete deleteCollection(String collection) { + return new Delete(collection); + } + // DELETE request public static class Delete extends AsyncCollectionSpecificAdminRequest { + /** + * @deprecated Use {@link #deleteCollection(String)} + */ + @Deprecated public Delete() { - super(CollectionAction.DELETE); + super(CollectionAction.DELETE, null); + } + + private Delete(String collection) { + super(CollectionAction.DELETE, collection); } @Override + @Deprecated public Delete setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public Delete setAsyncId(String id) { this.asyncId = id; return this; } } + /** + * Returns a SolrRequest to create a new shard in a collection + */ + public static CreateShard createShard(String collection, String shard) { + return new CreateShard(collection, shard); + } + // CREATESHARD request public static class CreateShard extends AsyncShardSpecificAdminRequest { @@ -490,11 +556,20 @@ public CreateShard setProperties(Properties properties) { return this; } + /** + * @deprecated use {@link #createShard(String, String)} + */ + @Deprecated public CreateShard() { - super(CollectionAction.CREATESHARD); + super(CollectionAction.CREATESHARD, null, null); + } + + private CreateShard(String collection, String shard) { + super(CollectionAction.CREATESHARD, collection, SolrIdentifierValidator.validateShardName(shard)); } @Override + @Deprecated public CreateShard setCollectionName(String collection) { this.collection = collection; return this; @@ -508,16 +583,14 @@ public CreateShard setCollectionName(String collection) { * @throws IllegalArgumentException if the shard name contains invalid characters. */ @Override + @Deprecated public CreateShard setShardName(String shardName) { - if (!SolrIdentifierValidator.validateShardName(shardName)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.SHARD, - shardName)); - } - this.shard = shardName; + this.shard = SolrIdentifierValidator.validateShardName(shardName); return this; } @Override + @Deprecated public CreateShard setAsyncId(String id) { this.asyncId = id; return this; @@ -538,6 +611,13 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to split a shard in a collection + */ + public static SplitShard splitShard(String collection, String shard) { + return new SplitShard(collection, shard); + } + // SPLITSHARD request public static class SplitShard extends AsyncShardSpecificAdminRequest { protected String ranges; @@ -545,8 +625,16 @@ public static class SplitShard extends AsyncShardSpecificAdminRequest { private Properties properties; + private SplitShard(String collection, String shard) { + super(CollectionAction.SPLITSHARD, collection, shard); + } + + /** + * @deprecated Use {@link #splitShard(String, String)} + */ + @Deprecated public SplitShard() { - super(CollectionAction.SPLITSHARD); + super(CollectionAction.SPLITSHARD, null, null); } public SplitShard setRanges(String ranges) { this.ranges = ranges; return this; } @@ -571,18 +659,21 @@ public SplitShard setProperties(Properties properties) { } @Override + @Deprecated public SplitShard setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public SplitShard setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public SplitShard setAsyncId(String id) { this.asyncId = id; return this; @@ -604,14 +695,29 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to delete a shard from a collection + */ + public static DeleteShard deleteShard(String collection, String shard) { + return new DeleteShard(collection, shard); + } + // DELETESHARD request public static class DeleteShard extends AsyncShardSpecificAdminRequest { private Boolean deleteInstanceDir; private Boolean deleteDataDir; + /** + * @deprecated Use {@link #deleteShard(String, String)} + */ + @Deprecated public DeleteShard() { - super(CollectionAction.DELETESHARD); + super(CollectionAction.DELETESHARD, null, null); + } + + private DeleteShard(String collection, String shard) { + super(CollectionAction.DELETESHARD, collection, shard); } public Boolean getDeleteInstanceDir() { @@ -633,18 +739,21 @@ public DeleteShard setDeleteDataDir(Boolean deleteDataDir) { } @Override + @Deprecated public DeleteShard setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public DeleteShard setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public DeleteShard setAsyncId(String id) { this.asyncId = id; return this; @@ -663,21 +772,41 @@ public SolrParams getParams() { } } + /** + * Returns a SolrRequest to force a leader election for a shard in a collection + * + * WARNING: This may cause data loss if the new leader does not contain updates + * acknowledged by the old leader. Use only if leadership elections are entirely + * broken. + */ + public static ForceLeader forceLeaderElection(String collection, String shard) { + return new ForceLeader(collection, shard); + } + // FORCELEADER request public static class ForceLeader extends ShardSpecificAdminRequest { + /** + * @deprecated Use {@link #forceLeaderElection(String, String)} + */ + @Deprecated public ForceLeader() { - super(CollectionAction.FORCELEADER); + super(CollectionAction.FORCELEADER, null, null); } + private ForceLeader(String collection, String shard) { + super(CollectionAction.FORCELEADER, collection, shard); + } @Override + @Deprecated public ForceLeader setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public ForceLeader setShardName(String shard) { this.shard = shard; return this; @@ -685,6 +814,9 @@ public ForceLeader setShardName(String shard) { } + /** + * A response object for {@link RequestStatus} requests + */ public static class RequestStatusResponse extends CollectionAdminResponse { public RequestStatusState getRequestStatus() { @@ -694,15 +826,34 @@ public RequestStatusState getRequestStatus() { } + /** + * Returns a SolrRequest for checking the status of an asynchronous request + * + * @see CollectionAdminRequest.AsyncCollectionAdminRequest + */ + public static RequestStatus requestStatus(String requestId) { + return new RequestStatus(requestId); + } + // REQUESTSTATUS request public static class RequestStatus extends CollectionAdminRequest { protected String requestId = null; + private RequestStatus(String requestId) { + super(CollectionAction.REQUESTSTATUS); + this.requestId = requestId; + } + + /** + * @deprecated Use {@link #requestStatus(String)} + */ + @Deprecated public RequestStatus() { super(CollectionAction.REQUESTSTATUS); } + @Deprecated public RequestStatus setRequestId(String requestId) { this.requestId = requestId; return this; @@ -726,6 +877,12 @@ protected RequestStatusResponse createResponse(SolrClient client) { return new RequestStatusResponse(); } + /** + * Wait until the asynchronous request is either completed or failed, up to a timeout + * @param client a SolrClient + * @param timeoutSeconds the maximum time to wait in seconds + * @return the last seen state of the request + */ public RequestStatusState waitFor(SolrClient client, long timeoutSeconds) throws IOException, SolrServerException, InterruptedException { long finishTime = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); @@ -733,7 +890,7 @@ public RequestStatusState waitFor(SolrClient client, long timeoutSeconds) while (System.nanoTime() < finishTime) { state = this.process(client).getRequestStatus(); if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) { - new DeleteStatus().setRequestId(requestId).process(client); + deleteAsyncId(requestId).process(client); return state; } TimeUnit.SECONDS.sleep(1); @@ -742,21 +899,43 @@ public RequestStatusState waitFor(SolrClient client, long timeoutSeconds) } } + /** + * Returns a SolrRequest to delete an asynchronous request status + */ + public static DeleteStatus deleteAsyncId(String requestId) { + return new DeleteStatus(requestId); + } + + public static DeleteStatus deleteAllAsyncIds() { + return new DeleteStatus().setFlush(true); + } + // DELETESTATUS request public static class DeleteStatus extends CollectionAdminRequest { protected String requestId = null; protected Boolean flush = null; + private DeleteStatus(String requestId) { + super(CollectionAction.DELETESTATUS); + this.requestId = requestId; + } + + /** + * @deprecated Use {@link #deleteAsyncId(String)} or {@link #deleteAllAsyncIds()} + */ + @Deprecated public DeleteStatus() { super(CollectionAction.DELETESTATUS); } + @Deprecated public DeleteStatus setRequestId(String requestId) { this.requestId = requestId; return this; } + @Deprecated public DeleteStatus setFlush(Boolean flush) { this.flush = flush; return this; @@ -773,9 +952,12 @@ public Boolean getFlush() { @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + if (requestId == null && flush == null) + throw new IllegalArgumentException("Either requestid or flush parameter must be specified."); + if (requestId != null && flush != null) + throw new IllegalArgumentException("Both requestid and flush parameters can not be specified together."); if (requestId != null) params.set(CoreAdminParams.REQUESTID, requestId); - if (flush != null) params.set(CollectionAdminParams.FLUSH, flush); return params; @@ -788,12 +970,31 @@ protected CollectionAdminResponse createResponse(SolrClient client) { } + /** + * Returns a SolrRequest to create a new alias + * @param aliasName the alias name + * @param aliasedCollections the collections to alias + */ + public static CreateAlias createAlias(String aliasName, String aliasedCollections) { + return new CreateAlias(aliasName, aliasedCollections); + } + // CREATEALIAS request public static class CreateAlias extends AsyncCollectionAdminRequest { protected String aliasName; protected String aliasedCollections; + private CreateAlias(String aliasName, String aliasedCollections) { + super(CollectionAction.CREATEALIAS); + this.aliasName = SolrIdentifierValidator.validateAliasName(aliasName); + this.aliasedCollections = aliasedCollections; + } + + /** + * @deprecated Use {@link #createAlias(String, String)} + */ + @Deprecated public CreateAlias() { super(CollectionAction.CREATEALIAS); } @@ -805,12 +1006,9 @@ public CreateAlias() { * * @throws IllegalArgumentException if the alias name contains invalid characters. */ + @Deprecated public CreateAlias setAliasName(String aliasName) { - if (!SolrIdentifierValidator.validateCollectionName(aliasName)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.ALIAS, - aliasName)); - } - this.aliasName = aliasName; + this.aliasName = SolrIdentifierValidator.validateAliasName(aliasName); return this; } @@ -818,6 +1016,7 @@ public String getAliasName() { return aliasName; } + @Deprecated public CreateAlias setAliasedCollections(String alias) { this.aliasedCollections = alias; return this; @@ -828,6 +1027,7 @@ public String getAliasedCollections() { } @Override + @Deprecated public CreateAlias setAsyncId(String id) { this.asyncId = id; return this; @@ -843,21 +1043,39 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to delete an alias + */ + public static DeleteAlias deleteAlias(String aliasName) { + return new DeleteAlias(aliasName); + } + // DELETEALIAS request public static class DeleteAlias extends AsyncCollectionAdminRequest { protected String aliasName; + private DeleteAlias(String aliasName) { + super(CollectionAction.DELETEALIAS); + this.aliasName = aliasName; + } + + /** + * @deprecated Use {@link #deleteAlias(String)} + */ + @Deprecated public DeleteAlias() { super(CollectionAction.DELETEALIAS); } + @Deprecated public DeleteAlias setAliasName(String aliasName) { this.aliasName = aliasName; return this; } @Override + @Deprecated public DeleteAlias setAsyncId(String id) { this.asyncId = id; return this; @@ -873,6 +1091,20 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to add a replica to a shard in a collection + */ + public static AddReplica addReplicaToShard(String collection, String shard) { + return new AddReplica(collection, shard, null); + } + + /** + * Returns a SolrRequest to add a replica to a collection using a route key + */ + public static AddReplica addReplicaByRouteKey(String collection, String routeKey) { + return new AddReplica(collection, null, routeKey); + } + // ADDREPLICA request public static class AddReplica extends AsyncCollectionAdminRequest { @@ -884,10 +1116,21 @@ public static class AddReplica extends AsyncCollectionAdminRequest { protected String dataDir; protected Properties properties; + /** + * @deprecated Use {@link #addReplicaByRouteKey(String, String)} or {@link #addReplicaToShard(String, String)} + */ + @Deprecated public AddReplica() { super(CollectionAction.ADDREPLICA); } + private AddReplica(String collection, String shard, String routeKey) { + super(CollectionAction.ADDREPLICA); + this.collection = collection; + this.shard = shard; + this.routeKey = routeKey; + } + public Properties getProperties() { return properties; } @@ -910,6 +1153,7 @@ public String getRouteKey() { return routeKey; } + @Deprecated public AddReplica setRouteKey(String routeKey) { this.routeKey = routeKey; return this; @@ -933,17 +1177,20 @@ public AddReplica setDataDir(String dataDir) { return this; } + @Deprecated public AddReplica setCollectionName(String collection) { this.collection = collection; return this; } + @Deprecated public AddReplica setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public AddReplica setAsyncId(String id) { this.asyncId = id; return this; @@ -957,7 +1204,7 @@ public SolrParams getParams() { params.add(CoreAdminParams.COLLECTION, collection); if (shard == null || shard.isEmpty()) { if (routeKey == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Either shard or routeKey must be provided"); + throw new IllegalArgumentException("Either shard or routeKey must be provided"); } params.add(ShardParams._ROUTE_, routeKey); } @@ -979,7 +1226,13 @@ public SolrParams getParams() { return params; } + } + /** + * Returns a SolrRequest to delete a replica from a shard in a collection + */ + public static DeleteReplica deleteReplica(String collection, String shard, String replica) { + return new DeleteReplica(collection, shard, replica); } // DELETEREPLICA request @@ -991,10 +1244,20 @@ public static class DeleteReplica extends AsyncShardSpecificAdminRequest { private Boolean deleteInstanceDir; private Boolean deleteIndexDir; + /** + * @deprecated Use {@link #deleteReplica(String, String, String)} + */ + @Deprecated public DeleteReplica() { - super(CollectionAction.DELETEREPLICA); + super(CollectionAction.DELETEREPLICA, null, null); + } + + private DeleteReplica(String collection, String shard, String replica) { + super(CollectionAction.DELETEREPLICA, collection, shard); + this.replica = replica; } + @Deprecated public DeleteReplica setReplica(String replica) { this.replica = replica; return this; @@ -1014,18 +1277,21 @@ public Boolean getOnlyIfDown() { } @Override + @Deprecated public DeleteReplica setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public DeleteReplica setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public DeleteReplica setAsyncId(String id) { this.asyncId = id; return this; @@ -1070,16 +1336,34 @@ public DeleteReplica setDeleteInstanceDir(Boolean deleteInstanceDir) { } } + /** + * Returns a SolrRequest to set a cluster property + */ + public static ClusterProp setClusterProperty(String propertyName, String propertyValue) { + return new ClusterProp(propertyName, propertyValue); + } + // CLUSTERPROP request public static class ClusterProp extends CollectionAdminRequest { private String propertyName; private String propertyValue; + /** + * @deprecated Use {@link #setClusterProperty(String, String)} + */ + @Deprecated public ClusterProp() { super(CollectionAction.CLUSTERPROP); } + private ClusterProp(String propertyName, String propertyValue) { + super(CollectionAction.CLUSTERPROP); + this.propertyName = propertyName; + this.propertyValue = propertyValue; + } + + @Deprecated public ClusterProp setPropertyName(String propertyName) { this.propertyName = propertyName; return this; @@ -1089,6 +1373,7 @@ public String getPropertyName() { return this.propertyName; } + @Deprecated public ClusterProp setPropertyValue(String propertyValue) { this.propertyValue = propertyValue; return this; @@ -1115,6 +1400,13 @@ protected CollectionAdminResponse createResponse(SolrClient client) { } + /** + * Returns a SolrRequest to migrate data matching a split key to another collection + */ + public static Migrate migrateData(String collection, String targetCollection, String splitKey) { + return new Migrate(collection, targetCollection, splitKey); + } + // MIGRATE request public static class Migrate extends AsyncCollectionAdminRequest { @@ -1124,10 +1416,22 @@ public static class Migrate extends AsyncCollectionAdminRequest { private Integer forwardTimeout; private Properties properties; + /** + * @deprecated Use {@link #migrateData(String, String, String)} + */ + @Deprecated public Migrate() { super(CollectionAction.MIGRATE); } + private Migrate(String collection, String targetCollection, String splitKey) { + super(CollectionAction.MIGRATE); + this.collection = collection; + this.targetCollection = targetCollection; + this.splitKey = splitKey; + } + + @Deprecated public Migrate setCollectionName(String collection) { this.collection = collection; return this; @@ -1137,6 +1441,7 @@ public String getCollectionName() { return collection; } + @Deprecated public Migrate setTargetCollection(String targetCollection) { this.targetCollection = targetCollection; return this; @@ -1146,6 +1451,7 @@ public String getTargetCollection() { return this.targetCollection; } + @Deprecated public Migrate setSplitKey(String splitKey) { this.splitKey = splitKey; return this; @@ -1174,6 +1480,7 @@ public Properties getProperties() { } @Override + @Deprecated public Migrate setAsyncId(String id) { this.asyncId = id; return this; @@ -1198,46 +1505,87 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to add a role to a node + */ + public static AddRole addRole(String node, String role) { + return new AddRole(node, role); + } + // ADDROLE request public static class AddRole extends CollectionAdminRoleRequest { + /** + * @deprecated Use {@link #addRole(String, String)} + */ + @Deprecated public AddRole() { - super(CollectionAction.ADDROLE); + super(CollectionAction.ADDROLE, null, null); + } + + private AddRole(String node, String role) { + super(CollectionAction.ADDROLE, node, role); } @Override + @Deprecated public AddRole setNode(String node) { this.node = node; return this; } @Override + @Deprecated public AddRole setRole(String role) { this.role = role; return this; } } + /** + * Returns a SolrRequest to remove a role from a node + */ + public static RemoveRole removeRole(String node, String role) { + return new RemoveRole(node, role); + } + // REMOVEROLE request public static class RemoveRole extends CollectionAdminRoleRequest { + /** + * @deprecated Use {@link #removeRole(String, String)} + */ + @Deprecated public RemoveRole() { - super(CollectionAction.REMOVEROLE); + super(CollectionAction.REMOVEROLE, null, null); + } + + private RemoveRole(String node, String role) { + super(CollectionAction.REMOVEROLE, node, role); } @Override + @Deprecated public RemoveRole setNode(String node) { this.node = node; return this; } @Override + @Deprecated public RemoveRole setRole(String role) { this.role = role; return this; } } + /** + * Return a SolrRequest to get the Overseer status + */ + public static OverseerStatus getOverseerStatus() { + return new OverseerStatus(); + } + // OVERSEERSTATUS request public static class OverseerStatus extends AsyncCollectionAdminRequest { @@ -1246,12 +1594,20 @@ public OverseerStatus () { } @Override + @Deprecated public OverseerStatus setAsyncId(String id) { this.asyncId = id; return this; } } + /** + * Return a SolrRequest to get the Cluster status + */ + public static ClusterStatus getClusterStatus() { + return new ClusterStatus(); + } + // CLUSTERSTATUS request public static class ClusterStatus extends CollectionAdminRequest { @@ -1310,7 +1666,13 @@ protected CollectionAdminResponse createResponse(SolrClient client) { return new CollectionAdminResponse(); } + } + /** + * Returns a SolrRequest to get a list of collections in the cluster + */ + public static List listCollections() { + return new List(); } // LIST request @@ -1325,6 +1687,14 @@ protected CollectionAdminResponse createResponse(SolrClient client) { } } + /** + * Returns a SolrRequest to add a property to a specific replica + */ + public static AddReplicaProp addReplicaProperty(String collection, String shard, String replica, + String propertyName, String propertyValue) { + return new AddReplicaProp(collection, shard, replica, propertyName, propertyValue); + } + // ADDREPLICAPROP request public static class AddReplicaProp extends AsyncShardSpecificAdminRequest { @@ -1333,14 +1703,26 @@ public static class AddReplicaProp extends AsyncShardSpecificAdminRequest { private String propertyValue; private Boolean shardUnique; + /** + * @deprecated Use {@link #addReplicaProperty(String, String, String, String, String)} + */ + @Deprecated public AddReplicaProp() { - super(CollectionAction.ADDREPLICAPROP); + super(CollectionAction.ADDREPLICAPROP, null, null); + } + + private AddReplicaProp(String collection, String shard, String replica, String propertyName, String propertyValue) { + super(CollectionAction.ADDREPLICAPROP, collection, shard); + this.replica = replica; + this.propertyName = propertyName; + this.propertyValue = propertyValue; } public String getReplica() { return replica; } + @Deprecated public AddReplicaProp setReplica(String replica) { this.replica = replica; return this; @@ -1350,6 +1732,7 @@ public String getPropertyName() { return propertyName; } + @Deprecated public AddReplicaProp setPropertyName(String propertyName) { this.propertyName = propertyName; return this; @@ -1359,6 +1742,7 @@ public String getPropertyValue() { return propertyValue; } + @Deprecated public AddReplicaProp setPropertyValue(String propertyValue) { this.propertyValue = propertyValue; return this; @@ -1374,18 +1758,21 @@ public AddReplicaProp setShardUnique(Boolean shardUnique) { } @Override + @Deprecated public AddReplicaProp setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public AddReplicaProp setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public AddReplicaProp setAsyncId(String id) { this.asyncId = id; return this; @@ -1407,20 +1794,39 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to delete a property from a specific replica + */ + public static DeleteReplicaProp deleteReplicaProperty(String collection, String shard, + String replica, String propertyName) { + return new DeleteReplicaProp(collection, shard, replica, propertyName); + } + // DELETEREPLICAPROP request public static class DeleteReplicaProp extends AsyncShardSpecificAdminRequest { private String replica; private String propertyName; + /** + * @deprecated Use {@link #deleteReplicaProperty(String, String, String, String)} + */ + @Deprecated public DeleteReplicaProp() { - super(CollectionAction.DELETEREPLICAPROP); + super(CollectionAction.DELETEREPLICAPROP, null, null); + } + + private DeleteReplicaProp(String collection, String shard, String replica, String propertyName) { + super(CollectionAction.DELETEREPLICAPROP, collection, shard); + this.replica = replica; + this.propertyName = propertyName; } public String getReplica() { return replica; } + @Deprecated public DeleteReplicaProp setReplica(String replica) { this.replica = replica; return this; @@ -1430,24 +1836,28 @@ public String getPropertyName() { return propertyName; } + @Deprecated public DeleteReplicaProp setPropertyName(String propertyName) { this.propertyName = propertyName; return this; } @Override + @Deprecated public DeleteReplicaProp setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public DeleteReplicaProp setShardName(String shard) { this.shard = shard; return this; } @Override + @Deprecated public DeleteReplicaProp setAsyncId(String id) { this.asyncId = id; return this; @@ -1464,21 +1874,41 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to migrate a collection state format + * + * This is an expert-level request, and should not generally be necessary. + */ + public static MigrateClusterState migrateCollectionFormat(String collection) { + return new MigrateClusterState(collection); + } + // MIGRATECLUSTERSTATE request public static class MigrateClusterState extends AsyncCollectionAdminRequest { protected String collection; + private MigrateClusterState(String collection) { + super(CollectionAction.MIGRATESTATEFORMAT); + this.collection = collection; + } + + /** + * @deprecated Use {@link #migrateCollectionFormat(String)} + */ + @Deprecated public MigrateClusterState() { super(CollectionAction.MIGRATESTATEFORMAT); } + @Deprecated public MigrateClusterState setCollectionName(String collection) { this.collection = collection; return this; } @Override + @Deprecated public MigrateClusterState setAsyncId(String id) { this.asyncId = id; return this; @@ -1494,6 +1924,13 @@ public SolrParams getParams() { } } + /** + * Returns a SolrRequest to balance a replica property across the shards of a collection + */ + public static BalanceShardUnique balanceReplicaProperty(String collection, String propertyName) { + return new BalanceShardUnique(collection, propertyName); + } + // BALANCESHARDUNIQUE request public static class BalanceShardUnique extends AsyncCollectionAdminRequest { @@ -1502,6 +1939,16 @@ public static class BalanceShardUnique extends AsyncCollectionAdminRequest { protected Boolean onlyActiveNodes; protected Boolean shardUnique; + private BalanceShardUnique(String collection, String propertyName) { + super(CollectionAction.BALANCESHARDUNIQUE); + this.collection = collection; + this.propertyName = propertyName; + } + + /** + * @deprecated Use {@link #balanceReplicaProperty(String, String)} + */ + @Deprecated public BalanceShardUnique() { super(CollectionAction.BALANCESHARDUNIQUE); } @@ -1510,6 +1957,7 @@ public String getPropertyName() { return propertyName; } + @Deprecated public BalanceShardUnique setPropertyName(String propertyName) { this.propertyName = propertyName; return this; @@ -1533,6 +1981,7 @@ public BalanceShardUnique setShardUnique(Boolean shardUnique) { return this; } + @Deprecated public BalanceShardUnique setCollection(String collection) { this.collection = collection; return this; @@ -1543,6 +1992,7 @@ public String getCollection() { } @Override + @Deprecated public BalanceShardUnique setAsyncId(String id) { this.asyncId = id; return this; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java index ab563ed87ed5..7d9e356e73fa 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java @@ -16,6 +16,11 @@ */ package org.apache.solr.client.solrj.request; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; @@ -29,11 +34,6 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - /** * This class is experimental and subject to change. * @@ -110,11 +110,7 @@ public String getConfigSet() { */ @Override public void setCoreName(String coreName) { - if (!SolrIdentifierValidator.validateCoreName(coreName)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.CORE, - coreName)); - } - this.core = coreName; + this.core = SolrIdentifierValidator.validateCoreName(coreName); } @Override @@ -559,14 +555,9 @@ public static CoreAdminResponse unloadCore(String name, boolean deleteIndex, boo */ public static CoreAdminResponse renameCore(String coreName, String newName, SolrClient client ) throws SolrServerException, IOException { - if (!SolrIdentifierValidator.validateCoreName(newName)) { - throw new IllegalArgumentException(SolrIdentifierValidator.getIdentifierMessage(SolrIdentifierValidator.IdentifierType.CORE, - newName)); - } - CoreAdminRequest req = new CoreAdminRequest(); req.setCoreName(coreName); - req.setOtherCoreName(newName); + req.setOtherCoreName(SolrIdentifierValidator.validateCoreName(newName)); req.setAction( CoreAdminAction.RENAME ); return req.process( client ); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java index 449c62127b90..57f9909f0514 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java @@ -32,18 +32,28 @@ public enum IdentifierType { SHARD, COLLECTION, CORE, ALIAS } - public static boolean validateShardName(String shardName) { - return validateIdentifier(shardName); + public static String validateName(IdentifierType type, String name) { + if (!validateIdentifier(name)) + throw new IllegalArgumentException(getIdentifierMessage(type, name)); + return name; } - - public static boolean validateCollectionName(String collectionName) { - return validateIdentifier(collectionName); + + public static String validateShardName(String shardName) { + return validateName(IdentifierType.SHARD, shardName); } - public static boolean validateCoreName(String name) { - return validateIdentifier(name); + public static String validateCollectionName(String collectionName) { + return validateName(IdentifierType.COLLECTION, collectionName); } - + + public static String validateAliasName(String alias) { + return validateName(IdentifierType.ALIAS, alias); + } + + public static String validateCoreName(String coreName) { + return validateName(IdentifierType.CORE, coreName); + } + private static boolean validateIdentifier(String identifier) { if (identifier == null || ! identifierPattern.matcher(identifier).matches()) { return false; From 8ff5b57cbde0aa80e4c2b53afe54ef0b38ffd879 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 09:28:50 -0500 Subject: [PATCH 0077/1113] add 2B point test --- .../org/apache/lucene/index/Test2BPoints.java | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java new file mode 100644 index 000000000000..bfe387e8ca9a --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.LuceneTestCase.Monster; +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.TimeUnits; + +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + +// e.g. run like this: ant test -Dtestcase=Test2BPoints -Dtests.nightly=true -Dtests.verbose=true -Dtests.monster=true +// +// or: python -u /l/util/src/python/repeatLuceneTest.py -once -nolog -tmpDir /b/tmp -logDir /l/logs Test2BPoints.test1D -verbose + +@SuppressCodecs({ "SimpleText", "Memory", "Direct", "Compressing" }) +@TimeoutSuite(millis = 16 * TimeUnits.HOUR) +@Monster("takes at least 4 hours and consumes many GB of temp disk space") +public class Test2BPoints extends LuceneTestCase { + public void test1D() throws Exception { + Directory dir = FSDirectory.open(createTempDir("2BPoints1D")); + System.out.println("DIR: " + ((FSDirectory) dir).getDirectory()); + + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())) + .setCodec(Codec.forName("Lucene60")) + .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH) + .setRAMBufferSizeMB(64.0) + .setMergeScheduler(new ConcurrentMergeScheduler()) + .setMergePolicy(newLogMergePolicy(false, 10)) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + + IndexWriter w = new IndexWriter(dir, iwc); + + MergePolicy mp = w.getConfig().getMergePolicy(); + if (mp instanceof LogByteSizeMergePolicy) { + // 1 petabyte: + ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024); + } + + final int numDocs = (Integer.MAX_VALUE / 26) + 1; + long counter = 0; + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + for (int j=0;j<26;j++) { + doc.add(new LongPoint("long", counter)); + counter++; + } + w.addDocument(doc); + if (VERBOSE && i % 100000 == 0) { + System.out.println(i + " of " + numDocs + "..."); + } + } + w.forceMerge(1); + DirectoryReader r = DirectoryReader.open(w); + IndexSearcher s = new IndexSearcher(r); + assertEquals(1250, s.count(LongPoint.newRangeQuery("long", 33640828, 33673327))); + assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE); + r.close(); + w.close(); + System.out.println("TEST: now CheckIndex"); + TestUtil.checkIndex(dir); + dir.close(); + } + + public void test2D() throws Exception { + Directory dir = FSDirectory.open(createTempDir("2BPoints2D")); + + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())) + .setCodec(Codec.forName("Lucene60")) + .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH) + .setRAMBufferSizeMB(64.0) + .setMergeScheduler(new ConcurrentMergeScheduler()) + .setMergePolicy(newLogMergePolicy(false, 10)) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + + IndexWriter w = new IndexWriter(dir, iwc); + + MergePolicy mp = w.getConfig().getMergePolicy(); + if (mp instanceof LogByteSizeMergePolicy) { + // 1 petabyte: + ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024); + } + + final int numDocs = (Integer.MAX_VALUE / 26) + 1; + long counter = 0; + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + for (int j=0;j<26;j++) { + doc.add(new LongPoint("long", counter, 2*counter+1)); + counter++; + } + w.addDocument(doc); + if (VERBOSE && i % 100000 == 0) { + System.out.println(i + " of " + numDocs + "..."); + } + } + w.forceMerge(1); + DirectoryReader r = DirectoryReader.open(w); + IndexSearcher s = new IndexSearcher(r); + assertEquals(1250, s.count(LongPoint.newRangeQuery("long", new long[] {33640828, 33673327}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE}))); + assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE); + r.close(); + w.close(); + System.out.println("TEST: now CheckIndex"); + TestUtil.checkIndex(dir); + dir.close(); + } +} From 9f8fe1239afb7089b9f85432d076bdd778d3cd50 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 10:07:15 -0500 Subject: [PATCH 0078/1113] LUCENE-7085: PointRangeQuery.equals sometimes returns false even if queries were in fact equal --- .../apache/lucene/search/PointInSetQuery.java | 10 +-- .../apache/lucene/search/PointRangeQuery.java | 38 ++++++--- .../lucene/search/TestPointQueries.java | 80 +++++++++++++++++++ .../lucene/document/InetAddressPoint.java | 23 +++++- .../lucene/document/TestBigIntegerPoint.java | 15 ++++ .../lucene/document/TestInetAddressPoint.java | 21 +++++ .../lucene/document/TestLatLonPoint.java | 20 +++++ .../spatial3d/PointInGeo3DShapeQuery.java | 4 +- .../lucene/spatial3d/TestGeo3DPoint.java | 14 ++++ 9 files changed, 202 insertions(+), 23 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index 944fadfb2bb2..bee864f8966d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -103,7 +103,7 @@ protected PointInSetQuery(String field, int numDims, int bytesPerDim, Stream pac } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public final Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { // We don't use RandomAccessWeight here: it's no good to approximate with "match all docs". // This is an inverted structure and should be used in the first pass: @@ -161,14 +161,12 @@ private class MergePointVisitor implements IntersectVisitor { private final DocIdSetBuilder result; private TermIterator iterator; private BytesRef nextQueryPoint; - private final byte[] lastMaxPackedValue; private final BytesRef scratch = new BytesRef(); private final PrefixCodedTerms sortedPackedPoints; public MergePointVisitor(PrefixCodedTerms sortedPackedPoints, DocIdSetBuilder result) throws IOException { this.result = result; this.sortedPackedPoints = sortedPackedPoints; - lastMaxPackedValue = new byte[bytesPerDim]; scratch.length = bytesPerDim; this.iterator = sortedPackedPoints.iterator(); nextQueryPoint = iterator.next(); @@ -304,7 +302,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { } @Override - public int hashCode() { + public final int hashCode() { int hash = super.hashCode(); hash = 31 * hash + sortedPackedPointsHashCode; hash = 31 * hash + numDims; @@ -313,7 +311,7 @@ public int hashCode() { } @Override - public boolean equals(Object other) { + public final boolean equals(Object other) { if (super.equals(other)) { final PointInSetQuery q = (PointInSetQuery) other; return q.numDims == numDims && @@ -326,7 +324,7 @@ public boolean equals(Object other) { } @Override - public String toString(String field) { + public final String toString(String field) { final StringBuilder sb = new StringBuilder(); if (this.field.equals(field) == false) { sb.append(this.field); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index ebbe7e2fbd6a..9384d235d3bc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -109,7 +109,7 @@ public static void checkArgs(String field, Object lowerPoint, Object upperPoint) } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public final Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { // We don't use RandomAccessWeight here: it's no good to approximate with "match all docs". // This is an inverted structure and should be used in the first pass: @@ -239,7 +239,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } @Override - public int hashCode() { + public final int hashCode() { int hash = super.hashCode(); hash = 31 * hash + Arrays.hashCode(lowerPoint); hash = 31 * hash + Arrays.hashCode(upperPoint); @@ -249,20 +249,36 @@ public int hashCode() { } @Override - public boolean equals(Object other) { - if (super.equals(other)) { - final PointRangeQuery q = (PointRangeQuery) other; - return q.numDims == numDims && - q.bytesPerDim == bytesPerDim && - Arrays.equals(lowerPoint, q.lowerPoint) && - Arrays.equals(upperPoint, q.upperPoint); + public final boolean equals(Object other) { + if (super.equals(other) == false) { + return false; } - return false; + final PointRangeQuery q = (PointRangeQuery) other; + if (q.numDims != numDims) { + return false; + } + + if (q.bytesPerDim != bytesPerDim) { + return false; + } + + // Cannot use Arrays.equals here, because it in turn uses byte[].equals + // to compare each value, which only uses "==" + for(int dim=0;dim() { + @Override + public int compare(byte[] a, byte[] b) { + return StringHelper.compare(BYTES, a, 0, b, 0); + } + }); final BytesRef encoded = new BytesRef(new byte[BYTES]); @@ -230,7 +245,7 @@ public BytesRef next() { if (upto == sortedValues.length) { return null; } else { - encoded.bytes = encode(sortedValues[upto]); + encoded.bytes = sortedValues[upto]; assert encoded.bytes.length == encoded.length; upto++; return encoded; diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index 500c2a320612..8f38bcd1aa46 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -93,4 +94,18 @@ public void testToString() throws Exception { new BigInteger[] {BigInteger.valueOf(17), BigInteger.valueOf(42)}).toString()); assertEquals("field:{1}", BigIntegerPoint.newSetQuery("field", BigInteger.ONE).toString()); } + + public void testQueryEquals() throws Exception { + Query q = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); + assertEquals(q, BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000))); + assertFalse(q.equals(BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(1000)))); + + q = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); + assertEquals(q, BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000))); + assertFalse(q.equals(BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1)))); + + q = BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000), BigInteger.valueOf(17)); + assertEquals(q, BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(17), BigInteger.valueOf(0), BigInteger.valueOf(1000))); + assertFalse(q.equals(BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(17), BigInteger.valueOf(1000)))); + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index d4ddb3adbfde..c91b52b252fb 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -45,6 +46,7 @@ public void testBasics() throws Exception { assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 24))); assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")))); assertEquals(1, searcher.count(InetAddressPoint.newSetQuery("field", InetAddress.getByName("1.2.3.4")))); + assertEquals(1, searcher.count(InetAddressPoint.newSetQuery("field", InetAddress.getByName("1.2.3.4"), InetAddress.getByName("1.2.3.5")))); assertEquals(0, searcher.count(InetAddressPoint.newSetQuery("field", InetAddress.getByName("1.2.3.3")))); assertEquals(0, searcher.count(InetAddressPoint.newSetQuery("field"))); @@ -88,4 +90,23 @@ public void testToString() throws Exception { assertEquals("field:[fdc8:57ed:f042:ad1:0:0:0:0 TO fdc8:57ed:f042:ad1:ffff:ffff:ffff:ffff]", InetAddressPoint.newPrefixQuery("field", InetAddress.getByName("fdc8:57ed:f042:0ad1:f66d:4ff:fe90:ce0c"), 64).toString()); assertEquals("field:{fdc8:57ed:f042:ad1:f66d:4ff:fe90:ce0c}", InetAddressPoint.newSetQuery("field", InetAddress.getByName("fdc8:57ed:f042:0ad1:f66d:4ff:fe90:ce0c")).toString()); } + + public void testQueryEquals() throws Exception { + Query q = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + assertEquals(q, InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5"))); + assertFalse(q.equals(InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); + + q = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); + assertEquals(q, InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16)); + assertFalse(q.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.1.3.5"), 16))); + assertFalse(q.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.5"), 24))); + + q = InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3")); + assertEquals(q, InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3"))); + assertFalse(q.equals(InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.5")))); + + q = InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + assertEquals(q, InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5"))); + assertFalse(q.equals(InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 0ef948d3f6dd..61c6754c6542 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -19,6 +19,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -170,4 +171,23 @@ public void testEncodeDecodeIsStable() throws Exception { assertEquals(lonEnc, lonEnc2, 0.0); } } + + public void testQueryEquals() throws Exception { + Query q = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); + assertEquals(q, LatLonPoint.newBoxQuery("field", 50, 70, -40, 20)); + assertFalse(q.equals(LatLonPoint.newBoxQuery("field", 50, 70, -40, 10))); + + q = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); + assertEquals(q, LatLonPoint.newDistanceQuery("field", 50, 70, 10000)); + assertFalse(q.equals(LatLonPoint.newDistanceQuery("field", 50, 70, 11000))); + assertFalse(q.equals(LatLonPoint.newDistanceQuery("field", 50, 60, 10000))); + + + double[] polyLats1 = new double[] {30, 40, 40, 30, 30}; + double[] polyLons1 = new double[] {90, 90, -40, -40, 90}; + double[] polyLats2 = new double[] {20, 40, 40, 20, 20}; + q = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); + assertEquals(q, LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1)); + assertFalse(q.equals(LatLonPoint.newPolygonQuery("field", polyLats2, polyLons1))); + } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 9df8752b1353..c9b5e4e635bb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -43,7 +43,7 @@ * * @lucene.experimental */ -class PointInGeo3DShapeQuery extends Query { +final class PointInGeo3DShapeQuery extends Query { final String field; final GeoShape shape; @@ -192,7 +192,7 @@ public boolean equals(Object o) { } @Override - public final int hashCode() { + public int hashCode() { int result = super.hashCode(); result = 31 * result + shape.hashCode(); return result; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index a4d8ed136316..3061b763f856 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -807,4 +807,18 @@ public void testShapeQueryToString() { private static Directory getDirectory() { return newDirectory(); } + + public void testEquals() { + GeoShape shape = randomShape(PlanetModel.WGS84); + Query q = Geo3DPoint.newShapeQuery("point", shape); + assertEquals(q, Geo3DPoint.newShapeQuery("point", shape)); + + // make a different random shape: + GeoShape shape2; + do { + shape2 = randomShape(PlanetModel.WGS84); + } while (shape.equals(shape2)); + + assertFalse(q.equals(Geo3DPoint.newShapeQuery("point", shape2))); + } } From ba0f63c0c25ca520b8816ac1ae803d7c7451dcca Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 9 Mar 2016 10:06:11 -0500 Subject: [PATCH 0079/1113] LUCENE-7088, LUCENE-7075: Add PointRangeQueryBuilder to xml-queryparser to replace LegacyNumericRangeQueryBuilder --- .../lucene/queryparser/xml/CoreParser.java | 1 + .../LegacyNumericRangeQueryBuilder.java | 2 + .../xml/builders/PointRangeQueryBuilder.java | 95 +++++++++++++++++++ .../queryparser/xml/PointRangeQuery.xml | 31 ++++++ .../queryparser/xml/TestCoreParser.java | 7 ++ 5 files changed, 136 insertions(+) create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/PointRangeQueryBuilder.java create mode 100644 lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/PointRangeQuery.xml diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java index 6bbb626a7507..1416f255bdd4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CoreParser.java @@ -67,6 +67,7 @@ protected CoreParser(String defaultField, Analyzer analyzer, QueryParser parser) queryFactory.addBuilder("MatchAllDocsQuery", new MatchAllDocsQueryBuilder()); queryFactory.addBuilder("BooleanQuery", new BooleanQueryBuilder(queryFactory)); queryFactory.addBuilder("LegacyNumericRangeQuery", new LegacyNumericRangeQueryBuilder()); + queryFactory.addBuilder("PointRangeQuery", new PointRangeQueryBuilder()); queryFactory.addBuilder("RangeQuery", new RangeQueryBuilder()); queryFactory.addBuilder("DisjunctionMaxQuery", new DisjunctionMaxQueryBuilder(queryFactory)); if (parser != null) { diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java index 2aba681347fd..e19596430e24 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LegacyNumericRangeQueryBuilder.java @@ -83,7 +83,9 @@ * A {@link ParserException} will be thrown if an error occurs parsing the * supplied lowerTerm or upperTerm into the numeric type * specified by type. + * @deprecated Index with points and use {@link PointRangeQueryBuilder} instead */ +@Deprecated public class LegacyNumericRangeQueryBuilder implements QueryBuilder { @Override diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/PointRangeQueryBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/PointRangeQueryBuilder.java new file mode 100644 index 000000000000..45483168bacd --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/PointRangeQueryBuilder.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.xml.builders; + +import org.apache.lucene.search.Query; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.queryparser.xml.DOMUtils; +import org.apache.lucene.queryparser.xml.ParserException; +import org.apache.lucene.queryparser.xml.QueryBuilder; +import org.w3c.dom.Element; + +/** + * Creates a range query across 1D {@link PointValues}. The table below specifies the required + * attributes and the defaults if optional attributes are omitted: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    Attribute nameValuesRequiredDefault
    fieldNameStringYesN/A
    lowerTermSpecified by typeYesN/A
    upperTermSpecified by typeYesN/A
    typeint, long, float, doubleNoint
    + *

    + * A {@link ParserException} will be thrown if an error occurs parsing the + * supplied lowerTerm or upperTerm into the numeric type + * specified by type. + */ +public class PointRangeQueryBuilder implements QueryBuilder { + + @Override + public Query getQuery(Element e) throws ParserException { + String field = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName"); + String lowerTerm = DOMUtils.getAttributeOrFail(e, "lowerTerm"); + String upperTerm = DOMUtils.getAttributeOrFail(e, "upperTerm"); + + String type = DOMUtils.getAttribute(e, "type", "int"); + try { + if (type.equalsIgnoreCase("int")) { + return IntPoint.newRangeQuery(field, Integer.valueOf(lowerTerm), Integer.valueOf(upperTerm)); + } else if (type.equalsIgnoreCase("long")) { + return LongPoint.newRangeQuery(field, Long.valueOf(lowerTerm), Long.valueOf(upperTerm)); + } else if (type.equalsIgnoreCase("double")) { + return DoublePoint.newRangeQuery(field, Double.valueOf(lowerTerm), Double.valueOf(upperTerm)); + } else if (type.equalsIgnoreCase("float")) { + return FloatPoint.newRangeQuery(field, Float.valueOf(lowerTerm), Float.valueOf(upperTerm)); + } else { + throw new ParserException("type attribute must be one of: [long, int, double, float]"); + } + } catch (NumberFormatException nfe) { + throw new ParserException("Could not parse lowerTerm or upperTerm into a number", nfe); + } + } +} diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/PointRangeQuery.xml b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/PointRangeQuery.xml new file mode 100644 index 000000000000..45af138b8dae --- /dev/null +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/PointRangeQuery.xml @@ -0,0 +1,31 @@ + + + + + merger + + + sumitomo + + + bank + + + + + diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java index e675723e320d..5cadec21f3f6 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -72,6 +73,7 @@ public static void beforeClass() throws Exception { doc.add(newTextField("date", date, Field.Store.YES)); doc.add(newTextField("contents", content, Field.Store.YES)); doc.add(new LegacyIntField("date2", Integer.valueOf(date), Field.Store.NO)); + doc.add(new IntPoint("date3", Integer.valueOf(date))); writer.addDocument(doc); line = d.readLine(); } @@ -164,6 +166,11 @@ public void testNumericRangeQueryXML() throws ParserException, IOException { Query q = parse("LegacyNumericRangeQuery.xml"); dumpResults("LegacyNumericRangeQuery", q, 5); } + + public void testPointRangeQuery() throws ParserException, IOException { + Query q = parse("PointRangeQuery.xml"); + dumpResults("PointRangeQuery", q, 5); + } //================= Helper methods =================================== From b9e204c07d6499256086422abd769235e4e1cda5 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 11:03:19 -0500 Subject: [PATCH 0080/1113] CheckIndex failed to say it was checking points --- .../org/apache/lucene/index/CheckIndex.java | 7 +++++ .../index/TestAllFilesCheckIndexHeader.java | 3 +- .../index/TestAllFilesDetectTruncation.java | 4 +-- .../apache/lucene/index/TestPointValues.java | 29 +++++++++++++++++++ .../lucene/index/TestSwappedIndexFiles.java | 3 +- .../lucene/store/MockDirectoryWrapper.java | 2 +- .../java/org/apache/lucene/util/TestUtil.java | 14 +++++---- 7 files changed, 48 insertions(+), 14 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index 3c437c1c5084..89b36efa2d32 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -1683,9 +1683,14 @@ public static Status.TermIndexStatus testPostings(CodecReader reader, PrintStrea * @lucene.experimental */ public static Status.PointsStatus testPoints(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException { + if (infoStream != null) { + infoStream.print(" test: points.............."); + } + long startNS = System.nanoTime(); FieldInfos fieldInfos = reader.getFieldInfos(); Status.PointsStatus status = new Status.PointsStatus(); try { + if (fieldInfos.hasPointValues()) { PointsReader values = reader.getPointsReader(); if (values == null) { @@ -1840,6 +1845,8 @@ private void checkPackedValue(String desc, byte[] packedValue, int docID) { } } } + msg(infoStream, String.format(Locale.ROOT, "OK [%d fields, %d points] [took %.3f sec]", status.totalValueFields, status.totalValuePoints, nsToSec(System.nanoTime()-startNS))); + } catch (Throwable e) { if (failFast) { IOUtils.reThrow(e); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index 68b7cc23ea58..f6c1486bc433 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -28,7 +28,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -132,7 +131,7 @@ private void checkOneFile(Directory dir, String victim) throws IOException { // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim); } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java index 16caae3baa54..c751417cd742 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -19,7 +19,6 @@ import java.io.EOFException; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import org.apache.lucene.analysis.MockAnalyzer; @@ -28,7 +27,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -116,7 +114,7 @@ private void truncateOneFile(Directory dir, String victim) throws IOException { // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim); } catch (CorruptIndexException | EOFException e) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 55d47944d302..9b18f0298ee3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; @@ -40,6 +41,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -628,4 +630,31 @@ public void testSparsePoints() throws Exception { w.close(); dir.close(); } + + public void testCheckIndexIncludesPoints() throws Exception { + Directory dir = new RAMDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + Document doc = new Document(); + doc.add(new IntPoint("int1", 17)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new IntPoint("int1", 44)); + doc.add(new IntPoint("int2", -17)); + w.addDocument(doc); + w.close(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + CheckIndex.Status status = TestUtil.checkIndex(dir, false, true, output); + assertEquals(1, status.segmentInfos.size()); + CheckIndex.Status.SegmentInfoStatus segStatus = status.segmentInfos.get(0); + // total 3 point values were index: + assertEquals(3, segStatus.pointsStatus.totalValuePoints); + // ... across 2 fields: + assertEquals(2, segStatus.pointsStatus.totalValueFields); + + // Make sure CheckIndex in fact declares that it is testing points! + assertTrue(output.toString(IOUtils.UTF_8).contains("test: points...")); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java index a284fdd9d100..d2205020d99f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java @@ -28,7 +28,6 @@ import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -118,7 +117,7 @@ private void swapOneFile(Directory dir1, Directory dir2, String victim) throws I // CheckIndex should also fail: try { - TestUtil.checkIndex(dirCopy, true, true); + TestUtil.checkIndex(dirCopy, true, true, null); fail("wrong file " + victim + " not detected"); } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { // expected diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 7fe7c3b0d5a3..c2544b4ffba0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -850,7 +850,7 @@ public synchronized void close() throws IOException { System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); } - TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true); + TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true, null); } // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index 5e328ba13661..d772ae321d3f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -279,28 +279,30 @@ public static CheckIndex.Status checkIndex(Directory dir) throws IOException { } public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException { - return checkIndex(dir, crossCheckTermVectors, false); + return checkIndex(dir, crossCheckTermVectors, false, null); } /** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to * look for any other corruption. */ - public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); + public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast, ByteArrayOutputStream output) throws IOException { + if (output == null) { + output = new ByteArrayOutputStream(1024); + } // TODO: actually use the dir's locking, unless test uses a special method? // some tests e.g. exception tests become much more complicated if they have to close the writer try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) { checker.setCrossCheckTermVectors(crossCheckTermVectors); checker.setFailFast(failFast); - checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false); + checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false); CheckIndex.Status indexStatus = checker.checkIndex(null); if (indexStatus == null || indexStatus.clean == false) { System.out.println("CheckIndex failed"); - System.out.println(bos.toString(IOUtils.UTF_8)); + System.out.println(output.toString(IOUtils.UTF_8)); throw new RuntimeException("CheckIndex failed"); } else { if (LuceneTestCase.INFOSTREAM) { - System.out.println(bos.toString(IOUtils.UTF_8)); + System.out.println(output.toString(IOUtils.UTF_8)); } return indexStatus; } From 94d333983e5a57d1aae7278c6f2c030c6d401dcf Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 11:25:00 -0500 Subject: [PATCH 0081/1113] add missing argument --- .../test/org/apache/lucene/index/TestMaxPositionInOldIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java index 87625d206742..3608b8067e75 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java @@ -82,7 +82,7 @@ public void testCorruptIndex() throws Exception { BaseDirectoryWrapper dir = newFSDirectory(path); dir.setCheckIndexOnClose(false); RuntimeException expected = expectThrows(RuntimeException.class, () -> { - TestUtil.checkIndex(dir, false, true); + TestUtil.checkIndex(dir, false, true, null); }); assertTrue(expected.getMessage().contains("pos 2147483647 > IndexWriter.MAX_POSITION=2147483519")); From 5a43a3e772e47bb086db3c1d635e9b17e28a39f7 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 9 Mar 2016 16:00:15 +0000 Subject: [PATCH 0082/1113] LUCENE-7084: fail precommit on comparingIdentical. also replaces one assert in the SingletonSortedSetDocValues constructor with equivalent test (TestSortedSetDocValues.testNoMoreOrdsConstant). --- .../index/SingletonSortedSetDocValues.java | 1 - .../lucene/index/TestSortedSetDocValues.java | 27 +++++++++++++++++++ lucene/tools/javadoc/ecj.javadocs.prefs | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestSortedSetDocValues.java diff --git a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java index c90fcc5fafd0..5077cd843202 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java @@ -33,7 +33,6 @@ final class SingletonSortedSetDocValues extends RandomAccessOrds { /** Creates a multi-valued view over the provided SortedDocValues */ public SingletonSortedSetDocValues(SortedDocValues in) { this.in = in; - assert NO_MORE_ORDS == -1; // this allows our nextOrd() to work for missing values without a check } /** Return the wrapped {@link SortedDocValues} */ diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortedSetDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestSortedSetDocValues.java new file mode 100644 index 000000000000..8cffeaf00a65 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestSortedSetDocValues.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestSortedSetDocValues extends LuceneTestCase { + + public void testNoMoreOrdsConstant() { + assertEquals(SortedSetDocValues.NO_MORE_ORDS, -1); + } + +} diff --git a/lucene/tools/javadoc/ecj.javadocs.prefs b/lucene/tools/javadoc/ecj.javadocs.prefs index d01148c602fa..bd98c99ac621 100644 --- a/lucene/tools/javadoc/ecj.javadocs.prefs +++ b/lucene/tools/javadoc/ecj.javadocs.prefs @@ -11,7 +11,7 @@ org.eclipse.jdt.core.compiler.doc.comment.support=enabled org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=ignore org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.autoboxing=ignore -org.eclipse.jdt.core.compiler.problem.comparingIdentical=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=error org.eclipse.jdt.core.compiler.problem.deadCode=ignore org.eclipse.jdt.core.compiler.problem.deprecation=ignore org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled From 42361a68bc27266d8f50e90b85ffd980ac953f36 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 9 Mar 2016 11:25:37 -0500 Subject: [PATCH 0083/1113] LUCENE-7079: add newSetQuery(String, Collection) to primitive Point types --- .../org/apache/lucene/document/DoublePoint.java | 16 ++++++++++++++++ .../org/apache/lucene/document/FloatPoint.java | 16 ++++++++++++++++ .../org/apache/lucene/document/IntPoint.java | 16 ++++++++++++++++ .../org/apache/lucene/document/LongPoint.java | 16 ++++++++++++++++ .../apache/lucene/search/TestPointQueries.java | 8 ++++++++ .../apache/lucene/search/join/TestBlockJoin.java | 7 +------ 6 files changed, 73 insertions(+), 6 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java index 26ac0ced6f10..1133b2213938 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Arrays; +import java.util.Collection; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; @@ -247,4 +248,19 @@ protected String toString(byte[] value) { } }; } + + /** + * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. + * + * @param field field name. must not be {@code null}. + * @param values all values to match + */ + public static Query newSetQuery(String field, Collection values) { + Double[] boxed = values.toArray(new Double[0]); + double[] unboxed = new double[boxed.length]; + for (int i = 0; i < boxed.length; i++) { + unboxed[i] = boxed[i]; + } + return newSetQuery(field, unboxed); + } } diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java index c58881ec8800..3d110db2cd86 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Arrays; +import java.util.Collection; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; @@ -247,4 +248,19 @@ protected String toString(byte[] value) { } }; } + + /** + * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. + * + * @param field field name. must not be {@code null}. + * @param values all values to match + */ + public static Query newSetQuery(String field, Collection values) { + Float[] boxed = values.toArray(new Float[0]); + float[] unboxed = new float[boxed.length]; + for (int i = 0; i < boxed.length; i++) { + unboxed[i] = boxed[i]; + } + return newSetQuery(field, unboxed); + } } diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java index cb8315f2f205..53ae3d3dd2ac 100644 --- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Arrays; +import java.util.Collection; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; @@ -247,4 +248,19 @@ protected String toString(byte[] value) { } }; } + + /** + * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. + * + * @param field field name. must not be {@code null}. + * @param values all values to match + */ + public static Query newSetQuery(String field, Collection values) { + Integer[] boxed = values.toArray(new Integer[0]); + int[] unboxed = new int[boxed.length]; + for (int i = 0; i < boxed.length; i++) { + unboxed[i] = boxed[i]; + } + return newSetQuery(field, unboxed); + } } diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java index ff78132d7b66..c4fd8875205b 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java @@ -17,6 +17,7 @@ package org.apache.lucene.document; import java.util.Arrays; +import java.util.Collection; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.PointInSetQuery; @@ -247,4 +248,19 @@ protected String toString(byte[] value) { } }; } + + /** + * Create a query matching any of the specified 1D values. This is the points equivalent of {@code TermsQuery}. + * + * @param field field name. must not be {@code null}. + * @param values all values to match + */ + public static Query newSetQuery(String field, Collection values) { + Long[] boxed = values.toArray(new Long[0]); + long[] unboxed = new long[boxed.length]; + for (int i = 0; i < boxed.length; i++) { + unboxed[i] = boxed[i]; + } + return newSetQuery(field, unboxed); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index 4d9aa593f061..c72ab44ac398 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -1602,6 +1602,14 @@ public void testBasicPointInSetQuery() throws Exception { r.close(); dir.close(); } + + /** Boxed methods for primitive types should behave the same as unboxed: just sugar */ + public void testPointIntSetBoxed() throws Exception { + assertEquals(IntPoint.newSetQuery("foo", 1, 2, 3), IntPoint.newSetQuery("foo", Arrays.asList(1, 2, 3))); + assertEquals(FloatPoint.newSetQuery("foo", 1F, 2F, 3F), FloatPoint.newSetQuery("foo", Arrays.asList(1F, 2F, 3F))); + assertEquals(LongPoint.newSetQuery("foo", 1L, 2L, 3L), LongPoint.newSetQuery("foo", Arrays.asList(1L, 2L, 3L))); + assertEquals(DoublePoint.newSetQuery("foo", 1D, 2D, 3D), DoublePoint.newSetQuery("foo", Arrays.asList(1D, 2D, 3D))); + } public void testBasicMultiValuedPointInSetQuery() throws Exception { Directory dir = newDirectory(); diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index b5f2038a0dcf..9c39299c4b4f 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -615,12 +615,7 @@ public void testRandom() throws Exception { } if (!toDelete.isEmpty()) { - // TODO: we should add newSetQuery(String, Collection) ? this is awkward. - int[] array = new int[toDelete.size()]; - for (int i = 0; i < toDelete.size(); i++) { - array[i] = toDelete.get(i); - } - Query query = IntPoint.newSetQuery("blockID", array); + Query query = IntPoint.newSetQuery("blockID", toDelete); w.deleteDocuments(query); joinW.deleteDocuments(query); } From 4e911f2d3a029ae30dad9ea5ffb42530398adcbc Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 9 Mar 2016 12:16:59 -0500 Subject: [PATCH 0084/1113] LUCENE-7076: improve deprecation message for LegacyNumericRangeQuery --- .../lucene/search/LegacyNumericRangeQuery.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java index c61f7a823bfe..fe6c9e24864e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/LegacyNumericRangeQuery.java @@ -21,9 +21,14 @@ import java.util.LinkedList; import java.util.Objects; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.FilteredTermsEnum; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.AttributeSource; @@ -158,7 +163,12 @@ * precision step). This query type was developed for a geographic portal, where the performance for * e.g. bounding boxes or exact date/time stamps is important.

    * - * @deprecated Please use {@link PointRangeQuery} instead + * @deprecated Instead index with {@link IntPoint}, {@link LongPoint}, {@link FloatPoint}, {@link DoublePoint}, and + * create range queries with {@link IntPoint#newRangeQuery(String, int, int) IntPoint.newRangeQuery()}, + * {@link LongPoint#newRangeQuery(String, long, long) LongPoint.newRangeQuery()}, + * {@link FloatPoint#newRangeQuery(String, float, float) FloatPoint.newRangeQuery()}, + * {@link DoublePoint#newRangeQuery(String, double, double) DoublePoint.newRangeQuery()} respectively. + * See {@link PointValues} for background information on Points. * * @since 2.9 **/ From 6b2f3638969e872c704e3d192caec07ad7ef99ed Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 9 Mar 2016 17:38:07 +0000 Subject: [PATCH 0085/1113] SOLR-8765: Set parameters correctly in async shard requests --- .../solr/client/solrj/request/CollectionAdminRequest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 4f28408ce210..76eb19f76409 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -203,6 +203,8 @@ protected abstract static class AsyncShardSpecificAdminRequest extends AsyncColl public AsyncShardSpecificAdminRequest(CollectionAction action, String collection, String shard) { super(action); + this.collection = collection; + this.shard = shard; } @Deprecated From a574f4724d4b963cdac1dcc1c36afe6715956125 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 9 Mar 2016 14:10:10 -0500 Subject: [PATCH 0086/1113] don't use slow composite wrapper in these tests --- .../lucene/search/join/TestBlockJoin.java | 18 +++++++++--------- .../lucene/queryparser/xml/TestCoreParser.java | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index 9c39299c4b4f..01a81351511f 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -176,7 +176,7 @@ public void testSimple() throws Exception { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Create a filter that defines "parent" documents in the index - in this case resumes BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -263,7 +263,7 @@ public void testBugCausedByRewritingTwice() throws IOException { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Hacky: this causes the query to need 2 rewrite // iterations: @@ -336,7 +336,7 @@ public void testSimpleFilter() throws Exception { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Create a filter that defines "parent" documents in the index - in this case resumes BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -640,7 +640,7 @@ public void testRandom() throws Exception { } } - final IndexSearcher s = newSearcher(r); + final IndexSearcher s = newSearcher(r, false); final IndexSearcher joinS = new IndexSearcher(joinR); @@ -1051,7 +1051,7 @@ public void testMultiChildTypes() throws Exception { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Create a filter that defines "parent" documents in the index - in this case resumes BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -1575,7 +1575,7 @@ public void testParentScoringBug() throws Exception { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Create a filter that defines "parent" documents in the index - in this case resumes BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -1613,7 +1613,7 @@ public void testToChildBlockJoinQueryExplain() throws Exception { IndexReader r = w.getReader(); w.close(); - IndexSearcher s = newSearcher(r); + IndexSearcher s = newSearcher(r, false); // Create a filter that defines "parent" documents in the index - in this case resumes BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -1644,7 +1644,7 @@ public void testToChildInitialAdvanceParentButNoKids() throws Exception { w.forceMerge(1); final IndexReader r = w.getReader(); - final IndexSearcher s = newSearcher(r); + final IndexSearcher s = newSearcher(r, false); w.close(); BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); @@ -1690,7 +1690,7 @@ public void testMultiChildQueriesOfDiffParentLevels() throws Exception { } final IndexReader r = w.getReader(); - final IndexSearcher s = newSearcher(r); + final IndexSearcher s = newSearcher(r, false); w.close(); BitSetProducer resumeFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume"))); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java index 5cadec21f3f6..c8b357ed30e4 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java @@ -80,7 +80,7 @@ public static void beforeClass() throws Exception { d.close(); writer.close(); reader = DirectoryReader.open(dir); - searcher = newSearcher(reader); + searcher = newSearcher(reader, false); } From a7cec361654c20153ac4fdd9ede9914422e34a47 Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Wed, 9 Mar 2016 11:40:44 -0800 Subject: [PATCH 0087/1113] SOLR-8813: Add test for MultiValued fields being returned in the correct order --- .../conf/schema-non-stored-docvalues.xml | 131 +++++++++--------- .../solr/schema/TestUseDocValuesAsStored.java | 60 +++++++- 2 files changed, 126 insertions(+), 65 deletions(-) diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml index aab4da4e2e87..f700e60a4169 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml @@ -35,70 +35,73 @@ 1.6: useDocValuesAsStored defaults to true. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java index 46f76ab8deca..839121acd3c1 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java +++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java @@ -76,7 +76,7 @@ public String getCoreName() { @Test public void testOnEmptyIndex() throws Exception { - assertU(delQ("*:*")); + clearIndex(); assertU(commit()); assertJQ(req("q", "*:*"), "/response/numFound==0"); assertJQ(req("q", "*:*", "fl", "*"), "/response/numFound==0"); @@ -238,4 +238,62 @@ private void doTest(String desc, String field, String type, String... value) { assertQ(desc + ": " + fl, req("q", "*:*", "fl", fl), xpaths); } + + // See SOLR-8740 for a discussion. This test is here to make sure we consciously change behavior of multiValued + // fields given that we can now return docValues fields. The behavior we've guaranteed in the past is that if + // multiValued fields are stored, they're returned in the document in the order they were added. + // There are four new fieldTypes added: + // + // + // + // + // + // If any of these tests break as a result of returning DocValues rather than stored values, make sure we reach some + // consensus that any breaks on back-compat are A Good Thing and that that behavior is carefully documented! + + @Test + public void testMultivaluedOrdering() throws Exception { + clearIndex(); + + // multiValued=true, docValues=true, stored=true. Should return in original order + assertU(adoc("id", "1", "test_mvt_dvt_st_str", "cccc", "test_mvt_dvt_st_str", "aaaa", "test_mvt_dvt_st_str", "bbbb")); + + // multiValued=true, docValues=true, stored=false. Should return in sorted order + assertU(adoc("id", "2", "test_mvt_dvt_sf_str", "cccc", "test_mvt_dvt_sf_str", "aaaa", "test_mvt_dvt_sf_str", "bbbb")); + + // multiValued=true, docValues=false, stored=true. Should return in original order + assertU(adoc("id", "3", "test_mvt_dvf_st_str", "cccc", "test_mvt_dvf_st_str", "aaaa", "test_mvt_dvf_st_str", "bbbb")); + + // multiValued=true, docValues=not specified, stored=true. Should return in original order + assertU(adoc("id", "4", "test_mvt_dvu_st_str", "cccc", "test_mvt_dvu_st_str", "aaaa", "test_mvt_dvu_st_str", "bbbb")); + + assertU(commit()); + + assertJQ(req("q", "id:1", "fl", "test_mvt_dvt_st_str"), + "/response/docs/[0]/test_mvt_dvt_st_str/[0]==cccc", + "/response/docs/[0]/test_mvt_dvt_st_str/[1]==aaaa", + "/response/docs/[0]/test_mvt_dvt_st_str/[2]==bbbb"); + + // Currently, this test fails since stored=false. When SOLR-8740 is committed, it should not throw an exception + // and should succeed, returning the field in sorted order. + try { + assertJQ(req("q", "id:2", "fl", "test_mvt_dvt_sf_str"), + "/response/docs/[0]/test_mvt_dvt_sf_str/[0]==aaaa", + "/response/docs/[0]/test_mvt_dvt_sf_str/[1]==bbbb", + "/response/docs/[0]/test_mvt_dvt_sf_str/[2]==cccc"); + } catch (Exception e) { + // do nothing until SOLR-8740 is committed. At that point this should not throw an exception. + // NOTE: I think the test is correct after 8740 so just remove the try/catch + } + assertJQ(req("q", "id:3", "fl", "test_mvt_dvf_st_str"), + "/response/docs/[0]/test_mvt_dvf_st_str/[0]==cccc", + "/response/docs/[0]/test_mvt_dvf_st_str/[1]==aaaa", + "/response/docs/[0]/test_mvt_dvf_st_str/[2]==bbbb"); + + assertJQ(req("q", "id:4", "fl", "test_mvt_dvu_st_str"), + "/response/docs/[0]/test_mvt_dvu_st_str/[0]==cccc", + "/response/docs/[0]/test_mvt_dvu_st_str/[1]==aaaa", + "/response/docs/[0]/test_mvt_dvu_st_str/[2]==bbbb"); + + } } From 8b408be7a2960a5f5a371cc6acb5af3e15b31344 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 9 Mar 2016 21:15:40 +0000 Subject: [PATCH 0088/1113] SOLR-8765: Throw SolrException rather than IAE on name validation --- .../client/solrj/util/SolrIdentifierValidator.java | 4 +++- .../solrj/request/TestCollectionAdminRequest.java | 9 +++++---- .../solr/client/solrj/request/TestCoreAdmin.java | 13 ++++++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java index 57f9909f0514..9473a2806b67 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrIdentifierValidator.java @@ -19,6 +19,8 @@ import java.util.Locale; import java.util.regex.Pattern; +import org.apache.solr.common.SolrException; + /** * Ensures that provided identifiers align with Solr's recommendations/requirements for choosing * collection, core, etc identifiers. @@ -34,7 +36,7 @@ public enum IdentifierType { public static String validateName(IdentifierType type, String name) { if (!validateIdentifier(name)) - throw new IllegalArgumentException(getIdentifierMessage(type, name)); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, getIdentifierMessage(type, name)); return name; } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java index 5d5c31517bd0..c21e523855c0 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java @@ -20,6 +20,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create; import org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateAlias; import org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateShard; +import org.apache.solr.common.SolrException; import org.junit.Test; /** @@ -33,7 +34,7 @@ public void testInvalidCollectionNameRejectedWhenCreatingCollection() { try { createRequest.setCollectionName("invalid$collection@name"); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(exceptionMessage.contains("Invalid collection")); assertTrue(exceptionMessage.contains("invalid$collection@name")); @@ -47,7 +48,7 @@ public void testInvalidShardNamesRejectedWhenCreatingCollection() { try { createRequest.setShards("invalid$shard@name"); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(exceptionMessage.contains("Invalid shard")); assertTrue(exceptionMessage.contains("invalid$shard@name")); @@ -61,7 +62,7 @@ public void testInvalidAliasNameRejectedWhenCreatingAlias() { try { createAliasRequest.setAliasName("invalid$alias@name"); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(exceptionMessage.contains("Invalid alias")); assertTrue(exceptionMessage.contains("invalid$alias@name")); @@ -75,7 +76,7 @@ public void testInvalidShardNameRejectedWhenCreatingShard() { try { createShardRequest.setShardName("invalid$shard@name"); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(exceptionMessage.contains("Invalid shard")); assertTrue(exceptionMessage.contains("invalid$shard@name")); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java index f3c3d55d5fcd..ef4dad7b6622 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java @@ -16,12 +16,11 @@ */ package org.apache.solr.client.solrj.request; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.core.Is.is; - import java.io.File; import java.lang.invoke.MethodHandles; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrIgnoredThreadsFilter; @@ -43,8 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.core.Is.is; @ThreadLeakFilters(defaultFilters = true, filters = {SolrIgnoredThreadsFilter.class}) public class TestCoreAdmin extends AbstractEmbeddedSolrServerTestCase { @@ -167,7 +166,7 @@ public void testInvalidCoreNamesAreRejectedWhenCreatingCore() { try { createRequest.setCoreName("invalid$core@name"); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(exceptionMessage.contains("Invalid core")); assertTrue(exceptionMessage.contains("invalid$core@name")); @@ -180,7 +179,7 @@ public void testInvalidCoreNamesAreRejectedWhenRenamingExistingCore() throws Exc try { CoreAdminRequest.renameCore("validExistingCoreName", "invalid$core@name", null); fail(); - } catch (IllegalArgumentException e) { + } catch (SolrException e) { final String exceptionMessage = e.getMessage(); assertTrue(e.getMessage(), exceptionMessage.contains("Invalid core")); assertTrue(exceptionMessage.contains("invalid$core@name")); From c8543efacf43dafa76214a96d47876f12c3cf666 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 9 Mar 2016 21:05:26 -0500 Subject: [PATCH 0089/1113] LUCENE-7089, LUCENE-7075: add points to flexible queryparser to replace legacy numerics support --- .../flexible/core/nodes/package-info.java | 4 +- .../standard/StandardQueryParser.java | 29 +++- ...> LegacyNumericRangeQueryNodeBuilder.java} | 26 +-- .../builders/PointRangeQueryNodeBuilder.java | 137 ++++++++++++++++ .../builders/StandardQueryTreeBuilder.java | 12 +- ...icConfig.java => LegacyNumericConfig.java} | 16 +- ... => LegacyNumericFieldConfigListener.java} | 26 +-- .../standard/config/PointsConfig.java | 124 ++++++++++++++ .../standard/config/PointsConfigListener.java | 65 ++++++++ .../config/StandardQueryConfigHandler.java | 39 +++-- .../nodes/LegacyNumericQueryNode.java | 153 ++++++++++++++++++ ....java => LegacyNumericRangeQueryNode.java} | 40 ++--- ...ericQueryNode.java => PointQueryNode.java} | 12 +- .../standard/nodes/PointRangeQueryNode.java | 124 ++++++++++++++ ...a => LegacyNumericQueryNodeProcessor.java} | 38 ++--- ...LegacyNumericRangeQueryNodeProcessor.java} | 38 ++--- .../processors/PointQueryNodeProcessor.java | 136 ++++++++++++++++ .../PointRangeQueryNodeProcessor.java | 148 +++++++++++++++++ .../StandardQueryNodeProcessorPipeline.java | 6 +- ...java => TestLegacyNumericQueryParser.java} | 12 +- .../standard/TestPointQueryParser.java | 82 ++++++++++ lucene/tools/junit4/cached-timehints.txt | 2 +- 22 files changed, 1148 insertions(+), 121 deletions(-) rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/{NumericRangeQueryNodeBuilder.java => LegacyNumericRangeQueryNodeBuilder.java} (75%) create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PointRangeQueryNodeBuilder.java rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/{NumericConfig.java => LegacyNumericConfig.java} (90%) rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/{NumericFieldConfigListener.java => LegacyNumericFieldConfigListener.java} (67%) create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericQueryNode.java rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/{NumericRangeQueryNode.java => LegacyNumericRangeQueryNode.java} (73%) rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/{NumericQueryNode.java => PointQueryNode.java} (91%) create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/{NumericQueryNodeProcessor.java => LegacyNumericQueryNodeProcessor.java} (78%) rename lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/{NumericRangeQueryNodeProcessor.java => LegacyNumericRangeQueryNodeProcessor.java} (81%) create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointQueryNodeProcessor.java create mode 100644 lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointRangeQueryNodeProcessor.java rename lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/{TestNumericQueryParser.java => TestLegacyNumericQueryParser.java} (97%) create mode 100644 lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/package-info.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/package-info.java index c9d55eab12c4..23c72a1ada9b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/package-info.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/nodes/package-info.java @@ -52,14 +52,14 @@ *
  • FuzzyQueryNode - fuzzy node
    • *
  • TermRangeQueryNode - used for parametric field:[low_value TO high_value]
    • *
  • ProximityQueryNode - used for proximity search
    • - *
  • NumericRangeQueryNode - used for numeric range search
    • + *
  • LegacyNumericRangeQueryNode - used for numeric range search
    • *
  • TokenizedPhraseQueryNode - used by tokenizers/lemmatizers/analyzers for phrases/autophrases
    • * *

    * Leaf Nodes: *

      *
    • FieldQueryNode - field/value node
      • - *
    • NumericQueryNode - used for numeric search
      • + *
    • LegacyNumericQueryNode - used for numeric search
      • *
    • PathQueryNode - {@link org.apache.lucene.queryparser.flexible.core.nodes.QueryNode} object used with path-like queries
      • *
    • OpaqueQueryNode - Used as for part of the query that can be parsed by other parsers. schema/value
      • *
    • PrefixWildcardQueryNode - non-phrase wildcard query
      • diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java index ada65a4ca7af..2774cf091ff1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.java @@ -29,7 +29,8 @@ import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler; import org.apache.lucene.queryparser.flexible.standard.builders.StandardQueryTreeBuilder; import org.apache.lucene.queryparser.flexible.standard.config.FuzzyConfig; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.Operator; @@ -322,12 +323,30 @@ public void setFuzzyPrefixLength(int fuzzyPrefixLength) { } - public void setNumericConfigMap(Map numericConfigMap) { - getQueryConfigHandler().set(ConfigurationKeys.NUMERIC_CONFIG_MAP, numericConfigMap); + /** + * Sets field configuration for legacy numeric fields + * @deprecated Index with points instead and use {@link #setPointsConfigMap(Map)} + */ + @Deprecated + public void setLegacyNumericConfigMap(Map legacyNumericConfigMap) { + getQueryConfigHandler().set(ConfigurationKeys.LEGACY_NUMERIC_CONFIG_MAP, legacyNumericConfigMap); + } + + /** + * Gets field configuration for legacy numeric fields + * @deprecated Index with points instead and use {@link #getPointsConfigMap()} + */ + @Deprecated + public Map getLegacyNumericConfigMap() { + return getQueryConfigHandler().get(ConfigurationKeys.LEGACY_NUMERIC_CONFIG_MAP); + } + + public void setPointsConfigMap(Map pointsConfigMap) { + getQueryConfigHandler().set(ConfigurationKeys.POINTS_CONFIG_MAP, pointsConfigMap); } - public Map getNumericConfigMap() { - return getQueryConfigHandler().get(ConfigurationKeys.NUMERIC_CONFIG_MAP); + public Map getPointsConfigMap() { + return getQueryConfigHandler().get(ConfigurationKeys.POINTS_CONFIG_MAP); } /** diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java similarity index 75% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java index 6c8790ff8f06..8ae7d5e2e43f 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/NumericRangeQueryNodeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/LegacyNumericRangeQueryNodeBuilder.java @@ -22,38 +22,40 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.util.StringUtils; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericQueryNode; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; import org.apache.lucene.search.LegacyNumericRangeQuery; /** - * Builds {@link org.apache.lucene.search.LegacyNumericRangeQuery}s out of {@link NumericRangeQueryNode}s. + * Builds {@link org.apache.lucene.search.LegacyNumericRangeQuery}s out of {@link LegacyNumericRangeQueryNode}s. * * @see org.apache.lucene.search.LegacyNumericRangeQuery - * @see NumericRangeQueryNode + * @see LegacyNumericRangeQueryNode + * @deprecated Index with points and use {@link PointRangeQueryNodeBuilder} instead. */ -public class NumericRangeQueryNodeBuilder implements StandardQueryBuilder { +@Deprecated +public class LegacyNumericRangeQueryNodeBuilder implements StandardQueryBuilder { /** - * Constructs a {@link NumericRangeQueryNodeBuilder} object. + * Constructs a {@link LegacyNumericRangeQueryNodeBuilder} object. */ - public NumericRangeQueryNodeBuilder() { + public LegacyNumericRangeQueryNodeBuilder() { // empty constructor } @Override public LegacyNumericRangeQuery build(QueryNode queryNode) throws QueryNodeException { - NumericRangeQueryNode numericRangeNode = (NumericRangeQueryNode) queryNode; + LegacyNumericRangeQueryNode numericRangeNode = (LegacyNumericRangeQueryNode) queryNode; - NumericQueryNode lowerNumericNode = numericRangeNode.getLowerBound(); - NumericQueryNode upperNumericNode = numericRangeNode.getUpperBound(); + LegacyNumericQueryNode lowerNumericNode = numericRangeNode.getLowerBound(); + LegacyNumericQueryNode upperNumericNode = numericRangeNode.getUpperBound(); Number lowerNumber = lowerNumericNode.getValue(); Number upperNumber = upperNumericNode.getValue(); - NumericConfig numericConfig = numericRangeNode.getNumericConfig(); + LegacyNumericConfig numericConfig = numericRangeNode.getNumericConfig(); FieldType.LegacyNumericType numberType = numericConfig.getType(); String field = StringUtils.toString(numericRangeNode.getField()); boolean minInclusive = numericRangeNode.isLowerInclusive(); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PointRangeQueryNodeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PointRangeQueryNodeBuilder.java new file mode 100644 index 000000000000..0cce4bf94f2b --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/PointRangeQueryNodeBuilder.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.builders; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.util.StringUtils; +import org.apache.lucene.queryparser.flexible.messages.MessageImpl; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointRangeQueryNode; +import org.apache.lucene.search.Query; + +/** + * Builds {@link PointValues} range queries out of {@link PointRangeQueryNode}s. + * + * @see PointRangeQueryNode + */ +public class PointRangeQueryNodeBuilder implements StandardQueryBuilder { + + /** + * Constructs a {@link PointRangeQueryNodeBuilder} object. + */ + public PointRangeQueryNodeBuilder() { + // empty constructor + } + + @Override + public Query build(QueryNode queryNode) throws QueryNodeException { + PointRangeQueryNode numericRangeNode = (PointRangeQueryNode) queryNode; + + PointQueryNode lowerNumericNode = numericRangeNode.getLowerBound(); + PointQueryNode upperNumericNode = numericRangeNode.getUpperBound(); + + Number lowerNumber = lowerNumericNode.getValue(); + Number upperNumber = upperNumericNode.getValue(); + + PointsConfig pointsConfig = numericRangeNode.getPointsConfig(); + Class numberType = pointsConfig.getType(); + String field = StringUtils.toString(numericRangeNode.getField()); + boolean minInclusive = numericRangeNode.isLowerInclusive(); + boolean maxInclusive = numericRangeNode.isUpperInclusive(); + + // TODO: push down cleaning up of crazy nulls and inclusive/exclusive elsewhere + if (Integer.class.equals(numberType)) { + Integer lower = (Integer) lowerNumber; + if (lower == null) { + lower = Integer.MIN_VALUE; + } + if (minInclusive == false) { + lower = lower + 1; + } + + Integer upper = (Integer) upperNumber; + if (upper == null) { + upper = Integer.MAX_VALUE; + } + if (maxInclusive == false) { + upper = upper - 1; + } + return IntPoint.newRangeQuery(field, lower, upper); + } else if (Long.class.equals(numberType)) { + Long lower = (Long) lowerNumber; + if (lower == null) { + lower = Long.MIN_VALUE; + } + if (minInclusive == false) { + lower = lower + 1; + } + + Long upper = (Long) upperNumber; + if (upper == null) { + upper = Long.MAX_VALUE; + } + if (maxInclusive == false) { + upper = upper - 1; + } + return LongPoint.newRangeQuery(field, lower, upper); + } else if (Float.class.equals(numberType)) { + Float lower = (Float) lowerNumber; + if (lower == null) { + lower = Float.NEGATIVE_INFINITY; + } + if (minInclusive == false) { + lower = Math.nextUp(lower); + } + + Float upper = (Float) upperNumber; + if (upper == null) { + upper = Float.POSITIVE_INFINITY; + } + if (maxInclusive == false) { + upper = Math.nextDown(upper); + } + return FloatPoint.newRangeQuery(field, lower, upper); + } else if (Double.class.equals(numberType)) { + Double lower = (Double) lowerNumber; + if (lower == null) { + lower = Double.NEGATIVE_INFINITY; + } + if (minInclusive == false) { + lower = Math.nextUp(lower); + } + + Double upper = (Double) upperNumber; + if (upper == null) { + upper = Double.POSITIVE_INFINITY; + } + if (maxInclusive == false) { + upper = Math.nextDown(upper); + } + return DoublePoint.newRangeQuery(field, lower, upper); + } else { + throw new QueryNodeException(new MessageImpl(QueryParserMessages.UNSUPPORTED_NUMERIC_DATA_TYPE, numberType)); + } + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java index 2d7c6434be86..360f6a7ff838 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardQueryTreeBuilder.java @@ -30,8 +30,10 @@ import org.apache.lucene.queryparser.flexible.core.nodes.SlopQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.TokenizedPhraseQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.MultiPhraseQueryNode; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericQueryNode; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.PrefixWildcardQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode; @@ -57,8 +59,10 @@ public StandardQueryTreeBuilder() { setBuilder(FieldQueryNode.class, new FieldQueryNodeBuilder()); setBuilder(BooleanQueryNode.class, new BooleanQueryNodeBuilder()); setBuilder(FuzzyQueryNode.class, new FuzzyQueryNodeBuilder()); - setBuilder(NumericQueryNode.class, new DummyQueryNodeBuilder()); - setBuilder(NumericRangeQueryNode.class, new NumericRangeQueryNodeBuilder()); + setBuilder(LegacyNumericQueryNode.class, new DummyQueryNodeBuilder()); + setBuilder(LegacyNumericRangeQueryNode.class, new LegacyNumericRangeQueryNodeBuilder()); + setBuilder(PointQueryNode.class, new DummyQueryNodeBuilder()); + setBuilder(PointRangeQueryNode.class, new PointRangeQueryNodeBuilder()); setBuilder(BoostQueryNode.class, new BoostQueryNodeBuilder()); setBuilder(ModifierQueryNode.class, new ModifierQueryNodeBuilder()); setBuilder(WildcardQueryNode.class, new WildcardQueryNodeBuilder()); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java similarity index 90% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java index c457a4ebc46b..985f55a616fd 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java @@ -28,8 +28,10 @@ * * @see org.apache.lucene.search.LegacyNumericRangeQuery * @see NumberFormat + * @deprecated Index with Points instead and use {@link PointsConfig} */ -public class NumericConfig { +@Deprecated +public class LegacyNumericConfig { private int precisionStep; @@ -38,7 +40,7 @@ public class NumericConfig { private FieldType.LegacyNumericType type; /** - * Constructs a {@link NumericConfig} object. + * Constructs a {@link LegacyNumericConfig} object. * * @param precisionStep * the precision used to index the numeric values @@ -48,11 +50,11 @@ public class NumericConfig { * @param type * the numeric type used to index the numeric values * - * @see NumericConfig#setPrecisionStep(int) - * @see NumericConfig#setNumberFormat(NumberFormat) + * @see LegacyNumericConfig#setPrecisionStep(int) + * @see LegacyNumericConfig#setNumberFormat(NumberFormat) * @see #setType(org.apache.lucene.document.FieldType.LegacyNumericType) */ - public NumericConfig(int precisionStep, NumberFormat format, + public LegacyNumericConfig(int precisionStep, NumberFormat format, LegacyNumericType type) { setPrecisionStep(precisionStep); setNumberFormat(format); @@ -141,8 +143,8 @@ public boolean equals(Object obj) { if (obj == this) return true; - if (obj instanceof NumericConfig) { - NumericConfig other = (NumericConfig) obj; + if (obj instanceof LegacyNumericConfig) { + LegacyNumericConfig other = (LegacyNumericConfig) obj; if (this.precisionStep == other.precisionStep && this.type == other.type diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java similarity index 67% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java index c28cf2c92dff..f2d312489f61 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/NumericFieldConfigListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java @@ -25,25 +25,27 @@ /** * This listener is used to listen to {@link FieldConfig} requests in - * {@link QueryConfigHandler} and add {@link ConfigurationKeys#NUMERIC_CONFIG} - * based on the {@link ConfigurationKeys#NUMERIC_CONFIG_MAP} set in the + * {@link QueryConfigHandler} and add {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG} + * based on the {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG_MAP} set in the * {@link QueryConfigHandler}. * - * @see NumericConfig + * @see LegacyNumericConfig * @see QueryConfigHandler - * @see ConfigurationKeys#NUMERIC_CONFIG - * @see ConfigurationKeys#NUMERIC_CONFIG_MAP + * @see ConfigurationKeys#LEGACY_NUMERIC_CONFIG + * @see ConfigurationKeys#LEGACY_NUMERIC_CONFIG_MAP + * @deprecated Index with Points instead and use {@link PointsConfigListener} */ -public class NumericFieldConfigListener implements FieldConfigListener { +@Deprecated +public class LegacyNumericFieldConfigListener implements FieldConfigListener { final private QueryConfigHandler config; /** - * Construcs a {@link NumericFieldConfigListener} object using the given {@link QueryConfigHandler}. + * Constructs a {@link LegacyNumericFieldConfigListener} object using the given {@link QueryConfigHandler}. * * @param config the {@link QueryConfigHandler} it will listen too */ - public NumericFieldConfigListener(QueryConfigHandler config) { + public LegacyNumericFieldConfigListener(QueryConfigHandler config) { if (config == null) { throw new IllegalArgumentException("config cannot be null!"); @@ -55,15 +57,15 @@ public NumericFieldConfigListener(QueryConfigHandler config) { @Override public void buildFieldConfig(FieldConfig fieldConfig) { - Map numericConfigMap = config - .get(ConfigurationKeys.NUMERIC_CONFIG_MAP); + Map numericConfigMap = config + .get(ConfigurationKeys.LEGACY_NUMERIC_CONFIG_MAP); if (numericConfigMap != null) { - NumericConfig numericConfig = numericConfigMap + LegacyNumericConfig numericConfig = numericConfigMap .get(fieldConfig.getField()); if (numericConfig != null) { - fieldConfig.set(ConfigurationKeys.NUMERIC_CONFIG, numericConfig); + fieldConfig.set(ConfigurationKeys.LEGACY_NUMERIC_CONFIG, numericConfig); } } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java new file mode 100644 index 000000000000..db59b48123a4 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.config; + +import java.text.NumberFormat; + +import org.apache.lucene.index.PointValues; + +/** + * This class holds the configuration used to parse numeric queries and create + * {@link PointValues} queries. + * + * @see PointValues + * @see NumberFormat + */ +public class PointsConfig { + + private NumberFormat format; + + private Class type; + + /** + * Constructs a {@link PointsConfig} object. + * + * @param format + * the {@link NumberFormat} used to parse a {@link String} to + * {@link Number} + * @param type + * the numeric type used to index the numeric values + * + * @see PointsConfig#setNumberFormat(NumberFormat) + */ + public PointsConfig(NumberFormat format, Class type) { + setNumberFormat(format); + setType(type); + } + + /** + * Returns the {@link NumberFormat} used to parse a {@link String} to + * {@link Number} + * + * @return the {@link NumberFormat} used to parse a {@link String} to + * {@link Number} + */ + public NumberFormat getNumberFormat() { + return format; + } + + /** + * Returns the numeric type used to index the numeric values + * + * @return the numeric type used to index the numeric values + */ + public Class getType() { + return type; + } + + /** + * Sets the numeric type used to index the numeric values + * + * @param type the numeric type used to index the numeric values + */ + public void setType(Class type) { + if (type == null) { + throw new IllegalArgumentException("type cannot be null!"); + } + if (Integer.class.equals(type) == false && + Long.class.equals(type) == false && + Float.class.equals(type) == false && + Double.class.equals(type) == false) { + throw new IllegalArgumentException("unsupported numeric type: " + type); + } + this.type = type; + } + + /** + * Sets the {@link NumberFormat} used to parse a {@link String} to + * {@link Number} + * + * @param format + * the {@link NumberFormat} used to parse a {@link String} to + * {@link Number}, cannot be null + */ + public void setNumberFormat(NumberFormat format) { + if (format == null) { + throw new IllegalArgumentException("format cannot be null!"); + } + this.format = format; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + format.hashCode(); + result = prime * result + type.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + PointsConfig other = (PointsConfig) obj; + if (!format.equals(other.format)) return false; + if (!type.equals(other.type)) return false; + return true; + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java new file mode 100644 index 000000000000..9efbbb7e5a01 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.config; + +import java.util.Map; + +import org.apache.lucene.queryparser.flexible.core.config.FieldConfig; +import org.apache.lucene.queryparser.flexible.core.config.FieldConfigListener; +import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler; +import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; + +/** + * This listener is used to listen to {@link FieldConfig} requests in + * {@link QueryConfigHandler} and add {@link ConfigurationKeys#POINTS_CONFIG} + * based on the {@link ConfigurationKeys#POINTS_CONFIG_MAP} set in the + * {@link QueryConfigHandler}. + * + * @see PointsConfig + * @see QueryConfigHandler + * @see ConfigurationKeys#POINTS_CONFIG + * @see ConfigurationKeys#POINTS_CONFIG_MAP + */ +public class PointsConfigListener implements FieldConfigListener { + + final private QueryConfigHandler config; + + /** + * Constructs a {@link PointsConfigListener} object using the given {@link QueryConfigHandler}. + * + * @param config the {@link QueryConfigHandler} it will listen too + */ + public PointsConfigListener(QueryConfigHandler config) { + if (config == null) { + throw new IllegalArgumentException("config cannot be null!"); + } + this.config = config; + } + + @Override + public void buildFieldConfig(FieldConfig fieldConfig) { + Map pointsConfigMap = config.get(ConfigurationKeys.POINTS_CONFIG_MAP); + + if (pointsConfigMap != null) { + PointsConfig pointsConfig = pointsConfigMap.get(fieldConfig.getField()); + + if (pointsConfig != null) { + fieldConfig.set(ConfigurationKeys.POINTS_CONFIG, pointsConfig); + } + } + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java index 77bd7bb639cb..bba95eed91f8 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/StandardQueryConfigHandler.java @@ -167,21 +167,41 @@ final public static class ConfigurationKeys { final public static ConfigurationKey BOOST = ConfigurationKey.newInstance(); /** - * Key used to set a field to its {@link NumericConfig}. + * Key used to set a field to its {@link LegacyNumericConfig}. * - * @see StandardQueryParser#setNumericConfigMap(Map) - * @see StandardQueryParser#getNumericConfigMap() + * @see StandardQueryParser#setLegacyNumericConfigMap(Map) + * @see StandardQueryParser#getLegacyNumericConfigMap() + * @deprecated Index with Points instead and use {@link #POINTS_CONFIG} */ - final public static ConfigurationKey NUMERIC_CONFIG = ConfigurationKey.newInstance(); + @Deprecated + final public static ConfigurationKey LEGACY_NUMERIC_CONFIG = ConfigurationKey.newInstance(); /** - * Key used to set the {@link NumericConfig} in {@link FieldConfig} for numeric fields. + * Key used to set the {@link LegacyNumericConfig} in {@link FieldConfig} for numeric fields. * - * @see StandardQueryParser#setNumericConfigMap(Map) - * @see StandardQueryParser#getNumericConfigMap() + * @see StandardQueryParser#setLegacyNumericConfigMap(Map) + * @see StandardQueryParser#getLegacyNumericConfigMap() + * @deprecated Index with Points instead and use {@link #POINTS_CONFIG_MAP} */ - final public static ConfigurationKey> NUMERIC_CONFIG_MAP = ConfigurationKey.newInstance(); + @Deprecated + final public static ConfigurationKey> LEGACY_NUMERIC_CONFIG_MAP = ConfigurationKey.newInstance(); + /** + * Key used to set a field to its {@link PointsConfig}. + * + * @see StandardQueryParser#setLegacyNumericConfigMap(Map) + * @see StandardQueryParser#getLegacyNumericConfigMap() + */ + final public static ConfigurationKey POINTS_CONFIG = ConfigurationKey.newInstance(); + + /** + * Key used to set the {@link PointsConfig} in {@link FieldConfig} for point fields. + * + * @see StandardQueryParser#setLegacyNumericConfigMap(Map) + * @see StandardQueryParser#getLegacyNumericConfigMap() + */ + final public static ConfigurationKey> POINTS_CONFIG_MAP = ConfigurationKey.newInstance(); + } /** @@ -195,7 +215,8 @@ public StandardQueryConfigHandler() { // Add listener that will build the FieldConfig. addFieldConfigListener(new FieldBoostMapFCListener(this)); addFieldConfigListener(new FieldDateResolutionFCListener(this)); - addFieldConfigListener(new NumericFieldConfigListener(this)); + addFieldConfigListener(new LegacyNumericFieldConfigListener(this)); + addFieldConfigListener(new PointsConfigListener(this)); // Default Values set(ConfigurationKeys.ALLOW_LEADING_WILDCARD, false); // default in 2.9 diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericQueryNode.java new file mode 100644 index 000000000000..b644d8a65f24 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericQueryNode.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.nodes; + +import java.text.NumberFormat; +import java.util.Locale; + +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldValuePairQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNodeImpl; +import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; +import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax.Type; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; + +/** + * This query node represents a field query that holds a numeric value. It is + * similar to {@link FieldQueryNode}, however the {@link #getValue()} returns a + * {@link Number}. + * + * @see LegacyNumericConfig + * @deprecated Index with Points instead and use {@link PointQueryNode} instead. + */ +@Deprecated +public class LegacyNumericQueryNode extends QueryNodeImpl implements + FieldValuePairQueryNode { + + private NumberFormat numberFormat; + + private CharSequence field; + + private Number value; + + /** + * Creates a {@link LegacyNumericQueryNode} object using the given field, + * {@link Number} value and {@link NumberFormat} used to convert the value to + * {@link String}. + * + * @param field the field associated with this query node + * @param value the value hold by this node + * @param numberFormat the {@link NumberFormat} used to convert the value to {@link String} + */ + public LegacyNumericQueryNode(CharSequence field, Number value, + NumberFormat numberFormat) { + + super(); + + setNumberFormat(numberFormat); + setField(field); + setValue(value); + + } + + /** + * Returns the field associated with this node. + * + * @return the field associated with this node + */ + @Override + public CharSequence getField() { + return this.field; + } + + /** + * Sets the field associated with this node. + * + * @param fieldName the field associated with this node + */ + @Override + public void setField(CharSequence fieldName) { + this.field = fieldName; + } + + /** + * This method is used to get the value converted to {@link String} and + * escaped using the given {@link EscapeQuerySyntax}. + * + * @param escaper the {@link EscapeQuerySyntax} used to escape the value {@link String} + * + * @return the value converte to {@link String} and escaped + */ + protected CharSequence getTermEscaped(EscapeQuerySyntax escaper) { + return escaper.escape(numberFormat.format(this.value), + Locale.ROOT, Type.NORMAL); + } + + @Override + public CharSequence toQueryString(EscapeQuerySyntax escapeSyntaxParser) { + if (isDefaultField(this.field)) { + return getTermEscaped(escapeSyntaxParser); + } else { + return this.field + ":" + getTermEscaped(escapeSyntaxParser); + } + } + + /** + * Sets the {@link NumberFormat} used to convert the value to {@link String}. + * + * @param format the {@link NumberFormat} used to convert the value to {@link String} + */ + public void setNumberFormat(NumberFormat format) { + this.numberFormat = format; + } + + /** + * Returns the {@link NumberFormat} used to convert the value to {@link String}. + * + * @return the {@link NumberFormat} used to convert the value to {@link String} + */ + public NumberFormat getNumberFormat() { + return this.numberFormat; + } + + /** + * Returns the numeric value as {@link Number}. + * + * @return the numeric value + */ + @Override + public Number getValue() { + return value; + } + + /** + * Sets the numeric value. + * + * @param value the numeric value + */ + @Override + public void setValue(Number value) { + this.value = value; + } + + @Override + public String toString() { + return ""; + } + +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java similarity index 73% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java index c132aa1dd0d1..088ab98e175a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java @@ -21,34 +21,36 @@ import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.messages.MessageImpl; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; /** - * This query node represents a range query composed by {@link NumericQueryNode} + * This query node represents a range query composed by {@link LegacyNumericQueryNode} * bounds, which means the bound values are {@link Number}s. * - * @see NumericQueryNode + * @see LegacyNumericQueryNode * @see AbstractRangeQueryNode + * @deprecated Index with Points instead and use {@link PointRangeQueryNode} instead. */ -public class NumericRangeQueryNode extends - AbstractRangeQueryNode { +@Deprecated +public class LegacyNumericRangeQueryNode extends + AbstractRangeQueryNode { - public NumericConfig numericConfig; + public LegacyNumericConfig numericConfig; /** - * Constructs a {@link NumericRangeQueryNode} object using the given - * {@link NumericQueryNode} as its bounds and {@link NumericConfig}. + * Constructs a {@link LegacyNumericRangeQueryNode} object using the given + * {@link LegacyNumericQueryNode} as its bounds and {@link LegacyNumericConfig}. * * @param lower the lower bound * @param upper the upper bound * @param lowerInclusive true if the lower bound is inclusive, otherwise, false * @param upperInclusive true if the upper bound is inclusive, otherwise, false - * @param numericConfig the {@link NumericConfig} that represents associated with the upper and lower bounds + * @param numericConfig the {@link LegacyNumericConfig} that represents associated with the upper and lower bounds * - * @see #setBounds(NumericQueryNode, NumericQueryNode, boolean, boolean, NumericConfig) + * @see #setBounds(LegacyNumericQueryNode, LegacyNumericQueryNode, boolean, boolean, LegacyNumericConfig) */ - public NumericRangeQueryNode(NumericQueryNode lower, NumericQueryNode upper, - boolean lowerInclusive, boolean upperInclusive, NumericConfig numericConfig) throws QueryNodeException { + public LegacyNumericRangeQueryNode(LegacyNumericQueryNode lower, LegacyNumericQueryNode upper, + boolean lowerInclusive, boolean upperInclusive, LegacyNumericConfig numericConfig) throws QueryNodeException { setBounds(lower, upper, lowerInclusive, upperInclusive, numericConfig); } @@ -73,17 +75,17 @@ private static LegacyNumericType getNumericDataType(Number number) throws QueryN /** * Sets the upper and lower bounds of this range query node and the - * {@link NumericConfig} associated with these bounds. + * {@link LegacyNumericConfig} associated with these bounds. * * @param lower the lower bound * @param upper the upper bound * @param lowerInclusive true if the lower bound is inclusive, otherwise, false * @param upperInclusive true if the upper bound is inclusive, otherwise, false - * @param numericConfig the {@link NumericConfig} that represents associated with the upper and lower bounds + * @param numericConfig the {@link LegacyNumericConfig} that represents associated with the upper and lower bounds * */ - public void setBounds(NumericQueryNode lower, NumericQueryNode upper, - boolean lowerInclusive, boolean upperInclusive, NumericConfig numericConfig) throws QueryNodeException { + public void setBounds(LegacyNumericQueryNode lower, LegacyNumericQueryNode upper, + boolean lowerInclusive, boolean upperInclusive, LegacyNumericConfig numericConfig) throws QueryNodeException { if (numericConfig == null) { throw new IllegalArgumentException("numericConfig cannot be null!"); @@ -123,11 +125,11 @@ public void setBounds(NumericQueryNode lower, NumericQueryNode upper, } /** - * Returns the {@link NumericConfig} associated with the lower and upper bounds. + * Returns the {@link LegacyNumericConfig} associated with the lower and upper bounds. * - * @return the {@link NumericConfig} associated with the lower and upper bounds + * @return the {@link LegacyNumericConfig} associated with the lower and upper bounds */ - public NumericConfig getNumericConfig() { + public LegacyNumericConfig getNumericConfig() { return this.numericConfig; } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointQueryNode.java similarity index 91% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointQueryNode.java index 7509a39bd71f..6d4cba778a86 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/NumericQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointQueryNode.java @@ -24,16 +24,16 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNodeImpl; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax.Type; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; /** - * This query node represents a field query that holds a numeric value. It is + * This query node represents a field query that holds a point value. It is * similar to {@link FieldQueryNode}, however the {@link #getValue()} returns a * {@link Number}. * - * @see NumericConfig + * @see PointsConfig */ -public class NumericQueryNode extends QueryNodeImpl implements +public class PointQueryNode extends QueryNodeImpl implements FieldValuePairQueryNode { private NumberFormat numberFormat; @@ -43,7 +43,7 @@ public class NumericQueryNode extends QueryNodeImpl implements private Number value; /** - * Creates a {@link NumericQueryNode} object using the given field, + * Creates a {@link PointQueryNode} object using the given field, * {@link Number} value and {@link NumberFormat} used to convert the value to * {@link String}. * @@ -51,7 +51,7 @@ public class NumericQueryNode extends QueryNodeImpl implements * @param value the value hold by this node * @param numberFormat the {@link NumberFormat} used to convert the value to {@link String} */ - public NumericQueryNode(CharSequence field, Number value, + public PointQueryNode(CharSequence field, Number value, NumberFormat numberFormat) { super(); diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java new file mode 100644 index 000000000000..cb838fc7d6cc --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.nodes; + +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +/** + * This query node represents a range query composed by {@link PointQueryNode} + * bounds, which means the bound values are {@link Number}s. + * + * @see PointQueryNode + * @see AbstractRangeQueryNode + */ +public class PointRangeQueryNode extends AbstractRangeQueryNode { + + public PointsConfig numericConfig; + + /** + * Constructs a {@link PointRangeQueryNode} object using the given + * {@link PointQueryNode} as its bounds and {@link PointsConfig}. + * + * @param lower the lower bound + * @param upper the upper bound + * @param lowerInclusive true if the lower bound is inclusive, otherwise, false + * @param upperInclusive true if the upper bound is inclusive, otherwise, false + * @param numericConfig the {@link PointsConfig} that represents associated with the upper and lower bounds + * + * @see #setBounds(PointQueryNode, PointQueryNode, boolean, boolean, PointsConfig) + */ + public PointRangeQueryNode(PointQueryNode lower, PointQueryNode upper, + boolean lowerInclusive, boolean upperInclusive, PointsConfig numericConfig) throws QueryNodeException { + setBounds(lower, upper, lowerInclusive, upperInclusive, numericConfig); + } + + /** + * Sets the upper and lower bounds of this range query node and the + * {@link PointsConfig} associated with these bounds. + * + * @param lower the lower bound + * @param upper the upper bound + * @param lowerInclusive true if the lower bound is inclusive, otherwise, false + * @param upperInclusive true if the upper bound is inclusive, otherwise, false + * @param pointsConfig the {@link PointsConfig} that represents associated with the upper and lower bounds + * + */ + public void setBounds(PointQueryNode lower, PointQueryNode upper, + boolean lowerInclusive, boolean upperInclusive, PointsConfig pointsConfig) throws QueryNodeException { + + if (pointsConfig == null) { + throw new IllegalArgumentException("pointsConfig cannot be null!"); + } + + Class lowerNumberType, upperNumberType; + + if (lower != null && lower.getValue() != null) { + lowerNumberType = lower.getValue().getClass(); + } else { + lowerNumberType = null; + } + + if (upper != null && upper.getValue() != null) { + upperNumberType = upper.getValue().getClass(); + } else { + upperNumberType = null; + } + + if (lowerNumberType != null + && !lowerNumberType.equals(pointsConfig.getType())) { + throw new IllegalArgumentException( + "lower value's type should be the same as numericConfig type: " + + lowerNumberType + " != " + pointsConfig.getType()); + } + + if (upperNumberType != null + && !upperNumberType.equals(pointsConfig.getType())) { + throw new IllegalArgumentException( + "upper value's type should be the same as numericConfig type: " + + upperNumberType + " != " + pointsConfig.getType()); + } + + super.setBounds(lower, upper, lowerInclusive, upperInclusive); + this.numericConfig = pointsConfig; + } + + /** + * Returns the {@link PointsConfig} associated with the lower and upper bounds. + * + * @return the {@link PointsConfig} associated with the lower and upper bounds + */ + public PointsConfig getPointsConfig() { + return this.numericConfig; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\n"); + sb.append(getLowerBound()).append('\n'); + sb.append(getUpperBound()).append('\n'); + sb.append(""); + return sb.toString(); + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericQueryNodeProcessor.java similarity index 78% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericQueryNodeProcessor.java index 10bd6baf8338..8b7182437ce4 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericQueryNodeProcessor.java @@ -30,19 +30,19 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.RangeQueryNode; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericQueryNode; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; /** * This processor is used to convert {@link FieldQueryNode}s to - * {@link NumericRangeQueryNode}s. It looks for - * {@link ConfigurationKeys#NUMERIC_CONFIG} set in the {@link FieldConfig} of + * {@link LegacyNumericRangeQueryNode}s. It looks for + * {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG} set in the {@link FieldConfig} of * every {@link FieldQueryNode} found. If - * {@link ConfigurationKeys#NUMERIC_CONFIG} is found, it considers that + * {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG} is found, it considers that * {@link FieldQueryNode} to be a numeric query and convert it to - * {@link NumericRangeQueryNode} with upper and lower inclusive and lower and + * {@link LegacyNumericRangeQueryNode} with upper and lower inclusive and lower and * upper equals to the value represented by the {@link FieldQueryNode} converted * to {@link Number}. It means that field:1 is converted to field:[1 * TO 1].
      @@ -50,17 +50,19 @@ * Note that {@link FieldQueryNode}s children of a * {@link RangeQueryNode} are ignored. * - * @see ConfigurationKeys#NUMERIC_CONFIG + * @see ConfigurationKeys#LEGACY_NUMERIC_CONFIG * @see FieldQueryNode - * @see NumericConfig - * @see NumericQueryNode + * @see LegacyNumericConfig + * @see LegacyNumericQueryNode + * @deprecated Index with points and use {@link PointQueryNodeProcessor} instead. */ -public class NumericQueryNodeProcessor extends QueryNodeProcessorImpl { +@Deprecated +public class LegacyNumericQueryNodeProcessor extends QueryNodeProcessorImpl { /** - * Constructs a {@link NumericQueryNodeProcessor} object. + * Constructs a {@link LegacyNumericQueryNodeProcessor} object. */ - public NumericQueryNodeProcessor() { + public LegacyNumericQueryNodeProcessor() { // empty constructor } @@ -78,8 +80,8 @@ protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { .getFieldAsString()); if (fieldConfig != null) { - NumericConfig numericConfig = fieldConfig - .get(ConfigurationKeys.NUMERIC_CONFIG); + LegacyNumericConfig numericConfig = fieldConfig + .get(ConfigurationKeys.LEGACY_NUMERIC_CONFIG); if (numericConfig != null) { @@ -118,12 +120,12 @@ protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { QueryParserMessages.NUMERIC_CANNOT_BE_EMPTY, fieldNode.getFieldAsString())); } - NumericQueryNode lowerNode = new NumericQueryNode(fieldNode + LegacyNumericQueryNode lowerNode = new LegacyNumericQueryNode(fieldNode .getField(), number, numberFormat); - NumericQueryNode upperNode = new NumericQueryNode(fieldNode + LegacyNumericQueryNode upperNode = new LegacyNumericQueryNode(fieldNode .getField(), number, numberFormat); - return new NumericRangeQueryNode(lowerNode, upperNode, true, true, + return new LegacyNumericRangeQueryNode(lowerNode, upperNode, true, true, numericConfig); } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericRangeQueryNodeProcessor.java similarity index 81% rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericRangeQueryNodeProcessor.java index bbe528452992..5a54b7b42056 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/NumericRangeQueryNodeProcessor.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/LegacyNumericRangeQueryNodeProcessor.java @@ -30,32 +30,34 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; import org.apache.lucene.queryparser.flexible.core.util.StringUtils; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericQueryNode; -import org.apache.lucene.queryparser.flexible.standard.nodes.NumericRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.LegacyNumericRangeQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode; /** * This processor is used to convert {@link TermRangeQueryNode}s to - * {@link NumericRangeQueryNode}s. It looks for - * {@link ConfigurationKeys#NUMERIC_CONFIG} set in the {@link FieldConfig} of + * {@link LegacyNumericRangeQueryNode}s. It looks for + * {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG} set in the {@link FieldConfig} of * every {@link TermRangeQueryNode} found. If - * {@link ConfigurationKeys#NUMERIC_CONFIG} is found, it considers that + * {@link ConfigurationKeys#LEGACY_NUMERIC_CONFIG} is found, it considers that * {@link TermRangeQueryNode} to be a numeric range query and convert it to - * {@link NumericRangeQueryNode}. + * {@link LegacyNumericRangeQueryNode}. * - * @see ConfigurationKeys#NUMERIC_CONFIG + * @see ConfigurationKeys#LEGACY_NUMERIC_CONFIG * @see TermRangeQueryNode - * @see NumericConfig - * @see NumericRangeQueryNode + * @see LegacyNumericConfig + * @see LegacyNumericRangeQueryNode + * @deprecated Index with points and use {@link PointRangeQueryNodeProcessor} instead. */ -public class NumericRangeQueryNodeProcessor extends QueryNodeProcessorImpl { +@Deprecated +public class LegacyNumericRangeQueryNodeProcessor extends QueryNodeProcessorImpl { /** - * Constructs an empty {@link NumericRangeQueryNode} object. + * Constructs an empty {@link LegacyNumericRangeQueryNode} object. */ - public NumericRangeQueryNodeProcessor() { + public LegacyNumericRangeQueryNodeProcessor() { // empty constructor } @@ -72,8 +74,8 @@ protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { if (fieldConfig != null) { - NumericConfig numericConfig = fieldConfig - .get(ConfigurationKeys.NUMERIC_CONFIG); + LegacyNumericConfig numericConfig = fieldConfig + .get(ConfigurationKeys.LEGACY_NUMERIC_CONFIG); if (numericConfig != null) { @@ -131,15 +133,15 @@ protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { if (lowerNumber != null) lowerNumber = lowerNumber.floatValue(); } - NumericQueryNode lowerNode = new NumericQueryNode( + LegacyNumericQueryNode lowerNode = new LegacyNumericQueryNode( termRangeNode.getField(), lowerNumber, numberFormat); - NumericQueryNode upperNode = new NumericQueryNode( + LegacyNumericQueryNode upperNode = new LegacyNumericQueryNode( termRangeNode.getField(), upperNumber, numberFormat); boolean lowerInclusive = termRangeNode.isLowerInclusive(); boolean upperInclusive = termRangeNode.isUpperInclusive(); - return new NumericRangeQueryNode(lowerNode, upperNode, + return new LegacyNumericRangeQueryNode(lowerNode, upperNode, lowerInclusive, upperInclusive, numericConfig); } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointQueryNodeProcessor.java new file mode 100644 index 000000000000..81a844961563 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointQueryNodeProcessor.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.processors; + +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.List; + +import org.apache.lucene.queryparser.flexible.messages.MessageImpl; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.config.FieldConfig; +import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler; +import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.RangeQueryNode; +import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; +import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointRangeQueryNode; + +/** + * This processor is used to convert {@link FieldQueryNode}s to + * {@link PointRangeQueryNode}s. It looks for + * {@link ConfigurationKeys#POINTS_CONFIG} set in the {@link FieldConfig} of + * every {@link FieldQueryNode} found. If + * {@link ConfigurationKeys#POINTS_CONFIG} is found, it considers that + * {@link FieldQueryNode} to be a numeric query and convert it to + * {@link PointRangeQueryNode} with upper and lower inclusive and lower and + * upper equals to the value represented by the {@link FieldQueryNode} converted + * to {@link Number}. It means that field:1 is converted to field:[1 + * TO 1].
      + *
      + * Note that {@link FieldQueryNode}s children of a + * {@link RangeQueryNode} are ignored. + * + * @see ConfigurationKeys#POINTS_CONFIG + * @see FieldQueryNode + * @see PointsConfig + * @see PointQueryNode + */ +public class PointQueryNodeProcessor extends QueryNodeProcessorImpl { + + /** + * Constructs a {@link PointQueryNodeProcessor} object. + */ + public PointQueryNodeProcessor() { + // empty constructor + } + + @Override + protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { + + if (node instanceof FieldQueryNode + && !(node.getParent() instanceof RangeQueryNode)) { + + QueryConfigHandler config = getQueryConfigHandler(); + + if (config != null) { + FieldQueryNode fieldNode = (FieldQueryNode) node; + FieldConfig fieldConfig = config.getFieldConfig(fieldNode + .getFieldAsString()); + + if (fieldConfig != null) { + PointsConfig numericConfig = fieldConfig.get(ConfigurationKeys.POINTS_CONFIG); + + if (numericConfig != null) { + + NumberFormat numberFormat = numericConfig.getNumberFormat(); + String text = fieldNode.getTextAsString(); + Number number = null; + + if (text.length() > 0) { + + try { + number = numberFormat.parse(text); + + } catch (ParseException e) { + throw new QueryNodeParseException(new MessageImpl( + QueryParserMessages.COULD_NOT_PARSE_NUMBER, fieldNode + .getTextAsString(), numberFormat.getClass() + .getCanonicalName()), e); + } + + if (Integer.class.equals(numericConfig.getType())) { + number = number.intValue(); + } else if (Long.class.equals(numericConfig.getType())) { + number = number.longValue(); + } else if (Double.class.equals(numericConfig.getType())) { + number = number.doubleValue(); + } else if (Float.class.equals(numericConfig.getType())) { + number = number.floatValue(); + } + + } else { + throw new QueryNodeParseException(new MessageImpl( + QueryParserMessages.NUMERIC_CANNOT_BE_EMPTY, fieldNode.getFieldAsString())); + } + + PointQueryNode lowerNode = new PointQueryNode(fieldNode.getField(), number, numberFormat); + PointQueryNode upperNode = new PointQueryNode(fieldNode.getField(), number, numberFormat); + + return new PointRangeQueryNode(lowerNode, upperNode, true, true, numericConfig); + } + } + } + } + return node; + } + + @Override + protected QueryNode preProcessNode(QueryNode node) throws QueryNodeException { + return node; + } + + @Override + protected List setChildrenOrder(List children) throws QueryNodeException { + return children; + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointRangeQueryNodeProcessor.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointRangeQueryNodeProcessor.java new file mode 100644 index 000000000000..2ffc43735af5 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/PointRangeQueryNodeProcessor.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard.processors; + +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.List; + +import org.apache.lucene.queryparser.flexible.messages.MessageImpl; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.config.FieldConfig; +import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler; +import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; +import org.apache.lucene.queryparser.flexible.core.util.StringUtils; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; +import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.PointRangeQueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode; + +/** + * This processor is used to convert {@link TermRangeQueryNode}s to + * {@link PointRangeQueryNode}s. It looks for + * {@link ConfigurationKeys#POINTS_CONFIG} set in the {@link FieldConfig} of + * every {@link TermRangeQueryNode} found. If + * {@link ConfigurationKeys#POINTS_CONFIG} is found, it considers that + * {@link TermRangeQueryNode} to be a numeric range query and convert it to + * {@link PointRangeQueryNode}. + * + * @see ConfigurationKeys#POINTS_CONFIG + * @see TermRangeQueryNode + * @see PointsConfig + * @see PointRangeQueryNode + */ +public class PointRangeQueryNodeProcessor extends QueryNodeProcessorImpl { + + /** + * Constructs an empty {@link PointRangeQueryNodeProcessor} object. + */ + public PointRangeQueryNodeProcessor() { + // empty constructor + } + + @Override + protected QueryNode postProcessNode(QueryNode node) throws QueryNodeException { + + if (node instanceof TermRangeQueryNode) { + QueryConfigHandler config = getQueryConfigHandler(); + + if (config != null) { + TermRangeQueryNode termRangeNode = (TermRangeQueryNode) node; + FieldConfig fieldConfig = config.getFieldConfig(StringUtils.toString(termRangeNode.getField())); + + if (fieldConfig != null) { + PointsConfig numericConfig = fieldConfig.get(ConfigurationKeys.POINTS_CONFIG); + + if (numericConfig != null) { + FieldQueryNode lower = termRangeNode.getLowerBound(); + FieldQueryNode upper = termRangeNode.getUpperBound(); + + String lowerText = lower.getTextAsString(); + String upperText = upper.getTextAsString(); + NumberFormat numberFormat = numericConfig.getNumberFormat(); + Number lowerNumber = null, upperNumber = null; + + if (lowerText.length() > 0) { + + try { + lowerNumber = numberFormat.parse(lowerText); + + } catch (ParseException e) { + throw new QueryNodeParseException(new MessageImpl( + QueryParserMessages.COULD_NOT_PARSE_NUMBER, lower + .getTextAsString(), numberFormat.getClass() + .getCanonicalName()), e); + } + + } + + if (upperText.length() > 0) { + + try { + upperNumber = numberFormat.parse(upperText); + + } catch (ParseException e) { + throw new QueryNodeParseException(new MessageImpl( + QueryParserMessages.COULD_NOT_PARSE_NUMBER, upper + .getTextAsString(), numberFormat.getClass() + .getCanonicalName()), e); + } + } + + if (Integer.class.equals(numericConfig.getType())) { + if (upperNumber != null) upperNumber = upperNumber.intValue(); + if (lowerNumber != null) lowerNumber = lowerNumber.intValue(); + } else if (Long.class.equals(numericConfig.getType())) { + if (upperNumber != null) upperNumber = upperNumber.longValue(); + if (lowerNumber != null) lowerNumber = lowerNumber.longValue(); + } else if (Double.class.equals(numericConfig.getType())) { + if (upperNumber != null) upperNumber = upperNumber.doubleValue(); + if (lowerNumber != null) lowerNumber = lowerNumber.doubleValue(); + } else if (Float.class.equals(numericConfig.getType())) { + if (upperNumber != null) upperNumber = upperNumber.floatValue(); + if (lowerNumber != null) lowerNumber = lowerNumber.floatValue(); + } + + PointQueryNode lowerNode = new PointQueryNode(termRangeNode.getField(), lowerNumber, numberFormat); + PointQueryNode upperNode = new PointQueryNode(termRangeNode.getField(), upperNumber, numberFormat); + + boolean lowerInclusive = termRangeNode.isLowerInclusive(); + boolean upperInclusive = termRangeNode.isUpperInclusive(); + + return new PointRangeQueryNode(lowerNode, upperNode, lowerInclusive, upperInclusive, numericConfig); + } + } + } + } + return node; + } + + @Override + protected QueryNode preProcessNode(QueryNode node) throws QueryNodeException { + return node; + } + + @Override + protected List setChildrenOrder(List children) throws QueryNodeException { + return children; + } +} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java index 06f38c2b2095..6e4a394f4454 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/StandardQueryNodeProcessorPipeline.java @@ -55,8 +55,10 @@ public StandardQueryNodeProcessorPipeline(QueryConfigHandler queryConfig) { add(new FuzzyQueryNodeProcessor()); add(new MatchAllDocsQueryNodeProcessor()); add(new OpenRangeQueryNodeProcessor()); - add(new NumericQueryNodeProcessor()); - add(new NumericRangeQueryNodeProcessor()); + add(new LegacyNumericQueryNodeProcessor()); + add(new LegacyNumericRangeQueryNodeProcessor()); + add(new PointQueryNodeProcessor()); + add(new PointRangeQueryNodeProcessor()); add(new LowercaseExpandedTermsQueryNodeProcessor()); add(new TermRangeQueryNodeProcessor()); add(new AllowLeadingWildcardProcessor()); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java similarity index 97% rename from lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java rename to lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java index c29573b89a6b..c6ab7f5ffffa 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestLegacyNumericQueryParser.java @@ -44,7 +44,7 @@ import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax; import org.apache.lucene.queryparser.flexible.standard.config.NumberDateFormat; -import org.apache.lucene.queryparser.flexible.standard.config.NumericConfig; +import org.apache.lucene.queryparser.flexible.standard.config.LegacyNumericConfig; import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -56,7 +56,7 @@ import org.junit.BeforeClass; import org.junit.Test; -public class TestNumericQueryParser extends LuceneTestCase { +public class TestLegacyNumericQueryParser extends LuceneTestCase { private static enum NumberType { NEGATIVE, ZERO, POSITIVE; @@ -193,12 +193,12 @@ public static void beforeClass() throws Exception { .setMergePolicy(newLogMergePolicy())); Document doc = new Document(); - HashMap numericConfigMap = new HashMap<>(); + HashMap numericConfigMap = new HashMap<>(); HashMap numericFieldMap = new HashMap<>(); - qp.setNumericConfigMap(numericConfigMap); + qp.setLegacyNumericConfigMap(numericConfigMap); for (LegacyNumericType type : LegacyNumericType.values()) { - numericConfigMap.put(type.name(), new NumericConfig(PRECISION_STEP, + numericConfigMap.put(type.name(), new LegacyNumericConfig(PRECISION_STEP, NUMBER_FORMAT, type)); FieldType ft = new FieldType(LegacyIntField.TYPE_NOT_STORED); @@ -229,7 +229,7 @@ public static void beforeClass() throws Exception { doc.add(field); } - numericConfigMap.put(DATE_FIELD_NAME, new NumericConfig(PRECISION_STEP, + numericConfigMap.put(DATE_FIELD_NAME, new LegacyNumericConfig(PRECISION_STEP, DATE_FORMAT, LegacyNumericType.LONG)); FieldType ft = new FieldType(LegacyLongField.TYPE_NOT_STORED); ft.setStored(true); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java new file mode 100644 index 000000000000..323b0ffba9fd --- /dev/null +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queryparser.flexible.standard; + +import java.text.NumberFormat; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; +import org.apache.lucene.util.LuceneTestCase; + +/** Simple test for point field integration into the flexible QP */ +public class TestPointQueryParser extends LuceneTestCase { + + public void testIntegers() throws Exception { + StandardQueryParser parser = new StandardQueryParser(); + Map pointsConfig = new HashMap<>(); + pointsConfig.put("intField", new PointsConfig(NumberFormat.getIntegerInstance(Locale.ROOT), Integer.class)); + parser.setPointsConfigMap(pointsConfig); + + assertEquals(IntPoint.newRangeQuery("intField", 1, 3), + parser.parse("intField:[1 TO 3]", "body")); + assertEquals(IntPoint.newRangeQuery("intField", 1, 1), + parser.parse("intField:1", "body")); + } + + public void testLongs() throws Exception { + StandardQueryParser parser = new StandardQueryParser(); + Map pointsConfig = new HashMap<>(); + pointsConfig.put("longField", new PointsConfig(NumberFormat.getIntegerInstance(Locale.ROOT), Long.class)); + parser.setPointsConfigMap(pointsConfig); + + assertEquals(LongPoint.newRangeQuery("longField", 1, 3), + parser.parse("longField:[1 TO 3]", "body")); + assertEquals(LongPoint.newRangeQuery("longField", 1, 1), + parser.parse("longField:1", "body")); + } + + public void testFloats() throws Exception { + StandardQueryParser parser = new StandardQueryParser(); + Map pointsConfig = new HashMap<>(); + pointsConfig.put("floatField", new PointsConfig(NumberFormat.getNumberInstance(Locale.ROOT), Float.class)); + parser.setPointsConfigMap(pointsConfig); + + assertEquals(FloatPoint.newRangeQuery("floatField", 1.5F, 3.6F), + parser.parse("floatField:[1.5 TO 3.6]", "body")); + assertEquals(FloatPoint.newRangeQuery("floatField", 1.5F, 1.5F), + parser.parse("floatField:1.5", "body")); + } + + public void testDoubles() throws Exception { + StandardQueryParser parser = new StandardQueryParser(); + Map pointsConfig = new HashMap<>(); + pointsConfig.put("doubleField", new PointsConfig(NumberFormat.getNumberInstance(Locale.ROOT), Double.class)); + parser.setPointsConfigMap(pointsConfig); + + assertEquals(DoublePoint.newRangeQuery("doubleField", 1.5D, 3.6D), + parser.parse("doubleField:[1.5 TO 3.6]", "body")); + assertEquals(DoublePoint.newRangeQuery("floatField", 1.5D, 1.5D), + parser.parse("doubleField:1.5", "body")); + } + +} diff --git a/lucene/tools/junit4/cached-timehints.txt b/lucene/tools/junit4/cached-timehints.txt index 9c8b22a390fb..f2b8974a600a 100644 --- a/lucene/tools/junit4/cached-timehints.txt +++ b/lucene/tools/junit4/cached-timehints.txt @@ -548,7 +548,7 @@ org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParser=711,339,113,55, org.apache.lucene.queryparser.flexible.spans.TestSpanQueryParserSimpleSample=51,644,92,32,129,60,21 org.apache.lucene.queryparser.flexible.standard.TestMultiAnalyzerQPHelper=44,84,87,34,260,35,229 org.apache.lucene.queryparser.flexible.standard.TestMultiFieldQPHelper=1152,1779,1888,384,179,1665,445 -org.apache.lucene.queryparser.flexible.standard.TestNumericQueryParser=344,496,451,1373,733,1420,367 +org.apache.lucene.queryparser.flexible.standard.TestLegacyNumericQueryParser=344,496,451,1373,733,1420,367 org.apache.lucene.queryparser.flexible.standard.TestQPHelper=287,676,989,2137,860,586,612 org.apache.lucene.queryparser.flexible.standard.TestStandardQP=1643,430,2182,2193,600,1506,741 org.apache.lucene.queryparser.surround.query.SrndQueryTest=1062,92,92,212,65,95,113 From 7b03e6e02a92878c540ec754fa504dd75fc631a9 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 10 Mar 2016 07:25:48 -0500 Subject: [PATCH 0090/1113] LUCENE-7081: prefix-compress compressible fixed-width data (like InetAddress/BigInteger) --- .../lucene54/Lucene54DocValuesConsumer.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java index 858c54b362f9..96acfd25b5ac 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene54/Lucene54DocValuesConsumer.java @@ -411,17 +411,32 @@ public void addBinaryField(FieldInfo field, Iterable values) throws IO /** expert: writes a value dictionary for a sorted/sortedset field */ private void addTermsDict(FieldInfo field, final Iterable values) throws IOException { - // first check if it's a "fixed-length" terms dict + // first check if it's a "fixed-length" terms dict, and compressibility if so int minLength = Integer.MAX_VALUE; int maxLength = Integer.MIN_VALUE; long numValues = 0; + BytesRefBuilder previousValue = new BytesRefBuilder(); + long prefixSum = 0; // only valid for fixed-width data, as we have a choice there for (BytesRef v : values) { minLength = Math.min(minLength, v.length); maxLength = Math.max(maxLength, v.length); + if (minLength == maxLength) { + int termPosition = (int) (numValues & INTERVAL_MASK); + if (termPosition == 0) { + // first term in block, save it away to compare against the last term later + previousValue.copyBytes(v); + } else if (termPosition == INTERVAL_COUNT - 1) { + // last term in block, accumulate shared prefix against first term + prefixSum += StringHelper.bytesDifference(previousValue.get(), v); + } + } numValues++; } - if (minLength == maxLength) { - // no index needed: direct addressing by mult + // for fixed width data, look at the avg(shared prefix) before deciding how to encode: + // prefix compression "costs" worst case 2 bytes per term because we must store suffix lengths. + // so if we share at least 3 bytes on average, always compress. + if (minLength == maxLength && prefixSum <= 3*(numValues >> INTERVAL_SHIFT)) { + // no index needed: not very compressible, direct addressing by mult addBinaryField(field, values); } else if (numValues < REVERSE_INTERVAL_COUNT) { // low cardinality: waste a few KB of ram, but can't really use fancy index etc From 02b0dd52ec115c942f95124ccc2ec1e2a461ccd9 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Thu, 10 Mar 2016 18:16:43 +0530 Subject: [PATCH 0091/1113] Fix javadocs for ClusterState.getCollectionOrNull (cherry picked from commit 66cd070) --- .../src/java/org/apache/solr/common/cloud/ClusterState.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java index ff0e6a3965f9..2495c41061a3 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java @@ -180,9 +180,9 @@ public CollectionRef getCollectionRef(String coll) { * if such a collection exists. Returns null otherwise. * * Implementation note: This method resolves the collection reference by calling - * {@link CollectionRef#get()} which can make a call to ZooKeeper. This is necessary + * {@link CollectionRef#get()} which may make a call to ZooKeeper. This is necessary * because the semantics of how collection list is loaded have changed in SOLR-6629. - * Please javadocs in {@link ZkStateReader#refreshCollectionList(Watcher)} + * Please see javadocs in {@link ZkStateReader#refreshCollectionList(Watcher)} */ public DocCollection getCollectionOrNull(String collectionName) { CollectionRef ref = collectionStates.get(collectionName); From 73db4cab6a7f4948e803f649759781c84fbb3e3f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 10 Mar 2016 09:12:38 +0100 Subject: [PATCH 0092/1113] LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' as document instead of 'Document' --- lucene/CHANGES.txt | 5 +++++ .../java/org/apache/lucene/index/memory/MemoryIndex.java | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3abd4fb5e80a..832f81971246 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -11,6 +11,11 @@ Optimizations * LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% speedup on merging 2D LatLonPoint values (Mike McCandless) +Other + +* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' + as document instead of 'Document'. (Martijn van Groningen) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 849cd6338685..9e01182ec902 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -265,7 +265,7 @@ public void addField(String fieldName, String text, Analyzer analyzer) { * @param analyzer the analyzer to use * @return a MemoryIndex */ - public static MemoryIndex fromDocument(Document document, Analyzer analyzer) { + public static MemoryIndex fromDocument(Iterable document, Analyzer analyzer) { return fromDocument(document, analyzer, false, false, 0); } @@ -277,7 +277,7 @@ public static MemoryIndex fromDocument(Document document, Analyzer analyzer) { * @param storePayloads true if payloads should be stored * @return a MemoryIndex */ - public static MemoryIndex fromDocument(Document document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads) { + public static MemoryIndex fromDocument(Iterable document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads) { return fromDocument(document, analyzer, storeOffsets, storePayloads, 0); } @@ -290,7 +290,7 @@ public static MemoryIndex fromDocument(Document document, Analyzer analyzer, boo * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after {@link #reset()} is called * @return a MemoryIndex */ - public static MemoryIndex fromDocument(Document document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads, long maxReusedBytes) { + public static MemoryIndex fromDocument(Iterable document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads, long maxReusedBytes) { MemoryIndex mi = new MemoryIndex(storeOffsets, storePayloads, maxReusedBytes); for (IndexableField field : document) { mi.addField(field, analyzer); From a3f72fcdec2188900bb511a710d4714d5053ea40 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 10 Mar 2016 08:28:51 -0500 Subject: [PATCH 0093/1113] LUCENE-7090, LUCENE-7075: deprecate single-valued LegacyNumerics fieldcaching, provide Points-based replacement. --- .../apache/lucene/uninverting/FieldCache.java | 113 ++- .../lucene/uninverting/FieldCacheImpl.java | 164 ++++- .../lucene/uninverting/UninvertingReader.java | 97 ++- .../lucene/uninverting/TestDocTermOrds.java | 2 +- .../lucene/uninverting/TestFieldCache.java | 141 ++-- .../uninverting/TestFieldCacheReopen.java | 9 +- .../TestFieldCacheSanityChecker.java | 10 +- .../uninverting/TestFieldCacheSort.java | 684 ++++++++++++++++-- .../uninverting/TestFieldCacheSortRandom.java | 8 +- .../TestFieldCacheVsDocValues.java | 4 +- .../TestFieldCacheWithThreads.java | 11 +- .../uninverting/TestLegacyFieldCache.java | 498 +++++++++++++ .../uninverting/TestNumericTerms32.java | 6 +- .../uninverting/TestNumericTerms64.java | 8 +- .../uninverting/TestUninvertingReader.java | 6 +- .../lucene/spatial/SpatialTestCase.java | 4 +- .../org/apache/solr/schema/EnumField.java | 2 +- .../org/apache/solr/schema/TrieField.java | 8 +- 18 files changed, 1575 insertions(+), 200 deletions(-) create mode 100644 lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java diff --git a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java index 314d6aa011cd..27d68e04abb0 100644 --- a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java +++ b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LegacyNumericUtils; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.RamUsageEstimator; /** @@ -71,21 +72,101 @@ public interface Parser { * @param terms the {@link Terms} instance to create the {@link TermsEnum} from. * @return a possibly filtered {@link TermsEnum} instance, this method must not return null. * @throws IOException if an {@link IOException} occurs + * @deprecated index with Points instead */ + @Deprecated public TermsEnum termsEnum(Terms terms) throws IOException; /** Parse's this field's value */ public long parseValue(BytesRef term); } + + /** + * Base class for points parsers. These parsers do not use the inverted index, but instead + * uninvert point data. + * + * This abstraction can be cleaned up when Parser.termsEnum is removed. + */ + public abstract class PointParser implements Parser { + public final TermsEnum termsEnum(Terms terms) throws IOException { + throw new UnsupportedOperationException("makes no sense for parsing points"); + } + } /** Expert: The cache used internally by sorting and range query classes. */ public static FieldCache DEFAULT = new FieldCacheImpl(); + /** + * A parser instance for int values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.document.IntPoint}. + */ + public static final Parser INT_POINT_PARSER = new PointParser() { + @Override + public long parseValue(BytesRef point) { + return NumericUtils.sortableBytesToInt(point.bytes, point.offset); + } + + @Override + public String toString() { + return FieldCache.class.getName()+".INT_POINT_PARSER"; + } + }; + + /** + * A parser instance for long values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.document.LongPoint}. + */ + public static final Parser LONG_POINT_PARSER = new PointParser() { + @Override + public long parseValue(BytesRef point) { + return NumericUtils.sortableBytesToLong(point.bytes, point.offset); + } + + @Override + public String toString() { + return FieldCache.class.getName()+".LONG_POINT_PARSER"; + } + }; + + /** + * A parser instance for float values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.document.FloatPoint}. + */ + public static final Parser FLOAT_POINT_PARSER = new PointParser() { + @Override + public long parseValue(BytesRef point) { + return NumericUtils.sortableFloatBits(NumericUtils.sortableBytesToInt(point.bytes, point.offset)); + } + + @Override + public String toString() { + return FieldCache.class.getName()+".FLOAT_POINT_PARSER"; + } + }; + + /** + * A parser instance for double values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed + * via {@link org.apache.lucene.document.DoublePoint}. + */ + public static final Parser DOUBLE_POINT_PARSER = new PointParser() { + @Override + public long parseValue(BytesRef point) { + return NumericUtils.sortableDoubleBits(NumericUtils.sortableBytesToLong(point.bytes, point.offset)); + } + + @Override + public String toString() { + return FieldCache.class.getName()+".DOUBLE_POINT_PARSER"; + } + }; + /** * A parser instance for int values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed * via {@link org.apache.lucene.document.LegacyIntField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * @deprecated Index with points and use {@link #INT_POINT_PARSER} instead. */ - public static final Parser NUMERIC_UTILS_INT_PARSER = new Parser() { + @Deprecated + public static final Parser LEGACY_INT_PARSER = new Parser() { @Override public long parseValue(BytesRef term) { return LegacyNumericUtils.prefixCodedToInt(term); @@ -98,15 +179,17 @@ public TermsEnum termsEnum(Terms terms) throws IOException { @Override public String toString() { - return FieldCache.class.getName()+".NUMERIC_UTILS_INT_PARSER"; + return FieldCache.class.getName()+".LEGACY_INT_PARSER"; } }; /** * A parser instance for float values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed * via {@link org.apache.lucene.document.LegacyFloatField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * @deprecated Index with points and use {@link #FLOAT_POINT_PARSER} instead. */ - public static final Parser NUMERIC_UTILS_FLOAT_PARSER = new Parser() { + @Deprecated + public static final Parser LEGACY_FLOAT_PARSER = new Parser() { @Override public long parseValue(BytesRef term) { int val = LegacyNumericUtils.prefixCodedToInt(term); @@ -116,7 +199,7 @@ public long parseValue(BytesRef term) { @Override public String toString() { - return FieldCache.class.getName()+".NUMERIC_UTILS_FLOAT_PARSER"; + return FieldCache.class.getName()+".LEGACY_FLOAT_PARSER"; } @Override @@ -128,15 +211,17 @@ public TermsEnum termsEnum(Terms terms) throws IOException { /** * A parser instance for long values encoded by {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed * via {@link org.apache.lucene.document.LegacyLongField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * @deprecated Index with points and use {@link #LONG_POINT_PARSER} instead. */ - public static final Parser NUMERIC_UTILS_LONG_PARSER = new Parser() { + @Deprecated + public static final Parser LEGACY_LONG_PARSER = new Parser() { @Override public long parseValue(BytesRef term) { return LegacyNumericUtils.prefixCodedToLong(term); } @Override public String toString() { - return FieldCache.class.getName()+".NUMERIC_UTILS_LONG_PARSER"; + return FieldCache.class.getName()+".LEGACY_LONG_PARSER"; } @Override @@ -148,8 +233,10 @@ public TermsEnum termsEnum(Terms terms) throws IOException { /** * A parser instance for double values encoded with {@link org.apache.lucene.util.LegacyNumericUtils}, e.g. when indexed * via {@link org.apache.lucene.document.LegacyDoubleField}/{@link org.apache.lucene.analysis.LegacyNumericTokenStream}. + * @deprecated Index with points and use {@link #DOUBLE_POINT_PARSER} instead. */ - public static final Parser NUMERIC_UTILS_DOUBLE_PARSER = new Parser() { + @Deprecated + public static final Parser LEGACY_DOUBLE_PARSER = new Parser() { @Override public long parseValue(BytesRef term) { long val = LegacyNumericUtils.prefixCodedToLong(term); @@ -158,7 +245,7 @@ public long parseValue(BytesRef term) { } @Override public String toString() { - return FieldCache.class.getName()+".NUMERIC_UTILS_DOUBLE_PARSER"; + return FieldCache.class.getName()+".LEGACY_DOUBLE_PARSER"; } @Override @@ -168,18 +255,20 @@ public TermsEnum termsEnum(Terms terms) throws IOException { }; /** Checks the internal cache for an appropriate entry, and if none is found, - * reads the terms in field and returns a bit set at the size of + * reads the terms/points in field and returns a bit set at the size of * reader.maxDoc(), with turned on bits for each docid that * does have a value for this field. + * @param parser May be {@code null} if coming from the inverted index, otherwise + * can be a {@link PointParser} to compute from point values. */ - public Bits getDocsWithField(LeafReader reader, String field) throws IOException; + public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException; /** * Returns a {@link NumericDocValues} over the values found in documents in the given * field. If the field was indexed as {@link NumericDocValuesField}, it simply * uses {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)} to read the values. * Otherwise, it checks the internal cache for an appropriate entry, and if - * none is found, reads the terms in field as longs and returns + * none is found, reads the terms/points in field as longs and returns * an array of size reader.maxDoc() of the value each document * has in the given field. * @@ -199,7 +288,7 @@ public TermsEnum termsEnum(Terms terms) throws IOException { * If any error occurs. */ public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser, boolean setDocsWithField) throws IOException; - + /** Checks the internal cache for an appropriate entry, and if none * is found, reads the term values in field * and returns a {@link BinaryDocValues} instance, providing a diff --git a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java index c6bd943725de..589da132d29c 100644 --- a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java +++ b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java @@ -34,6 +34,9 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; @@ -262,8 +265,68 @@ public int hashCode() { private static abstract class Uninvert { public Bits docsWithField; + final boolean points; + + // pass true to pull from points, otherwise postings. + Uninvert(boolean points) { + this.points = points; + } + + final void uninvert(LeafReader reader, String field, boolean setDocsWithField) throws IOException { + if (points) { + uninvertPoints(reader, field, setDocsWithField); + } else { + uninvertPostings(reader, field, setDocsWithField); + } + } + + final void uninvertPoints(LeafReader reader, String field, boolean setDocsWithField) throws IOException { + final int maxDoc = reader.maxDoc(); + PointValues values = reader.getPointValues(); + assert values != null; + assert values.size(field) > 0; + + if (setDocsWithField) { + final int docCount = values.getDocCount(field); + assert docCount <= maxDoc; + if (docCount == maxDoc) { + // Fast case: all docs have this field: + this.docsWithField = new Bits.MatchAllBits(maxDoc); + setDocsWithField = false; + } + } + + final boolean doDocsWithField = setDocsWithField; + BytesRef scratch = new BytesRef(); + values.intersect(field, new IntersectVisitor() { + @Override + public void visit(int docID) throws IOException { + throw new AssertionError(); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + scratch.bytes = packedValue; + scratch.length = packedValue.length; + visitTerm(scratch); + visitDoc(docID); + if (doDocsWithField) { + if (docsWithField == null) { + // Lazy init + docsWithField = new FixedBitSet(maxDoc); + } + ((FixedBitSet)docsWithField).set(docID); + } + } - public void uninvert(LeafReader reader, String field, boolean setDocsWithField) throws IOException { + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return Relation.CELL_CROSSES_QUERY; // inspect all byte-docid pairs + } + }); + } + + final void uninvertPostings(LeafReader reader, String field, boolean setDocsWithField) throws IOException { final int maxDoc = reader.maxDoc(); Terms terms = reader.terms(field); if (terms != null) { @@ -306,13 +369,15 @@ public void uninvert(LeafReader reader, String field, boolean setDocsWithField) } } + /** @deprecated remove this when legacy numerics are removed */ + @Deprecated protected abstract TermsEnum termsEnum(Terms terms) throws IOException; protected abstract void visitTerm(BytesRef term); protected abstract void visitDoc(int docID); } // null Bits means no docs matched - void setDocsWithField(LeafReader reader, String field, Bits docsWithField) { + void setDocsWithField(LeafReader reader, String field, Bits docsWithField, Parser parser) { final int maxDoc = reader.maxDoc(); final Bits bits; if (docsWithField == null) { @@ -329,7 +394,7 @@ void setDocsWithField(LeafReader reader, String field, Bits docsWithField) { } else { bits = docsWithField; } - caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), new BitsEntry(bits)); + caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, parser), new BitsEntry(bits)); } private static class HoldsOneThing { @@ -353,17 +418,25 @@ private static class GrowableWriterAndMinValue { public long minValue; } - public Bits getDocsWithField(LeafReader reader, String field) throws IOException { + public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException { final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); if (fieldInfo == null) { // field does not exist or has no value return new Bits.MatchNoBits(reader.maxDoc()); } else if (fieldInfo.getDocValuesType() != DocValuesType.NONE) { return reader.getDocsWithField(field); - } else if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { - return new Bits.MatchNoBits(reader.maxDoc()); + } + + if (parser instanceof PointParser) { + // points case + + } else { + // postings case + if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { + return new Bits.MatchNoBits(reader.maxDoc()); + } } - BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, null), false); + BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser), false); return bitsEntry.bits; } @@ -391,9 +464,48 @@ static final class DocsWithFieldCache extends Cache { } @Override - protected BitsEntry createValue(LeafReader reader, CacheKey key, boolean setDocsWithField /* ignored */) - throws IOException { + protected BitsEntry createValue(LeafReader reader, CacheKey key, boolean setDocsWithField /* ignored */) throws IOException { final String field = key.field; + final Parser parser = (Parser) key.custom; + if (parser instanceof PointParser) { + return createValuePoints(reader, field); + } else { + return createValuePostings(reader, field); + } + } + + private BitsEntry createValuePoints(LeafReader reader, String field) throws IOException { + final int maxDoc = reader.maxDoc(); + PointValues values = reader.getPointValues(); + assert values != null; + assert values.size(field) > 0; + + final int docCount = values.getDocCount(field); + assert docCount <= maxDoc; + if (docCount == maxDoc) { + // Fast case: all docs have this field: + return new BitsEntry(new Bits.MatchAllBits(maxDoc)); + } + + // otherwise a no-op uninvert! + Uninvert u = new Uninvert(true) { + @Override + protected TermsEnum termsEnum(Terms terms) throws IOException { + throw new AssertionError(); + } + + @Override + protected void visitTerm(BytesRef term) {} + + @Override + protected void visitDoc(int docID) {} + }; + u.uninvert(reader, field, true); + return new BitsEntry(u.docsWithField); + } + + // TODO: it is dumb that uninverting code is duplicated here in this method!! + private BitsEntry createValuePostings(LeafReader reader, String field) throws IOException { final int maxDoc = reader.maxDoc(); // Visit all docs that have terms for this field @@ -458,8 +570,32 @@ public NumericDocValues getNumerics(LeafReader reader, String field, Parser pars return DocValues.emptyNumeric(); } else if (info.getDocValuesType() != DocValuesType.NONE) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); - } else if (info.getIndexOptions() == IndexOptions.NONE) { - return DocValues.emptyNumeric(); + } + + if (parser instanceof PointParser) { + // points case + // no points in this segment + if (info.getPointDimensionCount() == 0) { + return DocValues.emptyNumeric(); + } + if (info.getPointDimensionCount() != 1) { + throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount()); + } + PointValues values = reader.getPointValues(); + // no actual points for this field (e.g. all points deleted) + if (values == null || values.size(field) == 0) { + return DocValues.emptyNumeric(); + } + // not single-valued + if (values.size(field) != values.getDocCount(field)) { + throw new IllegalStateException("Type mismatch: " + field + " was indexed with multiple values, numValues=" + values.size(field) + ",numDocs=" + values.getDocCount(field)); + } + } else { + // postings case + // not indexed + if (info.getIndexOptions() == IndexOptions.NONE) { + return DocValues.emptyNumeric(); + } } return (NumericDocValues) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField); } @@ -498,7 +634,7 @@ protected Accountable createValue(final LeafReader reader, CacheKey key, boolean final HoldsOneThing valuesRef = new HoldsOneThing<>(); - Uninvert u = new Uninvert() { + Uninvert u = new Uninvert(parser instanceof PointParser) { private long minValue; private long currentValue; private GrowableWriter values; @@ -542,7 +678,7 @@ protected TermsEnum termsEnum(Terms terms) throws IOException { u.uninvert(reader, key.field, setDocsWithField); if (setDocsWithField) { - wrapper.setDocsWithField(reader, key.field, u.docsWithField); + wrapper.setDocsWithField(reader, key.field, u.docsWithField, parser); } GrowableWriterAndMinValue values = valuesRef.get(); if (values == null) { @@ -872,7 +1008,7 @@ public boolean get(int index) { public int length() { return maxDoc; } - }); + }, null); } // maybe an int-only impl? return new BinaryDocValuesImpl(bytes.freeze(true), offsetReader); diff --git a/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java b/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java index 9f96b4f77447..10d1a5b9b3c1 100644 --- a/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java +++ b/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java @@ -57,34 +57,70 @@ public class UninvertingReader extends FilterLeafReader { * Specifies the type of uninversion to apply for the field. */ public static enum Type { + /** + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.IntPoint}) + *

      + * Fields with this type act as if they were indexed with + * {@link NumericDocValuesField}. + */ + INTEGER_POINT, + /** + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LongPoint}) + *

      + * Fields with this type act as if they were indexed with + * {@link NumericDocValuesField}. + */ + LONG_POINT, + /** + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.FloatPoint}) + *

      + * Fields with this type act as if they were indexed with + * {@link NumericDocValuesField}. + */ + FLOAT_POINT, + /** + * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.DoublePoint}) + *

      + * Fields with this type act as if they were indexed with + * {@link NumericDocValuesField}. + */ + DOUBLE_POINT, /** * Single-valued Integer, (e.g. indexed with {@link org.apache.lucene.document.LegacyIntField}) *

      * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. + * @deprecated Index with points and use {@link #INTEGER_POINT} instead. */ - INTEGER, + @Deprecated + LEGACY_INTEGER, /** * Single-valued Long, (e.g. indexed with {@link org.apache.lucene.document.LegacyLongField}) *

      * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. + * @deprecated Index with points and use {@link #LONG_POINT} instead. */ - LONG, + @Deprecated + LEGACY_LONG, /** * Single-valued Float, (e.g. indexed with {@link org.apache.lucene.document.LegacyFloatField}) *

      * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. + * @deprecated Index with points and use {@link #FLOAT_POINT} instead. */ - FLOAT, + @Deprecated + LEGACY_FLOAT, /** * Single-valued Double, (e.g. indexed with {@link org.apache.lucene.document.LegacyDoubleField}) *

      * Fields with this type act as if they were indexed with * {@link NumericDocValuesField}. + * @deprecated Index with points and use {@link #DOUBLE_POINT} instead. */ - DOUBLE, + @Deprecated + LEGACY_DOUBLE, /** * Single-valued Binary, (e.g. indexed with {@link StringField}) *

      @@ -181,14 +217,29 @@ public UninvertingReader(LeafReader in, Map mapping) { ArrayList filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); - if (fi.getIndexOptions() != IndexOptions.NONE && fi.getDocValuesType() == DocValuesType.NONE) { + if (type == DocValuesType.NONE) { Type t = mapping.get(fi.name); if (t != null) { + if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) { + // type uses points + if (fi.getPointDimensionCount() == 0) { + continue; + } + } else { + // type uses inverted index + if (fi.getIndexOptions() == IndexOptions.NONE) { + continue; + } + } switch(t) { - case INTEGER: - case LONG: - case FLOAT: - case DOUBLE: + case INTEGER_POINT: + case LONG_POINT: + case FLOAT_POINT: + case DOUBLE_POINT: + case LEGACY_INTEGER: + case LEGACY_LONG: + case LEGACY_FLOAT: + case LEGACY_DOUBLE: type = DocValuesType.NUMERIC; break; case BINARY: @@ -226,10 +277,14 @@ public NumericDocValues getNumericDocValues(String field) throws IOException { Type v = getType(field); if (v != null) { switch (v) { - case INTEGER: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.NUMERIC_UTILS_INT_PARSER, true); - case FLOAT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.NUMERIC_UTILS_FLOAT_PARSER, true); - case LONG: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.NUMERIC_UTILS_LONG_PARSER, true); - case DOUBLE: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true); + case INTEGER_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.INT_POINT_PARSER, true); + case FLOAT_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.FLOAT_POINT_PARSER, true); + case LONG_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LONG_POINT_PARSER, true); + case DOUBLE_POINT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.DOUBLE_POINT_PARSER, true); + case LEGACY_INTEGER: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_INT_PARSER, true); + case LEGACY_FLOAT: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_FLOAT_PARSER, true); + case LEGACY_LONG: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_LONG_PARSER, true); + case LEGACY_DOUBLE: return FieldCache.DEFAULT.getNumerics(in, field, FieldCache.LEGACY_DOUBLE_PARSER, true); } } return super.getNumericDocValues(field); @@ -275,8 +330,20 @@ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException @Override public Bits getDocsWithField(String field) throws IOException { - if (getType(field) != null) { - return FieldCache.DEFAULT.getDocsWithField(in, field); + Type v = getType(field); + if (v != null) { + switch (v) { + case INTEGER_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.INT_POINT_PARSER); + case FLOAT_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.FLOAT_POINT_PARSER); + case LONG_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LONG_POINT_PARSER); + case DOUBLE_POINT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.DOUBLE_POINT_PARSER); + case LEGACY_INTEGER: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_INT_PARSER); + case LEGACY_FLOAT: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_FLOAT_PARSER); + case LEGACY_LONG: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_LONG_PARSER); + case LEGACY_DOUBLE: return FieldCache.DEFAULT.getDocsWithField(in, field, FieldCache.LEGACY_DOUBLE_PARSER); + default: + return FieldCache.DEFAULT.getDocsWithField(in, field, null); + } } else { return in.getDocsWithField(field); } diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java index 8c1fae7b646b..4861cd35ca2f 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java @@ -353,7 +353,7 @@ private void verify(LeafReader r, int[][] idToOrds, BytesRef[] termsArray, Bytes TestUtil.nextInt(random(), 2, 10)); - final NumericDocValues docIDToID = FieldCache.DEFAULT.getNumerics(r, "id", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + final NumericDocValues docIDToID = FieldCache.DEFAULT.getNumerics(r, "id", FieldCache.LEGACY_INT_PARSER, false); /* for(int docID=0;docID { - FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER, false); }); BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true); @@ -460,12 +442,12 @@ public void testDocValuesIntegration() throws Exception { new DocTermOrds(ar, null, "binary"); }); - Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary"); + Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary", null); assertTrue(bits.get(0)); // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds() expectThrows(IllegalStateException.class, () -> { - FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER, false); }); expectThrows(IllegalStateException.class, () -> { @@ -488,11 +470,11 @@ public void testDocValuesIntegration() throws Exception { assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); assertEquals(1, sortedSet.getValueCount()); - bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted"); + bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted", null); assertTrue(bits.get(0)); // Numeric type: can be retrieved via getInts() and so on - NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER, false); assertEquals(42, numeric.get(0)); expectThrows(IllegalStateException.class, () -> { @@ -511,12 +493,12 @@ public void testDocValuesIntegration() throws Exception { new DocTermOrds(ar, null, "numeric"); }); - bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric"); + bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric", null); assertTrue(bits.get(0)); // SortedSet type: can be retrieved via getDocTermOrds() expectThrows(IllegalStateException.class, () -> { - FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER, false); }); expectThrows(IllegalStateException.class, () -> { @@ -538,7 +520,7 @@ public void testDocValuesIntegration() throws Exception { assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); assertEquals(2, sortedSet.getValueCount()); - bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset"); + bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset", null); assertTrue(bits.get(0)); ir.close(); @@ -559,16 +541,16 @@ public void testNonexistantFields() throws Exception { cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); - NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.NUMERIC_UTILS_INT_PARSER, true); + NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER, true); assertEquals(0, ints.get(0)); - NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.NUMERIC_UTILS_LONG_PARSER, true); + NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER, true); assertEquals(0, longs.get(0)); - NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, true); + NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER, true); assertEquals(0, floats.get(0)); - NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true); + NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER, true); assertEquals(0, doubles.get(0)); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); @@ -584,7 +566,7 @@ public void testNonexistantFields() throws Exception { sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); - Bits bits = cache.getDocsWithField(ar, "bogusbits"); + Bits bits = cache.getDocsWithField(ar, "bogusbits", null); assertFalse(bits.get(0)); // check that we cached nothing @@ -617,16 +599,16 @@ public void testNonIndexedFields() throws Exception { cache.purgeAllCaches(); assertEquals(0, cache.getCacheEntries().length); - NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.NUMERIC_UTILS_INT_PARSER, true); + NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.INT_POINT_PARSER, true); assertEquals(0, ints.get(0)); - NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.NUMERIC_UTILS_LONG_PARSER, true); + NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LONG_POINT_PARSER, true); assertEquals(0, longs.get(0)); - NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, true); + NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.FLOAT_POINT_PARSER, true); assertEquals(0, floats.get(0)); - NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true); + NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.DOUBLE_POINT_PARSER, true); assertEquals(0, doubles.get(0)); BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true); @@ -642,7 +624,7 @@ public void testNonIndexedFields() throws Exception { sortedSet.setDocument(0); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); - Bits bits = cache.getDocsWithField(ar, "bogusbits"); + Bits bits = cache.getDocsWithField(ar, "bogusbits", null); assertFalse(bits.get(0)); // check that we cached nothing @@ -658,8 +640,10 @@ public void testLongFieldCache() throws IOException { cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); - LegacyLongField field = new LegacyLongField("f", 0L, Store.YES); + LongPoint field = new LongPoint("f", 0L); + StoredField field2 = new StoredField("f", 0L); doc.add(field); + doc.add(field2); final long[] values = new long[TestUtil.nextInt(random(), 1, 10)]; for (int i = 0; i < values.length; ++i) { final long v; @@ -683,12 +667,13 @@ public void testLongFieldCache() throws IOException { iw.addDocument(new Document()); } else { field.setLongValue(v); + field2.setLongValue(v); iw.addDocument(doc); } } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.NUMERIC_UTILS_LONG_PARSER, false); + final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LONG_POINT_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], longs.get(i)); } @@ -704,7 +689,7 @@ public void testIntFieldCache() throws IOException { cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); - LegacyIntField field = new LegacyIntField("f", 0, Store.YES); + IntPoint field = new IntPoint("f", 0); doc.add(field); final int[] values = new int[TestUtil.nextInt(random(), 1, 10)]; for (int i = 0; i < values.length; ++i) { @@ -734,7 +719,7 @@ public void testIntFieldCache() throws IOException { } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.INT_POINT_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], ints.get(i)); } diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java index a85731ff4d01..0d5584e4544e 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java @@ -18,8 +18,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -42,14 +41,14 @@ public void testFieldCacheReuseAfterReopen() throws Exception { setMergePolicy(newLogMergePolicy(10)) ); Document doc = new Document(); - doc.add(new LegacyIntField("number", 17, Field.Store.NO)); + doc.add(new IntPoint("number", 17)); writer.addDocument(doc); writer.commit(); // Open reader1 DirectoryReader r = DirectoryReader.open(dir); LeafReader r1 = getOnlySegmentReader(r); - final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER, false); assertEquals(17, ints.get(0)); // Add new segment @@ -61,7 +60,7 @@ public void testFieldCacheReuseAfterReopen() throws Exception { assertNotNull(r2); r.close(); LeafReader sub0 = r2.leaves().get(0).reader(); - final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.INT_POINT_PARSER, false); r2.close(); assertTrue(ints == ints2); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSanityChecker.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSanityChecker.java index f7dc0489ca71..f5c62e291c3e 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSanityChecker.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSanityChecker.java @@ -94,11 +94,11 @@ public void testSanity() throws IOException { FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); - cache.getNumerics(readerA, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); - cache.getNumerics(readerAclone, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); - cache.getNumerics(readerB, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); + cache.getNumerics(readerA, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false); + cache.getNumerics(readerAclone, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false); + cache.getNumerics(readerB, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false); - cache.getNumerics(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER, false); // // // @@ -117,7 +117,7 @@ public void testInsanity1() throws IOException { FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); - cache.getNumerics(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false); + cache.getNumerics(readerX, "theInt", FieldCache.LEGACY_INT_PARSER, false); cache.getTerms(readerX, "theInt", false); // // // diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java index 717d36424e5e..f46bdde0c93d 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java @@ -23,11 +23,16 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LegacyFloatField; import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.LegacyLongField; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -446,6 +451,140 @@ public void testFieldScoreReverse() throws Exception { /** Tests sorting on type int */ public void testInt() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new IntPoint("value", 300000)); + doc.add(new StoredField("value", 300000)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.INTEGER_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.INT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("300000", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type int with a missing value */ + public void testIntMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.INTEGER_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.INT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */ + public void testIntMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.INTEGER_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + SortField sortField = new SortField("value", SortField.Type.INT); + sortField.setMissingValue(Integer.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a Integer.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type int in reverse */ + public void testIntReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new IntPoint("value", 300000)); + doc.add(new StoredField("value", 300000)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new IntPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.INTEGER_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.INT, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("300000", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type legacy int */ + public void testLegacyInt() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -458,7 +597,7 @@ public void testInt() throws IOException { doc.add(new LegacyIntField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.INTEGER)); + Collections.singletonMap("value", Type.LEGACY_INTEGER)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -475,8 +614,8 @@ public void testInt() throws IOException { dir.close(); } - /** Tests sorting on type int with a missing value */ - public void testIntMissing() throws IOException { + /** Tests sorting on type legacy int with a missing value */ + public void testLegacyIntMissing() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -488,7 +627,7 @@ public void testIntMissing() throws IOException { doc.add(new LegacyIntField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.INTEGER)); + Collections.singletonMap("value", Type.LEGACY_INTEGER)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -505,8 +644,8 @@ public void testIntMissing() throws IOException { dir.close(); } - /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */ - public void testIntMissingLast() throws IOException { + /** Tests sorting on type legacy int, specifying the missing value should be treated as Integer.MAX_VALUE */ + public void testLegacyIntMissingLast() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -518,7 +657,7 @@ public void testIntMissingLast() throws IOException { doc.add(new LegacyIntField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.INTEGER)); + Collections.singletonMap("value", Type.LEGACY_INTEGER)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -537,8 +676,8 @@ public void testIntMissingLast() throws IOException { dir.close(); } - /** Tests sorting on type int in reverse */ - public void testIntReverse() throws IOException { + /** Tests sorting on type legacy int in reverse */ + public void testLegacyIntReverse() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -551,7 +690,7 @@ public void testIntReverse() throws IOException { doc.add(new LegacyIntField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.INTEGER)); + Collections.singletonMap("value", Type.LEGACY_INTEGER)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -570,6 +709,140 @@ public void testIntReverse() throws IOException { /** Tests sorting on type long */ public void testLong() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new LongPoint("value", 3000000000L)); + doc.add(new StoredField("value", 3000000000L)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.LONG_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("3000000000", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type long with a missing value */ + public void testLongMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.LONG_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */ + public void testLongMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.LONG_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + SortField sortField = new SortField("value", SortField.Type.LONG); + sortField.setMissingValue(Long.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Long.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type long in reverse */ + public void testLongReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new LongPoint("value", 3000000000L)); + doc.add(new StoredField("value", 3000000000L)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", -1)); + doc.add(new StoredField("value", -1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new LongPoint("value", 4)); + doc.add(new StoredField("value", 4)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.LONG_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("3000000000", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type legacy long */ + public void testLegacyLong() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -582,7 +855,7 @@ public void testLong() throws IOException { doc.add(new LegacyLongField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.LONG)); + Collections.singletonMap("value", Type.LEGACY_LONG)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -599,8 +872,8 @@ public void testLong() throws IOException { dir.close(); } - /** Tests sorting on type long with a missing value */ - public void testLongMissing() throws IOException { + /** Tests sorting on type legacy long with a missing value */ + public void testLegacyLongMissing() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -612,7 +885,7 @@ public void testLongMissing() throws IOException { doc.add(new LegacyLongField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.LONG)); + Collections.singletonMap("value", Type.LEGACY_LONG)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -629,8 +902,8 @@ public void testLongMissing() throws IOException { dir.close(); } - /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */ - public void testLongMissingLast() throws IOException { + /** Tests sorting on type legacy long, specifying the missing value should be treated as Long.MAX_VALUE */ + public void testLegacyLongMissingLast() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -642,7 +915,7 @@ public void testLongMissingLast() throws IOException { doc.add(new LegacyLongField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.LONG)); + Collections.singletonMap("value", Type.LEGACY_LONG)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -661,8 +934,8 @@ public void testLongMissingLast() throws IOException { dir.close(); } - /** Tests sorting on type long in reverse */ - public void testLongReverse() throws IOException { + /** Tests sorting on type legacy long in reverse */ + public void testLegacyLongReverse() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -675,7 +948,7 @@ public void testLongReverse() throws IOException { doc.add(new LegacyLongField("value", 4, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.LONG)); + Collections.singletonMap("value", Type.LEGACY_LONG)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -694,6 +967,140 @@ public void testLongReverse() throws IOException { /** Tests sorting on type float */ public void testFloat() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new FloatPoint("value", 30.1f)); + doc.add(new StoredField("value", 30.1f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", -1.3f)); + doc.add(new StoredField("value", -1.3f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", 4.2f)); + doc.add(new StoredField("value", 4.2f)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.FLOAT_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("30.1", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type float with a missing value */ + public void testFloatMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", -1.3f)); + doc.add(new StoredField("value", -1.3f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", 4.2f)); + doc.add(new StoredField("value", 4.2f)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.FLOAT_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */ + public void testFloatMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", -1.3f)); + doc.add(new StoredField("value", -1.3f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", 4.2f)); + doc.add(new StoredField("value", 4.2f)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.FLOAT_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + SortField sortField = new SortField("value", SortField.Type.FLOAT); + sortField.setMissingValue(Float.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Float.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type float in reverse */ + public void testFloatReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new FloatPoint("value", 30.1f)); + doc.add(new StoredField("value", 30.1f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", -1.3f)); + doc.add(new StoredField("value", -1.3f)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new FloatPoint("value", 4.2f)); + doc.add(new StoredField("value", 4.2f)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.FLOAT_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1.3", searcher.doc(td.scoreDocs[2].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type legacy float */ + public void testLegacyFloat() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -706,7 +1113,7 @@ public void testFloat() throws IOException { doc.add(new LegacyFloatField("value", 4.2f, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.FLOAT)); + Collections.singletonMap("value", Type.LEGACY_FLOAT)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -723,8 +1130,8 @@ public void testFloat() throws IOException { dir.close(); } - /** Tests sorting on type float with a missing value */ - public void testFloatMissing() throws IOException { + /** Tests sorting on type legacy float with a missing value */ + public void testLegacyFloatMissing() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -736,7 +1143,7 @@ public void testFloatMissing() throws IOException { doc.add(new LegacyFloatField("value", 4.2f, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.FLOAT)); + Collections.singletonMap("value", Type.LEGACY_FLOAT)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -753,8 +1160,8 @@ public void testFloatMissing() throws IOException { dir.close(); } - /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */ - public void testFloatMissingLast() throws IOException { + /** Tests sorting on type legacy float, specifying the missing value should be treated as Float.MAX_VALUE */ + public void testLegacyFloatMissingLast() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -766,7 +1173,7 @@ public void testFloatMissingLast() throws IOException { doc.add(new LegacyFloatField("value", 4.2f, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.FLOAT)); + Collections.singletonMap("value", Type.LEGACY_FLOAT)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -785,8 +1192,8 @@ public void testFloatMissingLast() throws IOException { dir.close(); } - /** Tests sorting on type float in reverse */ - public void testFloatReverse() throws IOException { + /** Tests sorting on type legacy float in reverse */ + public void testLegacyFloatReverse() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -799,7 +1206,7 @@ public void testFloatReverse() throws IOException { doc.add(new LegacyFloatField("value", 4.2f, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.FLOAT)); + Collections.singletonMap("value", Type.LEGACY_FLOAT)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -818,6 +1225,195 @@ public void testFloatReverse() throws IOException { /** Tests sorting on type double */ public void testDouble() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new DoublePoint("value", 30.1)); + doc.add(new StoredField("value", 30.1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", -1.3)); + doc.add(new StoredField("value", -1.3)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333333)); + doc.add(new StoredField("value", 4.2333333333333)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333332)); + doc.add(new StoredField("value", 4.2333333333332)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.DOUBLE_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // numeric order + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("30.1", searcher.doc(td.scoreDocs[3].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type double with +/- zero */ + public void testDoubleSignedZero() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new DoublePoint("value", +0d)); + doc.add(new StoredField("value", +0d)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", -0d)); + doc.add(new StoredField("value", -0d)); + writer.addDocument(doc); + doc = new Document(); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.DOUBLE_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // numeric order + double v0 = searcher.doc(td.scoreDocs[0].doc).getField("value").numericValue().doubleValue(); + double v1 = searcher.doc(td.scoreDocs[1].doc).getField("value").numericValue().doubleValue(); + assertEquals(0, v0, 0d); + assertEquals(0, v1, 0d); + // check sign bits + assertEquals(1, Double.doubleToLongBits(v0) >>> 63); + assertEquals(0, Double.doubleToLongBits(v1) >>> 63); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type double with a missing value */ + public void testDoubleMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", -1.3)); + doc.add(new StoredField("value", -1.3)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333333)); + doc.add(new StoredField("value", 4.2333333333333)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333332)); + doc.add(new StoredField("value", 4.2333333333332)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.DOUBLE_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as a 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[3].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */ + public void testDoubleMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", -1.3)); + doc.add(new StoredField("value", -1.3)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333333)); + doc.add(new StoredField("value", 4.2333333333333)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333332)); + doc.add(new StoredField("value", 4.2333333333332)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.DOUBLE_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + SortField sortField = new SortField("value", SortField.Type.DOUBLE); + sortField.setMissingValue(Double.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as Double.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[3].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type double in reverse */ + public void testDoubleReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new DoublePoint("value", 30.1)); + doc.add(new StoredField("value", 30.1)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", -1.3)); + doc.add(new StoredField("value", -1.3)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333333)); + doc.add(new StoredField("value", 4.2333333333333)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new DoublePoint("value", 4.2333333333332)); + doc.add(new StoredField("value", 4.2333333333332)); + writer.addDocument(doc); + IndexReader ir = UninvertingReader.wrap(writer.getReader(), + Collections.singletonMap("value", Type.DOUBLE_POINT)); + writer.close(); + + IndexSearcher searcher = newSearcher(ir, false); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // numeric order + assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("-1.3", searcher.doc(td.scoreDocs[3].doc).get("value")); + TestUtil.checkReader(ir); + ir.close(); + dir.close(); + } + + /** Tests sorting on type legacy double */ + public void testLegacyDouble() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -833,7 +1429,7 @@ public void testDouble() throws IOException { doc.add(new LegacyDoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.DOUBLE)); + Collections.singletonMap("value", Type.LEGACY_DOUBLE)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -851,8 +1447,8 @@ public void testDouble() throws IOException { dir.close(); } - /** Tests sorting on type double with +/- zero */ - public void testDoubleSignedZero() throws IOException { + /** Tests sorting on type legacy double with +/- zero */ + public void testLegacyDoubleSignedZero() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -863,7 +1459,7 @@ public void testDoubleSignedZero() throws IOException { writer.addDocument(doc); doc = new Document(); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.DOUBLE)); + Collections.singletonMap("value", Type.LEGACY_DOUBLE)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -884,8 +1480,8 @@ public void testDoubleSignedZero() throws IOException { dir.close(); } - /** Tests sorting on type double with a missing value */ - public void testDoubleMissing() throws IOException { + /** Tests sorting on type legacy double with a missing value */ + public void testLegacyDoubleMissing() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -900,7 +1496,7 @@ public void testDoubleMissing() throws IOException { doc.add(new LegacyDoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.DOUBLE)); + Collections.singletonMap("value", Type.LEGACY_DOUBLE)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -918,8 +1514,8 @@ public void testDoubleMissing() throws IOException { dir.close(); } - /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */ - public void testDoubleMissingLast() throws IOException { + /** Tests sorting on type legacy double, specifying the missing value should be treated as Double.MAX_VALUE */ + public void testLegacyDoubleMissingLast() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -934,7 +1530,7 @@ public void testDoubleMissingLast() throws IOException { doc.add(new LegacyDoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.DOUBLE)); + Collections.singletonMap("value", Type.LEGACY_DOUBLE)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -954,8 +1550,8 @@ public void testDoubleMissingLast() throws IOException { dir.close(); } - /** Tests sorting on type double in reverse */ - public void testDoubleReverse() throws IOException { + /** Tests sorting on type legacy double in reverse */ + public void testLegacyDoubleReverse() throws IOException { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); @@ -971,7 +1567,7 @@ public void testDoubleReverse() throws IOException { doc.add(new LegacyDoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = UninvertingReader.wrap(writer.getReader(), - Collections.singletonMap("value", Type.DOUBLE)); + Collections.singletonMap("value", Type.LEGACY_DOUBLE)); writer.close(); IndexSearcher searcher = newSearcher(ir); @@ -1062,7 +1658,7 @@ public void testMaxScore() throws Exception { } IndexReader r = UninvertingReader.wrap(DirectoryReader.open(w), - Collections.singletonMap("id", Type.INTEGER)); + Collections.singletonMap("id", Type.LEGACY_INTEGER)); w.close(); Query q = new TermQuery(new Term("body", "text")); IndexSearcher s = newSearcher(r); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java index 0b6292d3c65c..f3bd455e6915 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java @@ -30,7 +30,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; @@ -118,7 +119,8 @@ private void testRandomStringSort(SortField.Type type) throws Exception { docValues.add(null); } - doc.add(new LegacyIntField("id", numDocs, Field.Store.YES)); + doc.add(new IntPoint("id", numDocs)); + doc.add(new StoredField("id", numDocs)); writer.addDocument(doc); numDocs++; @@ -130,7 +132,7 @@ private void testRandomStringSort(SortField.Type type) throws Exception { Map mapping = new HashMap<>(); mapping.put("stringdv", Type.SORTED); - mapping.put("id", Type.INTEGER); + mapping.put("id", Type.INTEGER_POINT); final IndexReader r = UninvertingReader.wrap(writer.getReader(), mapping); writer.close(); if (VERBOSE) { diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java index 1b1452289cc8..23b7d0c7a912 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java @@ -458,8 +458,8 @@ private void doTestMissingVsFieldCache(LongProducer longs) throws Exception { DirectoryReader ir = DirectoryReader.open(dir); for (LeafReaderContext context : ir.leaves()) { LeafReader r = context.reader(); - Bits expected = FieldCache.DEFAULT.getDocsWithField(r, "indexed"); - Bits actual = FieldCache.DEFAULT.getDocsWithField(r, "dv"); + Bits expected = FieldCache.DEFAULT.getDocsWithField(r, "indexed", null); + Bits actual = FieldCache.DEFAULT.getDocsWithField(r, "dv", null); assertEquals(expected, actual); } ir.close(); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java index 9b05ee112015..e716419de7ca 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java @@ -42,6 +42,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +// TODO: what happened to this test... its not actually uninverting? public class TestFieldCacheWithThreads extends LuceneTestCase { public void test() throws Exception { @@ -83,7 +84,7 @@ public void test() throws Exception { public void run() { try { //NumericDocValues ndv = ar.getNumericDocValues("number"); - NumericDocValues ndv = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_LONG_PARSER, false); + NumericDocValues ndv = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false); //BinaryDocValues bdv = ar.getBinaryDocValues("bytes"); BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes", false); SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted"); @@ -93,16 +94,16 @@ public void run() { int docID = threadRandom.nextInt(numDocs); switch(threadRandom.nextInt(4)) { case 0: - assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_INT_PARSER, false).get(docID)); + assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.INT_POINT_PARSER, false).get(docID)); break; case 1: - assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_LONG_PARSER, false).get(docID)); + assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER, false).get(docID)); break; case 2: - assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, false).get(docID)); + assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.FLOAT_POINT_PARSER, false).get(docID)); break; case 3: - assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false).get(docID)); + assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.DOUBLE_POINT_PARSER, false).get(docID)); break; } BytesRef term = bdv.get(docID); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java new file mode 100644 index 000000000000..c4ef1c4f4b76 --- /dev/null +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.uninverting; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LegacyDoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.LegacyFloatField; +import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.LegacyLongField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LegacyNumericUtils; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** random assortment of tests against legacy numerics */ +public class TestLegacyFieldCache extends LuceneTestCase { + private static LeafReader reader; + private static int NUM_DOCS; + private static Directory directory; + + @BeforeClass + public static void beforeClass() throws Exception { + NUM_DOCS = atLeast(500); + directory = newDirectory(); + RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); + long theLong = Long.MAX_VALUE; + double theDouble = Double.MAX_VALUE; + int theInt = Integer.MAX_VALUE; + float theFloat = Float.MAX_VALUE; + if (VERBOSE) { + System.out.println("TEST: setUp"); + } + for (int i = 0; i < NUM_DOCS; i++){ + Document doc = new Document(); + doc.add(new LegacyLongField("theLong", theLong--, Field.Store.NO)); + doc.add(new LegacyDoubleField("theDouble", theDouble--, Field.Store.NO)); + doc.add(new LegacyIntField("theInt", theInt--, Field.Store.NO)); + doc.add(new LegacyFloatField("theFloat", theFloat--, Field.Store.NO)); + if (i%2 == 0) { + doc.add(new LegacyIntField("sparse", i, Field.Store.NO)); + } + + if (i%2 == 0) { + doc.add(new LegacyIntField("numInt", i, Field.Store.NO)); + } + writer.addDocument(doc); + } + IndexReader r = writer.getReader(); + reader = SlowCompositeReaderWrapper.wrap(r); + TestUtil.checkReader(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + reader.close(); + reader = null; + directory.close(); + directory = null; + } + + public void testInfoStream() throws Exception { + try { + FieldCache cache = FieldCache.DEFAULT; + ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); + cache.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8)); + cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, false); + cache.getNumerics(reader, "theDouble", new FieldCache.Parser() { + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return LegacyNumericUtils.filterPrefixCodedLongs(terms.iterator()); + } + @Override + public long parseValue(BytesRef term) { + int val = (int) LegacyNumericUtils.prefixCodedToLong(term); + if (val<0) val ^= 0x7fffffff; + return val; + } + }, false); + assertTrue(bos.toString(IOUtils.UTF_8).indexOf("WARNING") != -1); + } finally { + FieldCache.DEFAULT.setInfoStream(null); + FieldCache.DEFAULT.purgeAllCaches(); + } + } + + public void test() throws IOException { + FieldCache cache = FieldCache.DEFAULT; + NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean()); + assertSame("Second request to cache return same array", doubles, cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, random().nextBoolean())); + for (int i = 0; i < NUM_DOCS; i++) { + assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.get(i)); + } + + NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean()); + assertSame("Second request to cache return same array", longs, cache.getNumerics(reader, "theLong", FieldCache.LEGACY_LONG_PARSER, random().nextBoolean())); + for (int i = 0; i < NUM_DOCS; i++) { + assertEquals(Long.MAX_VALUE - i, longs.get(i)); + } + + NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean()); + assertSame("Second request to cache return same array", ints, cache.getNumerics(reader, "theInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean())); + for (int i = 0; i < NUM_DOCS; i++) { + assertEquals(Integer.MAX_VALUE - i, ints.get(i)); + } + + NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean()); + assertSame("Second request to cache return same array", floats, cache.getNumerics(reader, "theFloat", FieldCache.LEGACY_FLOAT_PARSER, random().nextBoolean())); + for (int i = 0; i < NUM_DOCS; i++) { + assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.get(i)); + } + + Bits docsWithField = cache.getDocsWithField(reader, "theLong", null); + assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "theLong", null)); + assertTrue("docsWithField(theLong) must be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits); + assertTrue("docsWithField(theLong) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS); + for (int i = 0; i < docsWithField.length(); i++) { + assertTrue(docsWithField.get(i)); + } + + docsWithField = cache.getDocsWithField(reader, "sparse", null); + assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "sparse", null)); + assertFalse("docsWithField(sparse) must not be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits); + assertTrue("docsWithField(sparse) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS); + for (int i = 0; i < docsWithField.length(); i++) { + assertEquals(i%2 == 0, docsWithField.get(i)); + } + + FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey()); + } + + public void testEmptyIndex() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer= new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500)); + writer.close(); + IndexReader r = DirectoryReader.open(dir); + LeafReader reader = SlowCompositeReaderWrapper.wrap(r); + TestUtil.checkReader(reader); + FieldCache.DEFAULT.getTerms(reader, "foobar", true); + FieldCache.DEFAULT.getTermsIndex(reader, "foobar"); + FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey()); + r.close(); + dir.close(); + } + + public void testDocsWithField() throws Exception { + FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + assertEquals(0, cache.getCacheEntries().length); + cache.getNumerics(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER, true); + + // The double[] takes one slots, and docsWithField should also + // have been populated: + assertEquals(2, cache.getCacheEntries().length); + Bits bits = cache.getDocsWithField(reader, "theDouble", FieldCache.LEGACY_DOUBLE_PARSER); + + // No new entries should appear: + assertEquals(2, cache.getCacheEntries().length); + assertTrue(bits instanceof Bits.MatchAllBits); + + NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true); + assertEquals(4, cache.getCacheEntries().length); + Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.LEGACY_INT_PARSER); + assertEquals(4, cache.getCacheEntries().length); + for (int i = 0; i < docsWithField.length(); i++) { + if (i%2 == 0) { + assertTrue(docsWithField.get(i)); + assertEquals(i, ints.get(i)); + } else { + assertFalse(docsWithField.get(i)); + } + } + + NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.LEGACY_INT_PARSER, random().nextBoolean()); + docsWithField = cache.getDocsWithField(reader, "numInt", FieldCache.LEGACY_INT_PARSER); + for (int i = 0; i < docsWithField.length(); i++) { + if (i%2 == 0) { + assertTrue(docsWithField.get(i)); + assertEquals(i, numInts.get(i)); + } else { + assertFalse(docsWithField.get(i)); + } + } + } + + public void testGetDocsWithFieldThreadSafety() throws Exception { + final FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + + int NUM_THREADS = 3; + Thread[] threads = new Thread[NUM_THREADS]; + final AtomicBoolean failed = new AtomicBoolean(); + final AtomicInteger iters = new AtomicInteger(); + final int NUM_ITER = 200 * RANDOM_MULTIPLIER; + final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS, + new Runnable() { + @Override + public void run() { + cache.purgeAllCaches(); + iters.incrementAndGet(); + } + }); + for(int threadIDX=0;threadIDX= NUM_ITER) { + break; + } + } else if (op == 1) { + Bits docsWithField = cache.getDocsWithField(reader, "sparse", null); + for (int i = 0; i < docsWithField.length(); i++) { + assertEquals(i%2 == 0, docsWithField.get(i)); + } + } else { + NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.LEGACY_INT_PARSER, true); + Bits docsWithField = cache.getDocsWithField(reader, "sparse", null); + for (int i = 0; i < docsWithField.length(); i++) { + if (i%2 == 0) { + assertTrue(docsWithField.get(i)); + assertEquals(i, ints.get(i)); + } else { + assertFalse(docsWithField.get(i)); + } + } + } + } + } catch (Throwable t) { + failed.set(true); + restart.reset(); + throw new RuntimeException(t); + } + } + }; + threads[threadIDX].start(); + } + + for(int threadIDX=0;threadIDX { + FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER, false); + }); + + // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds() + expectThrows(IllegalStateException.class, () -> { + FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER, false); + }); + + // Numeric type: can be retrieved via getInts() and so on + NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER, false); + assertEquals(42, numeric.get(0)); + + // SortedSet type: can be retrieved via getDocTermOrds() + expectThrows(IllegalStateException.class, () -> { + FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER, false); + }); + + ir.close(); + dir.close(); + } + + public void testNonexistantFields() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + iw.addDocument(doc); + DirectoryReader ir = iw.getReader(); + iw.close(); + + LeafReader ar = getOnlySegmentReader(ir); + + final FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + assertEquals(0, cache.getCacheEntries().length); + + NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true); + assertEquals(0, ints.get(0)); + + NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true); + assertEquals(0, longs.get(0)); + + NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true); + assertEquals(0, floats.get(0)); + + NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true); + assertEquals(0, doubles.get(0)); + + // check that we cached nothing + assertEquals(0, cache.getCacheEntries().length); + ir.close(); + dir.close(); + } + + public void testNonIndexedFields() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new StoredField("bogusbytes", "bogus")); + doc.add(new StoredField("bogusshorts", "bogus")); + doc.add(new StoredField("bogusints", "bogus")); + doc.add(new StoredField("boguslongs", "bogus")); + doc.add(new StoredField("bogusfloats", "bogus")); + doc.add(new StoredField("bogusdoubles", "bogus")); + doc.add(new StoredField("bogusbits", "bogus")); + iw.addDocument(doc); + DirectoryReader ir = iw.getReader(); + iw.close(); + + LeafReader ar = getOnlySegmentReader(ir); + + final FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + assertEquals(0, cache.getCacheEntries().length); + + NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.LEGACY_INT_PARSER, true); + assertEquals(0, ints.get(0)); + + NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.LEGACY_LONG_PARSER, true); + assertEquals(0, longs.get(0)); + + NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.LEGACY_FLOAT_PARSER, true); + assertEquals(0, floats.get(0)); + + NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.LEGACY_DOUBLE_PARSER, true); + assertEquals(0, doubles.get(0)); + + // check that we cached nothing + assertEquals(0, cache.getCacheEntries().length); + ir.close(); + dir.close(); + } + + // Make sure that the use of GrowableWriter doesn't prevent from using the full long range + public void testLongFieldCache() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random())); + cfg.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); + Document doc = new Document(); + LegacyLongField field = new LegacyLongField("f", 0L, Store.YES); + doc.add(field); + final long[] values = new long[TestUtil.nextInt(random(), 1, 10)]; + for (int i = 0; i < values.length; ++i) { + final long v; + switch (random().nextInt(10)) { + case 0: + v = Long.MIN_VALUE; + break; + case 1: + v = 0; + break; + case 2: + v = Long.MAX_VALUE; + break; + default: + v = TestUtil.nextLong(random(), -10, 10); + break; + } + values[i] = v; + if (v == 0 && random().nextBoolean()) { + // missing + iw.addDocument(new Document()); + } else { + field.setLongValue(v); + iw.addDocument(doc); + } + } + iw.forceMerge(1); + final DirectoryReader reader = iw.getReader(); + final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LEGACY_LONG_PARSER, false); + for (int i = 0; i < values.length; ++i) { + assertEquals(values[i], longs.get(i)); + } + reader.close(); + iw.close(); + dir.close(); + } + + // Make sure that the use of GrowableWriter doesn't prevent from using the full int range + public void testIntFieldCache() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random())); + cfg.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); + Document doc = new Document(); + LegacyIntField field = new LegacyIntField("f", 0, Store.YES); + doc.add(field); + final int[] values = new int[TestUtil.nextInt(random(), 1, 10)]; + for (int i = 0; i < values.length; ++i) { + final int v; + switch (random().nextInt(10)) { + case 0: + v = Integer.MIN_VALUE; + break; + case 1: + v = 0; + break; + case 2: + v = Integer.MAX_VALUE; + break; + default: + v = TestUtil.nextInt(random(), -10, 10); + break; + } + values[i] = v; + if (v == 0 && random().nextBoolean()) { + // missing + iw.addDocument(new Document()); + } else { + field.setIntValue(v); + iw.addDocument(doc); + } + } + iw.forceMerge(1); + final DirectoryReader reader = iw.getReader(); + final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LEGACY_INT_PARSER, false); + for (int i = 0; i < values.length; ++i) { + assertEquals(values[i], ints.get(i)); + } + reader.close(); + iw.close(); + dir.close(); + } + +} diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java index bc85db4b3713..a0cddf88c6b4 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java @@ -96,9 +96,9 @@ public static void beforeClass() throws Exception { } Map map = new HashMap<>(); - map.put("field2", Type.INTEGER); - map.put("field4", Type.INTEGER); - map.put("field8", Type.INTEGER); + map.put("field2", Type.LEGACY_INTEGER); + map.put("field4", Type.LEGACY_INTEGER); + map.put("field8", Type.LEGACY_INTEGER); reader = UninvertingReader.wrap(writer.getReader(), map); searcher=newSearcher(reader); writer.close(); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java index d9fcc92eeddf..0724d86feeb6 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java @@ -100,10 +100,10 @@ public static void beforeClass() throws Exception { writer.addDocument(doc); } Map map = new HashMap<>(); - map.put("field2", Type.LONG); - map.put("field4", Type.LONG); - map.put("field6", Type.LONG); - map.put("field8", Type.LONG); + map.put("field2", Type.LEGACY_LONG); + map.put("field4", Type.LEGACY_LONG); + map.put("field6", Type.LEGACY_LONG); + map.put("field8", Type.LEGACY_LONG); reader = UninvertingReader.wrap(writer.getReader(), map); searcher=newSearcher(reader); writer.close(); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java index 99df329284d6..0a1cf3d40488 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java @@ -363,8 +363,9 @@ public void testFieldInfos() throws IOException { iw.close(); Map uninvertingMap = new HashMap<>(); - uninvertingMap.put("int", Type.INTEGER); - uninvertingMap.put("dv", Type.INTEGER); + uninvertingMap.put("int", Type.LEGACY_INTEGER); + uninvertingMap.put("dv", Type.LEGACY_INTEGER); + uninvertingMap.put("dint", Type.INTEGER_POINT); DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), uninvertingMap); @@ -376,6 +377,7 @@ public void testFieldInfos() throws IOException { assertEquals(0, intFInfo.getPointNumBytes()); FieldInfo dintFInfo = leafReader.getFieldInfos().fieldInfo("dint"); + assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType()); assertEquals(1, dintFInfo.getPointDimensionCount()); assertEquals(4, dintFInfo.getPointNumBytes()); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java index 529e98bc5587..8ccb9afdbe56 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -73,8 +73,8 @@ public void setUp() throws Exception { super.setUp(); // TODO: change this module to index docvalues instead of uninverting uninvertMap.clear(); - uninvertMap.put("pointvector__x", Type.DOUBLE); - uninvertMap.put("pointvector__y", Type.DOUBLE); + uninvertMap.put("pointvector__x", Type.LEGACY_DOUBLE); + uninvertMap.put("pointvector__y", Type.LEGACY_DOUBLE); directory = newDirectory(); final Random random = random(); diff --git a/solr/core/src/java/org/apache/solr/schema/EnumField.java b/solr/core/src/java/org/apache/solr/schema/EnumField.java index e1fb42057beb..cbf1d4eb9e8d 100644 --- a/solr/core/src/java/org/apache/solr/schema/EnumField.java +++ b/solr/core/src/java/org/apache/solr/schema/EnumField.java @@ -194,7 +194,7 @@ public Type getUninversionType(SchemaField sf) { if (sf.multiValued()) { return Type.SORTED_SET_INTEGER; } else { - return Type.INTEGER; + return Type.LEGACY_INTEGER; } } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index 572bf8884704..c4899a106045 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -203,14 +203,14 @@ public Type getUninversionType(SchemaField sf) { } else { switch (type) { case INTEGER: - return Type.INTEGER; + return Type.LEGACY_INTEGER; case LONG: case DATE: - return Type.LONG; + return Type.LEGACY_LONG; case FLOAT: - return Type.FLOAT; + return Type.LEGACY_FLOAT; case DOUBLE: - return Type.DOUBLE; + return Type.LEGACY_DOUBLE; default: throw new AssertionError(); } From 6689e1c55a7efffadb18d25179d9d87c121af5d0 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 11 Mar 2016 00:49:49 +0530 Subject: [PATCH 0094/1113] SOLR-8135: SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection reproducible failure --- .../src/java/org/apache/solr/core/CoreContainer.java | 6 ++++-- solr/core/src/java/org/apache/solr/core/SolrCore.java | 10 +++++++--- .../src/java/org/apache/solr/update/SolrCoreState.java | 8 +++++++- .../org/apache/solr/cloud/SolrCloudExampleTest.java | 1 - 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 1d614e343dca..c140fb4dba8e 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -62,6 +62,7 @@ import org.apache.solr.security.HttpClientInterceptorPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.SecurityPluginHolder; +import org.apache.solr.update.SolrCoreState; import org.apache.solr.update.UpdateShardHandler; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.zookeeper.KeeperException; @@ -916,8 +917,9 @@ public void reload(String name) { log.info("Reloading SolrCore '{}' using configuration from {}", cd.getName(), coreConfig.getName()); SolrCore newCore = core.reload(coreConfig); registerCore(name, newCore, false); - } - catch (Exception e) { + } catch (SolrCoreState.CoreIsClosedException e) { + throw e; + } catch (Exception e) { coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, e)); throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to reload core [" + cd.getName() + "]", e); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index cde878a5ebf3..7a65a72decbb 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -534,8 +534,8 @@ void initIndex(boolean reload) throws IOException { // Create the index if it doesn't exist. if(!indexExists) { - log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist." - + " Creating new index..."); + log.warn(logid + "Solr index directory '" + new File(indexDir) + "' doesn't exist." + + " Creating new index..."); SolrIndexWriter writer = SolrIndexWriter.create(this, "SolrCore.initIndex", indexDir, getDirectoryFactory(), true, getLatestSchema(), solrConfig.indexConfig, solrDelPolicy, codec); @@ -2501,7 +2501,11 @@ public static Runnable getConfListener(SolrCore core, ZkSolrResourceLoader zkSol checkStale(zkClient, solrConfigPath, overlayVersion) || checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) { log.info("core reload {}", coreName); - cc.reload(coreName); + try { + cc.reload(coreName); + } catch (SolrCoreState.CoreIsClosedException e) { + /*no problem this core is already closed*/ + } return; } //some files in conf directory may have other than managedschema, overlay, params diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 42727b4a64a4..fc0bca8a125c 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -51,7 +51,7 @@ public Object getUpdateLock() { public void increfSolrCoreState() { synchronized (this) { if (solrCoreStateRefCnt == 0) { - throw new IllegalStateException("IndexWriter has been closed"); + throw new CoreIsClosedException("IndexWriter has been closed"); } solrCoreStateRefCnt++; } @@ -157,4 +157,10 @@ public interface IndexWriterCloser { public abstract boolean getLastReplicateIndexSuccess(); public abstract void setLastReplicateIndexSuccess(boolean success); + + public static class CoreIsClosedException extends IllegalStateException { + public CoreIsClosedException(String s) { + super(s); + } + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java index e889d90b3c72..f084c2073455 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java @@ -43,7 +43,6 @@ * this test is useful for catching regressions in indexing the example docs in collections that * use data-driven schema and managed schema features provided by configsets/data_driven_schema_configs. */ -@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-8135") public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); From 96b059c9dd67eef3a49da63d388fac7f4d3809f2 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 11 Mar 2016 00:52:52 +0530 Subject: [PATCH 0095/1113] SOLR-8135: SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection reproducible failure --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index feb0e10dc2ea..0aca8e09c5af 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -283,6 +283,8 @@ Bug Fixes other than count, resulted in incorrect results. This has been fixed, and facet.prefix support for facet.method=uif has been enabled. (Mikhail Khludnev, yonik) +* SOLR-8135: If a core reload is fired after a core close, it is not a recoverable error (noble) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been From 4a795ee371c1bb19285cca0fb7336807f6e2840f Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 11 Mar 2016 00:57:26 +0530 Subject: [PATCH 0096/1113] SOLR-8135: typo in comment --- solr/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0aca8e09c5af..a1b14d6afdc8 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -283,7 +283,7 @@ Bug Fixes other than count, resulted in incorrect results. This has been fixed, and facet.prefix support for facet.method=uif has been enabled. (Mikhail Khludnev, yonik) -* SOLR-8135: If a core reload is fired after a core close, it is not a recoverable error (noble) +* SOLR-8135: If a core reload is fired after a core close, it is not a non-recoverable error (noble) Optimizations ---------------------- From 9f48f53a086f809490e2b1e00cabe3cc29bf2b08 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 10 Mar 2016 16:34:04 -0500 Subject: [PATCH 0097/1113] fix rare-ish test bug --- .../apache/lucene/index/TestAllFilesCheckIndexHeader.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index f6c1486bc433..c430281b117f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -28,6 +28,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; @@ -86,6 +87,12 @@ private void checkIndexHeader(Directory dir) throws IOException { private void checkOneFile(Directory dir, String victim) throws IOException { try (BaseDirectoryWrapper dirCopy = newDirectory()) { dirCopy.setCheckIndexOnClose(false); + + if (dirCopy instanceof MockDirectoryWrapper) { + // The while(true) loop below, under rarish circumstances, can sometimes double write: + ((MockDirectoryWrapper) dirCopy).setPreventDoubleWrite(false); + } + long victimLength = dir.fileLength(victim); int wrongBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength)); assert victimLength > 0; From 7927a3101e1f446939bfa311968346ddd57c29f3 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 10 Mar 2016 16:08:59 -0500 Subject: [PATCH 0098/1113] LUCENE-7086: move SlowCompositeReaderWrapper to misc module, and throw clear exc if you try to use in with points Squashed commits: commit e26b065c71388407bc6725256ca43d7bb30dee29 Author: Mike McCandless Date: Thu Mar 10 14:16:45 2016 -0500 simplify the checking for incoming points commit b7254376dcb398c7739aab4544118bb4526961d5 Merge: 8ec82a0 d35d569 Author: Mike McCandless Date: Thu Mar 10 14:05:24 2016 -0500 Merge branch 'master' into slow_wrapper Conflicts: lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java commit 8ec82a0d9a0dd946d96da20962bb2ea95758edbe Author: Mike McCandless Date: Thu Mar 10 13:56:25 2016 -0500 LUCENE-7086: move SlowCompositeReaderWrapper to misc module --- .../lucene50/TestLucene50DocValuesFormat.java | 2 +- .../ClassificationTestBase.java | 7 +- .../DocumentClassificationTestBase.java | 4 +- .../utils/DataSplitterTest.java | 5 +- .../apache/lucene/index/CompositeReader.java | 2 - .../org/apache/lucene/index/IndexReader.java | 2 - .../apache/lucene/index/MultiDocValues.java | 6 +- .../lucene54/TestLucene54DocValuesFormat.java | 6 +- .../index/TestBinaryDocValuesUpdates.java | 28 +++--- .../apache/lucene/index/TestCustomNorms.java | 4 +- .../index/TestDemoParallelLeafReader.java | 21 +---- .../lucene/index/TestDirectoryReader.java | 2 +- .../apache/lucene/index/TestDocValues.java | 14 +-- .../lucene/index/TestDocValuesIndexing.java | 25 +++-- .../lucene/index/TestDocsAndPositions.java | 4 +- .../lucene/index/TestDocumentWriter.java | 2 +- .../lucene/index/TestDuelingCodecs.java | 4 +- .../index/TestExitableDirectoryReader.java | 17 ++-- .../lucene/index/TestFilterLeafReader.java | 8 +- .../org/apache/lucene/index/TestFlex.java | 2 +- .../lucene/index/TestIndexReaderClose.java | 52 +---------- .../apache/lucene/index/TestIndexWriter.java | 6 +- .../lucene/index/TestLazyProxSkipping.java | 2 +- .../lucene/index/TestMultiDocValues.java | 16 ++-- .../lucene/index/TestMultiLevelSkipList.java | 2 +- .../org/apache/lucene/index/TestNorms.java | 4 +- .../index/TestNumericDocValuesUpdates.java | 30 +++--- .../apache/lucene/index/TestOmitNorms.java | 8 +- .../lucene/index/TestOmitPositions.java | 2 +- .../org/apache/lucene/index/TestOmitTf.java | 6 +- .../apache/lucene/index/TestOrdinalMap.java | 5 +- .../index/TestParallelCompositeReader.java | 34 +------ .../lucene/index/TestParallelLeafReader.java | 28 +++--- .../index/TestParallelReaderEmptyIndex.java | 22 ++--- .../lucene/index/TestParallelTermEnum.java | 4 +- .../org/apache/lucene/index/TestPayloads.java | 11 ++- .../apache/lucene/index/TestPointValues.java | 4 +- .../lucene/index/TestPostingsOffsets.java | 5 +- .../apache/lucene/index/TestReaderClosed.java | 6 +- .../index/TestReaderWrapperDVTypeCheck.java | 5 +- .../lucene/index/TestSegmentTermEnum.java | 2 +- .../lucene/index/TestStressAdvance.java | 2 +- .../apache/lucene/index/TestTermsEnum.java | 6 +- .../search/TestDisjunctionMaxQuery.java | 6 +- .../lucene/search/TestMinShouldMatch2.java | 2 +- .../lucene/search/TestMultiPhraseEnum.java | 8 +- .../apache/lucene/search/TestPhraseQuery.java | 4 +- .../lucene/search/TestPositionIncrement.java | 10 +- .../lucene/search/TestSimilarityProvider.java | 11 +-- .../apache/lucene/search/TestTermScorer.java | 7 +- .../TestUsageTrackingFilterCachingPolicy.java | 17 +++- .../spans/TestFieldMaskingSpanQuery.java | 11 ++- .../search/spans/TestNearSpansOrdered.java | 29 +++--- .../search/spans/TestSpanCollection.java | 9 +- .../search/spans/TestSpanContainQuery.java | 7 +- .../apache/lucene/search/spans/TestSpans.java | 13 +-- .../DefaultSortedSetDocValuesReaderState.java | 55 +++++++++-- .../TestSortedSetDocValuesFacets.java | 36 -------- .../lucene/search/grouping/TestGrouping.java | 12 +-- .../lucene/search/join/TestJoinUtil.java | 8 +- .../memory/TestMemoryIndexAgainstRAMDir.java | 12 +-- .../index/SlowCompositeReaderWrapper.java | 4 + .../index/TestSlowCompositeReaderWrapper.java | 91 +++++++++++++++++++ .../lucene/uninverting/TestDocTermOrds.java | 12 +-- .../lucene/uninverting/TestFieldCache.java | 10 +- .../uninverting/TestFieldCacheReopen.java | 2 +- .../TestFieldCacheVsDocValues.java | 2 +- .../TestFieldCacheWithThreads.java | 2 +- .../uninverting/TestLegacyFieldCache.java | 10 +- .../lucene/queries/CommonTermsQueryTest.java | 6 +- .../apache/lucene/queries/TermsQueryTest.java | 4 +- .../function/TestSortedSetFieldSource.java | 2 +- .../queries/payloads/PayloadHelper.java | 3 +- .../queries/payloads/TestPayloadSpans.java | 39 ++++---- .../payloads/TestPayloadTermQuery.java | 11 ++- .../index/BaseDocValuesFormatTestCase.java | 64 +++++++------ .../index/BaseIndexFileFormatTestCase.java | 6 +- .../index/BasePointsFormatTestCase.java | 4 +- .../index/BasePostingsFormatTestCase.java | 66 +++++++------- .../index/BaseStoredFieldsFormatTestCase.java | 2 +- .../index/BaseTermVectorsFormatTestCase.java | 12 +-- .../org/apache/lucene/search/QueryUtils.java | 6 +- .../search/spans/MultiSpansWrapper.java | 51 ----------- .../apache/lucene/util/LuceneTestCase.java | 53 +++++------ .../lucene/analysis/TestMockAnalyzer.java | 2 +- .../TestCompressingStoredFieldsFormat.java | 5 +- .../TestCompressingTermVectorsFormat.java | 8 +- .../lucene/index/TestAssertingLeafReader.java | 5 +- 88 files changed, 557 insertions(+), 609 deletions(-) rename lucene/{core => misc}/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (98%) create mode 100644 lucene/misc/src/test/org/apache/lucene/index/TestSlowCompositeReaderWrapper.java delete mode 100644 lucene/test-framework/src/java/org/apache/lucene/search/spans/MultiSpansWrapper.java diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java index 2c38728446ce..9f174dd23523 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java @@ -200,7 +200,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // now compare again after the merge ir = writer.getReader(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); Terms terms = ar.terms("indexed"); if (terms != null) { assertEquals(terms.size(), ar.getSortedSetDocValues("dv").getValueCount()); diff --git a/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java b/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java index 605b4905e801..331a74b70427 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java @@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -190,7 +189,8 @@ protected LeafReader getSampleIndex(Analyzer analyzer) throws IOException { indexWriter.addDocument(doc); indexWriter.commit(); - return SlowCompositeReaderWrapper.wrap(indexWriter.getReader()); + indexWriter.forceMerge(1); + return getOnlyLeafReader(indexWriter.getReader()); } protected LeafReader getRandomIndex(Analyzer analyzer, int size) throws IOException { @@ -213,7 +213,8 @@ protected LeafReader getRandomIndex(Analyzer analyzer, int size) throws IOExcept indexWriter.addDocument(doc); } indexWriter.commit(); - return SlowCompositeReaderWrapper.wrap(indexWriter.getReader()); + indexWriter.forceMerge(1); + return getOnlyLeafReader(indexWriter.getReader()); } private String createRandomString(Random random) { diff --git a/lucene/classification/src/test/org/apache/lucene/classification/document/DocumentClassificationTestBase.java b/lucene/classification/src/test/org/apache/lucene/classification/document/DocumentClassificationTestBase.java index 316802b0eee4..4193bde1679c 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/document/DocumentClassificationTestBase.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/document/DocumentClassificationTestBase.java @@ -30,7 +30,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.util.BytesRef; import org.junit.Before; @@ -202,7 +201,8 @@ protected LeafReader populateDocumentClassificationIndex(Analyzer analyzer) thro indexWriter.addDocument(doc); indexWriter.commit(); - return SlowCompositeReaderWrapper.wrap(indexWriter.getReader()); + indexWriter.forceMerge(1); + return getOnlyLeafReader(indexWriter.getReader()); } protected Document getVideoGameDocument() { diff --git a/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java b/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java index d69fefba1e84..2984bb55d450 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java @@ -27,7 +27,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.util.TestUtil; @@ -75,9 +74,9 @@ public void setUp() throws Exception { } indexWriter.commit(); + indexWriter.forceMerge(1); - originalIndex = SlowCompositeReaderWrapper.wrap(indexWriter.getReader()); - + originalIndex = getOnlyLeafReader(indexWriter.getReader()); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java index 0f6a44eb1e39..83bb92a82782 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java @@ -26,8 +26,6 @@ be used to get stored fields from the underlying LeafReaders, but it is not possible to directly retrieve postings. To do that, get the {@link LeafReaderContext} for all sub-readers via {@link #leaves()}. - Alternatively, you can mimic an {@link LeafReader} (with a serious slowdown), - by wrapping composite readers with {@link SlowCompositeReaderWrapper}.

      IndexReader instances for indexes on disk are usually constructed with a call to one of the static DirectoryReader.open() methods, diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java index 865f816bea20..976f548317bd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java @@ -56,8 +56,6 @@ be used to get stored fields from the underlying LeafReaders, but it is not possible to directly retrieve postings. To do that, get the sub-readers via {@link CompositeReader#getSequentialSubReaders}. - Alternatively, you can mimic an {@link LeafReader} (with a serious slowdown), - by wrapping composite readers with {@link SlowCompositeReaderWrapper}.

    IndexReader instances for indexes on disk are usually constructed diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java index 383139f45f84..33947974bcee 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java @@ -486,8 +486,8 @@ public static OrdinalMap build(Object owner, TermsEnum subs[], long[] weights, f private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OrdinalMap.class); - // cache key of whoever asked for this awful thing - final Object owner; + /** Cache key of whoever asked for this awful thing */ + public final Object owner; // globalOrd -> (globalOrd - segmentOrd) where segmentOrd is the the ordinal in the first segment that contains this term final PackedLongValues globalOrdDeltas; // globalOrd -> first segment container @@ -703,7 +703,7 @@ public static class MultiSortedSetDocValues extends SortedSetDocValues { LongValues currentGlobalOrds; /** Creates a new MultiSortedSetDocValues over values */ - MultiSortedSetDocValues(SortedSetDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException { + public MultiSortedSetDocValues(SortedSetDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException { assert docStarts.length == values.length + 1; this.values = values; this.docStarts = docStarts; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java index cede1d7efc38..b6f178dc061e 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene54/TestLucene54DocValuesFormat.java @@ -362,7 +362,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { // now compare again after the merge ir = writer.getReader(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); Terms terms = ar.terms("indexed"); if (terms != null) { assertEquals(terms.size(), ar.getSortedSetDocValues("dv").getValueCount()); @@ -541,7 +541,7 @@ public void testSortedSetAroundBlockSize() throws IOException { w.forceMerge(1); DirectoryReader r = DirectoryReader.open(w); w.close(); - SegmentReader sr = getOnlySegmentReader(r); + LeafReader sr = getOnlyLeafReader(r); assertEquals(maxDoc, sr.maxDoc()); SortedSetDocValues values = sr.getSortedSetDocValues("sset"); assertNotNull(values); @@ -591,7 +591,7 @@ public void testSortedNumericAroundBlockSize() throws IOException { w.forceMerge(1); DirectoryReader r = DirectoryReader.open(w); w.close(); - SegmentReader sr = getOnlySegmentReader(r); + LeafReader sr = getOnlyLeafReader(r); assertEquals(maxDoc, sr.maxDoc()); SortedNumericDocValues values = sr.getSortedNumericDocValues("snum"); assertNotNull(values); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java index 4025f58bcbc1..8dc8a3c8a031 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java @@ -253,16 +253,14 @@ public void testUpdatesAndDeletes() throws Exception { writer.close(); } - LeafReader slow = SlowCompositeReaderWrapper.wrap(reader); - - Bits liveDocs = slow.getLiveDocs(); + Bits liveDocs = MultiFields.getLiveDocs(reader); boolean[] expectedLiveDocs = new boolean[] { true, false, false, true, true, true }; for (int i = 0; i < expectedLiveDocs.length; i++) { assertEquals(expectedLiveDocs[i], liveDocs.get(i)); } long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17}; - BinaryDocValues bdv = slow.getBinaryDocValues("val"); + BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "val"); for (int i = 0; i < expectedValues.length; i++) { assertEquals(expectedValues[i], getValue(bdv, i)); } @@ -469,10 +467,9 @@ public DocValuesFormat getDocValuesFormatForField(String field) { final DirectoryReader reader = DirectoryReader.open(dir); - LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - BinaryDocValues bdv = r.getBinaryDocValues("bdv"); - SortedDocValues sdv = r.getSortedDocValues("sorted"); - for (int i = 0; i < r.maxDoc(); i++) { + BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv"); + SortedDocValues sdv = MultiDocValues.getSortedValues(reader, "sorted"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(17, getValue(bdv, i)); BytesRef term = sdv.get(i); assertEquals(new BytesRef("value"), term); @@ -499,9 +496,8 @@ public void testUpdateSameDocMultipleTimes() throws Exception { writer.close(); final DirectoryReader reader = DirectoryReader.open(dir); - final LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - BinaryDocValues bdv = r.getBinaryDocValues("bdv"); - for (int i = 0; i < r.maxDoc(); i++) { + BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(3, getValue(bdv, i)); } reader.close(); @@ -598,9 +594,8 @@ public void testUpdateDocumentByMultipleTerms() throws Exception { writer.close(); final DirectoryReader reader = DirectoryReader.open(dir); - final LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - BinaryDocValues bdv = r.getBinaryDocValues("bdv"); - for (int i = 0; i < r.maxDoc(); i++) { + BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(3, getValue(bdv, i)); } reader.close(); @@ -1018,9 +1013,8 @@ public DocValuesFormat getDocValuesFormatForField(String field) { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - BinaryDocValues f1 = r.getBinaryDocValues("f1"); - BinaryDocValues f2 = r.getBinaryDocValues("f2"); + BinaryDocValues f1 = MultiDocValues.getBinaryValues(reader, "f1"); + BinaryDocValues f2 = MultiDocValues.getBinaryValues(reader, "f2"); assertEquals(12L, getValue(f1, 0)); assertEquals(13L, getValue(f2, 0)); assertEquals(17L, getValue(f1, 1)); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java index 1d0ba540045b..c513093c42dc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java @@ -66,8 +66,8 @@ public void testFloatNorms() throws IOException { } writer.commit(); writer.close(); - LeafReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - NumericDocValues norms = open.getNormValues(floatTestField); + DirectoryReader open = DirectoryReader.open(dir); + NumericDocValues norms = MultiDocValues.getNormValues(open, floatTestField); assertNotNull(norms); for (int i = 0; i < open.maxDoc(); i++) { Document document = open.document(i); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java index 0034cee0c453..9f3339c8c692 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java @@ -405,13 +405,8 @@ LeafReader getParallelLeafReader(final LeafReader leaf, boolean doCache, long sc //TestUtil.checkIndex(dir); SegmentInfos infos = SegmentInfos.readLatestCommit(dir); - final LeafReader parLeafReader; - if (infos.size() == 1) { - parLeafReader = new SegmentReader(infos.info(0), IOContext.DEFAULT); - } else { - // This just means we didn't forceMerge above: - parLeafReader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - } + assert infos.size() == 1; + final LeafReader parLeafReader = new SegmentReader(infos.info(0), IOContext.DEFAULT); //checkParallelReader(leaf, parLeafReader, schemaGen); @@ -682,9 +677,7 @@ protected void reindex(long oldSchemaGen, long newSchemaGen, LeafReader reader, w.addDocument(newDoc); } - if (random().nextBoolean()) { - w.forceMerge(1); - } + w.forceMerge(1); w.close(); } @@ -750,9 +743,7 @@ protected void reindex(long oldSchemaGen, long newSchemaGen, LeafReader reader, } } - if (random().nextBoolean()) { - w.forceMerge(1); - } + w.forceMerge(1); w.close(); } @@ -845,9 +836,7 @@ protected void reindex(long oldSchemaGen, long newSchemaGen, LeafReader reader, } } - if (random().nextBoolean()) { - w.forceMerge(1); - } + w.forceMerge(1); w.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java index 24b2c5095e57..8e62094c991a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java @@ -761,7 +761,7 @@ public void testUniqueTermCount() throws Exception { writer.commit(); DirectoryReader r = DirectoryReader.open(dir); - LeafReader r1 = getOnlySegmentReader(r); + LeafReader r1 = getOnlyLeafReader(r); assertEquals(26, r1.terms("field").size()); assertEquals(10, r1.terms("number").size()); writer.addDocument(doc); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java index ad4d60cad9dc..2266caf19f00 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValues.java @@ -41,7 +41,7 @@ public void testEmptyIndex() throws Exception { IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null)); iw.addDocument(new Document()); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getBinary(r, "bogus")); @@ -66,7 +66,7 @@ public void testMisconfiguredField() throws Exception { doc.add(new StringField("foo", "bar", Field.Store.NO)); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // errors expectThrows(IllegalStateException.class, () -> { @@ -103,7 +103,7 @@ public void testNumericField() throws Exception { doc.add(new NumericDocValuesField("foo", 3)); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getNumeric(r, "foo")); @@ -136,7 +136,7 @@ public void testBinaryField() throws Exception { doc.add(new BinaryDocValuesField("foo", new BytesRef("bar"))); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getBinary(r, "foo")); @@ -171,7 +171,7 @@ public void testSortedField() throws Exception { doc.add(new SortedDocValuesField("foo", new BytesRef("bar"))); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getBinary(r, "foo")); @@ -202,7 +202,7 @@ public void testSortedSetField() throws Exception { doc.add(new SortedSetDocValuesField("foo", new BytesRef("bar"))); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getSortedSet(r, "foo")); @@ -237,7 +237,7 @@ public void testSortedNumericField() throws Exception { doc.add(new SortedNumericDocValuesField("foo", 3)); iw.addDocument(doc); DirectoryReader dr = DirectoryReader.open(iw); - LeafReader r = getOnlySegmentReader(dr); + LeafReader r = getOnlyLeafReader(dr); // ok assertNotNull(DocValues.getSortedNumeric(r, "foo")); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java index 44b5b75bbe30..2e0cbd901f53 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java @@ -58,7 +58,7 @@ public void testAddIndexes() throws IOException { doc.add(newStringField("id", "1", Field.Store.YES)); doc.add(new NumericDocValuesField("dv", 1)); w.addDocument(doc); - IndexReader r1 = w.getReader(); + DirectoryReader r1 = w.getReader(); w.close(); Directory d2 = newDirectory(); @@ -67,12 +67,12 @@ public void testAddIndexes() throws IOException { doc.add(newStringField("id", "2", Field.Store.YES)); doc.add(new NumericDocValuesField("dv", 2)); w.addDocument(doc); - IndexReader r2 = w.getReader(); + DirectoryReader r2 = w.getReader(); w.close(); Directory d3 = newDirectory(); w = new RandomIndexWriter(random(), d3); - w.addIndexes(SlowCodecReaderWrapper.wrap(SlowCompositeReaderWrapper.wrap(r1)), SlowCodecReaderWrapper.wrap(SlowCompositeReaderWrapper.wrap(r2))); + w.addIndexes(SlowCodecReaderWrapper.wrap(getOnlyLeafReader(r1)), SlowCodecReaderWrapper.wrap(getOnlyLeafReader(r2))); r1.close(); d1.close(); r2.close(); @@ -81,7 +81,7 @@ public void testAddIndexes() throws IOException { w.forceMerge(1); DirectoryReader r3 = w.getReader(); w.close(); - LeafReader sr = getOnlySegmentReader(r3); + LeafReader sr = getOnlyLeafReader(r3); assertEquals(2, sr.numDocs()); NumericDocValues docValues = sr.getNumericDocValues("dv"); assertNotNull(docValues); @@ -109,7 +109,7 @@ public void testMultiValuedDocValuesField() throws Exception { DirectoryReader r = w.getReader(); w.close(); - assertEquals(17, DocValues.getNumeric(getOnlySegmentReader(r), "field").get(0)); + assertEquals(17, DocValues.getNumeric(getOnlyLeafReader(r), "field").get(0)); r.close(); d.close(); } @@ -130,7 +130,7 @@ public void testDifferentTypedDocValuesField() throws Exception { DirectoryReader r = w.getReader(); w.close(); - assertEquals(17, DocValues.getNumeric(getOnlySegmentReader(r), "field").get(0)); + assertEquals(17, DocValues.getNumeric(getOnlyLeafReader(r), "field").get(0)); r.close(); d.close(); } @@ -150,7 +150,7 @@ public void testDifferentTypedDocValuesField2() throws Exception { }); DirectoryReader r = w.getReader(); - assertEquals(17, getOnlySegmentReader(r).getNumericDocValues("field").get(0)); + assertEquals(17, getOnlyLeafReader(r).getNumericDocValues("field").get(0)); r.close(); w.close(); d.close(); @@ -171,7 +171,7 @@ public void testLengthPrefixAcrossTwoPages() throws Exception { w.addDocument(doc); w.forceMerge(1); DirectoryReader r = w.getReader(); - BinaryDocValues s = DocValues.getSorted(getOnlySegmentReader(r), "field"); + BinaryDocValues s = DocValues.getSorted(getOnlyLeafReader(r), "field"); BytesRef bytes1 = s.get(0); assertEquals(bytes.length, bytes1.length); @@ -199,19 +199,18 @@ public void testDocValuesUnstored() throws IOException { writer.addDocument(doc); } DirectoryReader r = writer.getReader(); - LeafReader slow = SlowCompositeReaderWrapper.wrap(r); - FieldInfos fi = slow.getFieldInfos(); + FieldInfos fi = MultiFields.getMergedFieldInfos(r); FieldInfo dvInfo = fi.fieldInfo("dv"); assertTrue(dvInfo.getDocValuesType() != DocValuesType.NONE); - NumericDocValues dv = slow.getNumericDocValues("dv"); + NumericDocValues dv = MultiDocValues.getNumericValues(r, "dv"); for (int i = 0; i < 50; i++) { assertEquals(i, dv.get(i)); - Document d = slow.document(i); + Document d = r.document(i); // cannot use d.get("dv") due to another bug! assertNull(d.getField("dv")); assertEquals(Integer.toString(i), d.get("docId")); } - slow.close(); + r.close(); writer.close(); dir.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java index dc49db154e3d..2ef5824c5747 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java @@ -335,7 +335,7 @@ public void testDocsEnumStart() throws Exception { doc.add(newStringField("foo", "bar", Field.Store.NO)); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - LeafReader r = getOnlySegmentReader(reader); + LeafReader r = getOnlyLeafReader(reader); PostingsEnum disi = TestUtil.docs(random(), r, "foo", new BytesRef("bar"), null, PostingsEnum.NONE); int docid = disi.docID(); assertEquals(-1, docid); @@ -360,7 +360,7 @@ public void testDocsAndPositionsEnumStart() throws Exception { doc.add(newTextField("foo", "bar", Field.Store.NO)); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - LeafReader r = getOnlySegmentReader(reader); + LeafReader r = getOnlyLeafReader(reader); PostingsEnum disi = r.postings(new Term("foo", "bar"), PostingsEnum.ALL); int docid = disi.docID(); assertEquals(-1, docid); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java index 489a185b326e..a814c4c351dc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java @@ -284,7 +284,7 @@ public void testLUCENE_1590() throws Exception { TestUtil.checkIndex(dir); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(dir)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(dir)); FieldInfos fi = reader.getFieldInfos(); // f1 assertFalse("f1 should have no norms", fi.fieldInfo("f1").hasNorms()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java index b79e638ba078..62fe28aedb24 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java @@ -162,8 +162,8 @@ public void testCrazyReaderEquals() throws IOException { createRandomIndex(numdocs, leftWriter, seed); createRandomIndex(numdocs, rightWriter, seed); - leftReader = wrapReader(leftWriter.getReader(), false); - rightReader = wrapReader(rightWriter.getReader(), false); + leftReader = wrapReader(leftWriter.getReader()); + rightReader = wrapReader(rightWriter.getReader()); // check that our readers are valid TestUtil.checkReader(leftReader); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java index 36c9e0d80a1c..65cf84e6d9c9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java @@ -78,8 +78,8 @@ public BytesRef next() throws IOException { } } - public TestReader(IndexReader reader) throws IOException { - super(SlowCompositeReaderWrapper.wrap(reader)); + public TestReader(LeafReader reader) throws IOException { + super(reader); } @Override @@ -107,6 +107,7 @@ public void testExitableFilterIndexReader() throws Exception { Document d3 = new Document(); d3.add(newTextField("default", "ones two four", Field.Store.YES)); writer.addDocument(d3); + writer.forceMerge(1); writer.commit(); writer.close(); @@ -122,43 +123,39 @@ public void testExitableFilterIndexReader() throws Exception { // Not checking the validity of the result, all we are bothered about in this test is the timing out. directoryReader = DirectoryReader.open(directory); exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(1000)); - reader = new TestReader(exitableDirectoryReader); + reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); searcher = new IndexSearcher(reader); searcher.search(query, 10); reader.close(); - exitableDirectoryReader.close(); // Set a really low timeout value (1 millisecond) and expect an Exception directoryReader = DirectoryReader.open(directory); exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(1)); - reader = new TestReader(exitableDirectoryReader); + reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); IndexSearcher slowSearcher = new IndexSearcher(reader); expectThrows(ExitingReaderException.class, () -> { slowSearcher.search(query, 10); }); reader.close(); - exitableDirectoryReader.close(); // Set maximum time out and expect the query to complete. // Not checking the validity of the result, all we are bothered about in this test is the timing out. directoryReader = DirectoryReader.open(directory); exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(Long.MAX_VALUE)); - reader = new TestReader(exitableDirectoryReader); + reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); searcher = new IndexSearcher(reader); searcher.search(query, 10); reader.close(); - exitableDirectoryReader.close(); // Set a negative time allowed and expect the query to complete (should disable timeouts) // Not checking the validity of the result, all we are bothered about in this test is the timing out. directoryReader = DirectoryReader.open(directory); exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, new QueryTimeoutImpl(-189034L)); - reader = new TestReader(exitableDirectoryReader); + reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); searcher = new IndexSearcher(reader); searcher.search(query, 10); reader.close(); - exitableDirectoryReader.close(); directory.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java index cad47a4e1268..e9f6fe28854e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java @@ -98,8 +98,8 @@ public int nextDoc() throws IOException { } } - public TestReader(IndexReader reader) throws IOException { - super(SlowCompositeReaderWrapper.wrap(reader)); + public TestReader(LeafReader reader) throws IOException { + super(reader); } @Override @@ -128,7 +128,7 @@ public void testFilterIndexReader() throws Exception { Document d3 = new Document(); d3.add(newTextField("default", "two four", Field.Store.YES)); writer.addDocument(d3); - + writer.forceMerge(1); writer.close(); Directory target = newDirectory(); @@ -137,7 +137,7 @@ public void testFilterIndexReader() throws Exception { ((BaseDirectoryWrapper) target).setCrossCheckTermVectorsOnClose(false); writer = new IndexWriter(target, newIndexWriterConfig(new MockAnalyzer(random()))); - try (LeafReader reader = new TestReader(DirectoryReader.open(directory))) { + try (LeafReader reader = new TestReader(getOnlyLeafReader(DirectoryReader.open(directory)))) { writer.addIndexes(SlowCodecReaderWrapper.wrap(reader)); } writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java index 3d716e13fefa..d91301fa6acd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFlex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFlex.java @@ -70,7 +70,7 @@ public void testTermOrd() throws Exception { w.addDocument(doc); w.forceMerge(1); DirectoryReader r = w.getReader(); - TermsEnum terms = getOnlySegmentReader(r).fields().terms("f").iterator(); + TermsEnum terms = getOnlyLeafReader(r).fields().terms("f").iterator(); assertTrue(terms.next() != null); try { assertEquals(0, terms.ord()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java index 6b22fd711e20..91dcb6ea6766 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java @@ -37,14 +37,15 @@ public class TestIndexReaderClose extends LuceneTestCase { public void testCloseUnderException() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), new MockAnalyzer(random()))); + writer.addDocument(new Document()); writer.commit(); writer.close(); final int iters = 1000 + 1 + random().nextInt(20); for (int j = 0; j < iters; j++) { DirectoryReader open = DirectoryReader.open(dir); final boolean throwOnClose = !rarely(); - LeafReader wrap = SlowCompositeReaderWrapper.wrap(open); - FilterLeafReader reader = new FilterLeafReader(wrap) { + LeafReader leaf = getOnlyLeafReader(open); + FilterLeafReader reader = new FilterLeafReader(leaf) { @Override protected void doClose() throws IOException { super.doClose(); @@ -87,54 +88,10 @@ protected void doClose() throws IOException { reader.close(); // call it again } assertEquals(0, count.get()); - wrap.close(); } dir.close(); } - public void testCoreListenerOnSlowCompositeReaderWrapper() throws IOException { - RandomIndexWriter w = new RandomIndexWriter(random(), newDirectory()); - final int numDocs = TestUtil.nextInt(random(), 1, 5); - for (int i = 0; i < numDocs; ++i) { - w.addDocument(new Document()); - if (random().nextBoolean()) { - w.commit(); - } - } - w.commit(); - w.close(); - - final IndexReader reader = DirectoryReader.open(w.w.getDirectory()); - final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader); - - final int numListeners = TestUtil.nextInt(random(), 1, 10); - final List listeners = new ArrayList<>(); - AtomicInteger counter = new AtomicInteger(numListeners); - - for (int i = 0; i < numListeners; ++i) { - CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheKey()); - listeners.add(listener); - leafReader.addCoreClosedListener(listener); - } - for (int i = 0; i < 100; ++i) { - leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size()))); - } - final int removed = random().nextInt(numListeners); - Collections.shuffle(listeners, random()); - for (int i = 0; i < removed; ++i) { - leafReader.removeCoreClosedListener(listeners.get(i)); - } - assertEquals(numListeners, counter.get()); - // make sure listeners are registered on the wrapped reader and that closing any of them has the same effect - if (random().nextBoolean()) { - reader.close(); - } else { - leafReader.close(); - } - assertEquals(removed, counter.get()); - w.w.getDirectory().close(); - } - public void testCoreListenerOnWrapperWithDifferentCacheKey() throws IOException { RandomIndexWriter w = new RandomIndexWriter(random(), newDirectory()); final int numDocs = TestUtil.nextInt(random(), 1, 5); @@ -144,13 +101,14 @@ public void testCoreListenerOnWrapperWithDifferentCacheKey() throws IOException w.commit(); } } + w.forceMerge(1); w.commit(); w.close(); final IndexReader reader = DirectoryReader.open(w.w.getDirectory()); // We explicitly define a different cache key final Object coreCacheKey = new Object(); - final LeafReader leafReader = new FilterLeafReader(SlowCompositeReaderWrapper.wrap(reader)) { + final LeafReader leafReader = new FilterLeafReader(getOnlyLeafReader(reader)) { @Override public Object getCoreCacheKey() { return coreCacheKey; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 7461618bc55a..2c3543e04386 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -692,7 +692,7 @@ public void testEmptyFieldNameTerms() throws IOException { writer.addDocument(doc); writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - LeafReader subreader = getOnlySegmentReader(reader); + LeafReader subreader = getOnlyLeafReader(reader); TermsEnum te = subreader.fields().terms("").iterator(); assertEquals(new BytesRef("a"), te.next()); assertEquals(new BytesRef("b"), te.next()); @@ -713,7 +713,7 @@ public void testEmptyFieldNameWithEmptyTerm() throws IOException { writer.addDocument(doc); writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - LeafReader subreader = getOnlySegmentReader(reader); + LeafReader subreader = getOnlyLeafReader(reader); TermsEnum te = subreader.fields().terms("").iterator(); assertEquals(new BytesRef(""), te.next()); assertEquals(new BytesRef("a"), te.next()); @@ -2549,7 +2549,7 @@ public void testEmptyNorm() throws Exception { w.commit(); w.close(); DirectoryReader r = DirectoryReader.open(d); - assertEquals(0, getOnlySegmentReader(r).getNormValues("foo").get(0)); + assertEquals(0, getOnlyLeafReader(r).getNormValues("foo").get(0)); r.close(); d.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java index a928fd2b1023..ff79e5e4113b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java @@ -104,7 +104,7 @@ public TokenStreamComponents createComponents(String fieldName) { writer.forceMerge(1); writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(directory)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(directory)); this.searcher = newSearcher(reader); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java index 121e85c34755..5b70c38a7fa3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java @@ -56,7 +56,7 @@ public void testNumerics() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); NumericDocValues multi = MultiDocValues.getNumericValues(ir, "numbers"); @@ -91,7 +91,7 @@ public void testBinary() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); BinaryDocValues multi = MultiDocValues.getBinaryValues(ir, "bytes"); @@ -131,7 +131,7 @@ public void testSorted() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes"); @@ -173,7 +173,7 @@ public void testSortedWithLotsOfDups() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes"); @@ -214,7 +214,7 @@ public void testSortedSet() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); SortedSetDocValues multi = MultiDocValues.getSortedSetValues(ir, "bytes"); @@ -276,7 +276,7 @@ public void testSortedSetWithDups() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); SortedSetDocValues multi = MultiDocValues.getSortedSetValues(ir, "bytes"); @@ -337,7 +337,7 @@ public void testSortedNumeric() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); SortedNumericDocValues multi = MultiDocValues.getSortedNumericValues(ir, "nums"); @@ -388,7 +388,7 @@ public void testDocsWithField() throws Exception { DirectoryReader ir = iw.getReader(); iw.forceMerge(1); DirectoryReader ir2 = iw.getReader(); - LeafReader merged = getOnlySegmentReader(ir2); + LeafReader merged = getOnlyLeafReader(ir2); iw.close(); Bits multi = MultiDocValues.getDocsWithField(ir, "numbers"); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java index a82444bf9ea7..bc14cb8a690d 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java @@ -80,7 +80,7 @@ public void testSimpleSkip() throws IOException { writer.forceMerge(1); writer.close(); - LeafReader reader = getOnlySegmentReader(DirectoryReader.open(dir)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(dir)); for (int i = 0; i < 2; i++) { counter = 0; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java index 562cefb77740..45db69a5ef10 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java @@ -111,8 +111,8 @@ public void testCustomEncoder() throws Exception { public void testMaxByteNorms() throws IOException { Directory dir = newFSDirectory(createTempDir("TestNorms.testMaxByteNorms")); buildIndex(dir); - LeafReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - NumericDocValues normValues = open.getNormValues(byteTestField); + DirectoryReader open = DirectoryReader.open(dir); + NumericDocValues normValues = MultiDocValues.getNormValues(open, byteTestField); assertNotNull(normValues); for (int i = 0; i < open.maxDoc(); i++) { Document document = open.document(i); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java index 727f6ff7fcbe..15ecc0f8b008 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java @@ -238,16 +238,14 @@ public void testUpdatesAndDeletes() throws Exception { writer.close(); } - LeafReader slow = SlowCompositeReaderWrapper.wrap(reader); - - Bits liveDocs = slow.getLiveDocs(); + Bits liveDocs = MultiFields.getLiveDocs(reader); boolean[] expectedLiveDocs = new boolean[] { true, false, false, true, true, true }; for (int i = 0; i < expectedLiveDocs.length; i++) { assertEquals(expectedLiveDocs[i], liveDocs.get(i)); } long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17}; - NumericDocValues ndv = slow.getNumericDocValues("val"); + NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "val"); for (int i = 0; i < expectedValues.length; i++) { assertEquals(expectedValues[i], ndv.get(i)); } @@ -460,10 +458,9 @@ public DocValuesFormat getDocValuesFormatForField(String field) { final DirectoryReader reader = DirectoryReader.open(dir); - LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - NumericDocValues ndv = r.getNumericDocValues("ndv"); - SortedDocValues sdv = r.getSortedDocValues("sorted"); - for (int i = 0; i < r.maxDoc(); i++) { + NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv"); + SortedDocValues sdv = MultiDocValues.getSortedValues(reader, "sorted"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(17, ndv.get(i)); final BytesRef term = sdv.get(i); assertEquals(new BytesRef("value"), term); @@ -491,9 +488,8 @@ public void testUpdateSameDocMultipleTimes() throws Exception { writer.close(); final DirectoryReader reader = DirectoryReader.open(dir); - final LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - NumericDocValues ndv = r.getNumericDocValues("ndv"); - for (int i = 0; i < r.maxDoc(); i++) { + NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(3, ndv.get(i)); } reader.close(); @@ -592,9 +588,8 @@ public void testUpdateDocumentByMultipleTerms() throws Exception { writer.close(); final DirectoryReader reader = DirectoryReader.open(dir); - final LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - NumericDocValues ndv = r.getNumericDocValues("ndv"); - for (int i = 0; i < r.maxDoc(); i++) { + NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv"); + for (int i = 0; i < reader.maxDoc(); i++) { assertEquals(3, ndv.get(i)); } reader.close(); @@ -806,7 +801,7 @@ public void testUpdateSegmentWithNoDocValues2() throws Exception { writer.close(); reader = DirectoryReader.open(dir); - LeafReader ar = getOnlySegmentReader(reader); + LeafReader ar = getOnlyLeafReader(reader); assertEquals(DocValuesType.NUMERIC, ar.getFieldInfos().fieldInfo("foo").getDocValuesType()); IndexSearcher searcher = new IndexSearcher(reader); TopFieldDocs td; @@ -1103,9 +1098,8 @@ public DocValuesFormat getDocValuesFormatForField(String field) { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - LeafReader r = SlowCompositeReaderWrapper.wrap(reader); - NumericDocValues f1 = r.getNumericDocValues("f1"); - NumericDocValues f2 = r.getNumericDocValues("f2"); + NumericDocValues f1 = MultiDocValues.getNumericValues(reader, "f1"); + NumericDocValues f2 = MultiDocValues.getNumericValues(reader, "f2"); assertEquals(12L, f1.get(0)); assertEquals(13L, f2.get(0)); assertEquals(17L, f1.get(1)); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java index 83dfd193b91f..bc3c3e83be30 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java @@ -66,7 +66,7 @@ public void testOmitNorms() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -120,7 +120,7 @@ public void testMixedMerge() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -168,7 +168,7 @@ public void testMixedRAM() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitsNorms()); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitsNorms()); @@ -297,7 +297,7 @@ NumericDocValues getNorms(String field, Field f1, Field f2) throws IOException { // fully merge and validate MultiNorms against single segment. riw.forceMerge(1); DirectoryReader ir2 = riw.getReader(); - NumericDocValues norms2 = getOnlySegmentReader(ir2).getNormValues(field); + NumericDocValues norms2 = getOnlyLeafReader(ir2).getNormValues(field); if (norms1 == null) { assertNull(norms2); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java index 38c725118034..f5a74b56a50e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java @@ -153,7 +153,7 @@ public void testPositions() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); // docs + docs = docs assertEquals(IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java index 807c704ca0ff..3c12a0db4509 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java @@ -105,7 +105,7 @@ public void testOmitTermFreqAndPositions() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions()); @@ -157,7 +157,7 @@ public void testMixedMerge() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions()); @@ -200,7 +200,7 @@ public void testMixedRAM() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").getIndexOptions()); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java b/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java index e0fab18573b3..1f9ff118d689 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java @@ -83,13 +83,12 @@ public void testRamBytesUsed() throws IOException { } iw.commit(); DirectoryReader r = iw.getReader(); - LeafReader ar = SlowCompositeReaderWrapper.wrap(r); - SortedDocValues sdv = ar.getSortedDocValues("sdv"); + SortedDocValues sdv = MultiDocValues.getSortedValues(r, "sdv"); if (sdv instanceof MultiSortedDocValues) { OrdinalMap map = ((MultiSortedDocValues) sdv).mapping; assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed()); } - SortedSetDocValues ssdv = ar.getSortedSetDocValues("ssdv"); + SortedSetDocValues ssdv = MultiDocValues.getSortedSetValues(r, "ssdv"); if (ssdv instanceof MultiSortedSetDocValues) { OrdinalMap map = ((MultiSortedSetDocValues) ssdv).mapping; assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java index 7078380c9381..166c9e4ded9b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java @@ -18,6 +18,8 @@ import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; @@ -276,32 +278,6 @@ public void testIncompatibleIndexes2() throws IOException { dir2.close(); } - public void testIncompatibleIndexes3() throws IOException { - Directory dir1 = getDir1(random()); - Directory dir2 = getDir2(random()); - - CompositeReader ir1 = new MultiReader(DirectoryReader.open(dir1), SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1))), - ir2 = new MultiReader(DirectoryReader.open(dir2), DirectoryReader.open(dir2)); - CompositeReader[] readers = new CompositeReader[] {ir1, ir2}; - - expectThrows(IllegalArgumentException.class, () -> { - new ParallelCompositeReader(readers); - }); - - expectThrows(IllegalArgumentException.class, () -> { - new ParallelCompositeReader(random().nextBoolean(), readers, readers); - }); - - assertEquals(1, ir1.getRefCount()); - assertEquals(1, ir2.getRefCount()); - ir1.close(); - ir2.close(); - assertEquals(0, ir1.getRefCount()); - assertEquals(0, ir2.getRefCount()); - dir1.close(); - dir2.close(); - } - public void testIgnoreStoredFields() throws IOException { Directory dir1 = getDir1(random()); Directory dir2 = getDir2(random()); @@ -317,7 +293,7 @@ public void testIgnoreStoredFields() throws IOException { assertNull(pr.document(0).get("f3")); assertNull(pr.document(0).get("f4")); // check that fields are there - LeafReader slow = SlowCompositeReaderWrapper.wrap(pr); + Fields slow = MultiFields.getFields(pr); assertNotNull(slow.terms("f1")); assertNotNull(slow.terms("f2")); assertNotNull(slow.terms("f3")); @@ -333,7 +309,7 @@ public void testIgnoreStoredFields() throws IOException { assertNull(pr.document(0).get("f3")); assertNull(pr.document(0).get("f4")); // check that fields are there - slow = SlowCompositeReaderWrapper.wrap(pr); + slow = MultiFields.getFields(pr); assertNull(slow.terms("f1")); assertNull(slow.terms("f2")); assertNotNull(slow.terms("f3")); @@ -349,7 +325,7 @@ public void testIgnoreStoredFields() throws IOException { assertNull(pr.document(0).get("f3")); assertNull(pr.document(0).get("f4")); // check that fields are there - slow = SlowCompositeReaderWrapper.wrap(pr); + slow = MultiFields.getFields(pr); assertNull(slow.terms("f1")); assertNull(slow.terms("f2")); assertNotNull(slow.terms("f3")); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java index 31aa603b4741..f7f401fca9a1 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java @@ -63,8 +63,8 @@ public void testQueries() throws Exception { public void testFieldNames() throws Exception { Directory dir1 = getDir1(random()); Directory dir2 = getDir2(random()); - ParallelLeafReader pr = new ParallelLeafReader(SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)), - SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2))); + ParallelLeafReader pr = new ParallelLeafReader(getOnlyLeafReader(DirectoryReader.open(dir1)), + getOnlyLeafReader(DirectoryReader.open(dir2))); FieldInfos fieldInfos = pr.getFieldInfos(); assertEquals(4, fieldInfos.size()); assertNotNull(fieldInfos.fieldInfo("f1")); @@ -81,8 +81,8 @@ public void testRefCounts1() throws IOException { Directory dir2 = getDir2(random()); LeafReader ir1, ir2; // close subreaders, ParallelReader will not change refCounts, but close on its own close - ParallelLeafReader pr = new ParallelLeafReader(ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)), - ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2))); + ParallelLeafReader pr = new ParallelLeafReader(ir1 = getOnlyLeafReader(DirectoryReader.open(dir1)), + ir2 = getOnlyLeafReader(DirectoryReader.open(dir2))); // check RefCounts assertEquals(1, ir1.getRefCount()); @@ -97,8 +97,8 @@ public void testRefCounts1() throws IOException { public void testRefCounts2() throws IOException { Directory dir1 = getDir1(random()); Directory dir2 = getDir2(random()); - LeafReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)); - LeafReader ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2)); + LeafReader ir1 = getOnlyLeafReader(DirectoryReader.open(dir1)); + LeafReader ir2 = getOnlyLeafReader(DirectoryReader.open(dir2)); // don't close subreaders, so ParallelReader will increment refcounts ParallelLeafReader pr = new ParallelLeafReader(false, ir1, ir2); // check RefCounts @@ -117,7 +117,7 @@ public void testRefCounts2() throws IOException { public void testCloseInnerReader() throws Exception { Directory dir1 = getDir1(random()); - LeafReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)); + LeafReader ir1 = getOnlyLeafReader(DirectoryReader.open(dir1)); // with overlapping ParallelLeafReader pr = new ParallelLeafReader(true, @@ -149,8 +149,8 @@ public void testIncompatibleIndexes() throws IOException { w2.addDocument(d3); w2.close(); - LeafReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)); - LeafReader ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2)); + LeafReader ir1 = getOnlyLeafReader(DirectoryReader.open(dir1)); + LeafReader ir2 = getOnlyLeafReader(DirectoryReader.open(dir2)); // indexes don't have the same number of documents expectThrows(IllegalArgumentException.class, () -> { @@ -175,8 +175,8 @@ public void testIncompatibleIndexes() throws IOException { public void testIgnoreStoredFields() throws IOException { Directory dir1 = getDir1(random()); Directory dir2 = getDir2(random()); - LeafReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)); - LeafReader ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2)); + LeafReader ir1 = getOnlyLeafReader(DirectoryReader.open(dir1)); + LeafReader ir2 = getOnlyLeafReader(DirectoryReader.open(dir2)); // with overlapping ParallelLeafReader pr = new ParallelLeafReader(false, @@ -276,8 +276,8 @@ private IndexSearcher parallel(Random random) throws IOException { dir1 = getDir1(random); dir2 = getDir2(random); ParallelLeafReader pr = new ParallelLeafReader( - SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1)), - SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2))); + getOnlyLeafReader(DirectoryReader.open(dir1)), + getOnlyLeafReader(DirectoryReader.open(dir2))); TestUtil.checkReader(pr); return newSearcher(pr); } @@ -293,6 +293,7 @@ private Directory getDir1(Random random) throws IOException { d2.add(newTextField("f1", "v2", Field.Store.YES)); d2.add(newTextField("f2", "v2", Field.Store.YES)); w1.addDocument(d2); + w1.forceMerge(1); w1.close(); return dir1; } @@ -308,6 +309,7 @@ private Directory getDir2(Random random) throws IOException { d4.add(newTextField("f3", "v2", Field.Store.YES)); d4.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d4); + w2.forceMerge(1); w2.close(); return dir2; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java index 373a125eeb6f..61c84dccab88 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java @@ -50,15 +50,7 @@ public void testEmptyIndex() throws IOException { IndexWriter iwOut = new IndexWriter(rdOut, newIndexWriterConfig(new MockAnalyzer(random()))); - ParallelLeafReader apr = new ParallelLeafReader( - SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd1)), - SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd2))); - - // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum) - iwOut.addIndexes(SlowCodecReaderWrapper.wrap(apr)); - iwOut.forceMerge(1); - - // 2nd try with a readerless parallel reader + // add a readerless parallel reader iwOut.addIndexes(SlowCodecReaderWrapper.wrap(new ParallelLeafReader())); iwOut.forceMerge(1); @@ -136,16 +128,18 @@ public void testEmptyIndexWithVectors() throws IOException { Directory rdOut = newDirectory(); IndexWriter iwOut = new IndexWriter(rdOut, newIndexWriterConfig(new MockAnalyzer(random()))); - final DirectoryReader reader1, reader2; - ParallelLeafReader pr = new ParallelLeafReader( - SlowCompositeReaderWrapper.wrap(reader1 = DirectoryReader.open(rd1)), - SlowCompositeReaderWrapper.wrap(reader2 = DirectoryReader.open(rd2))); + DirectoryReader reader1 = DirectoryReader.open(rd1); + DirectoryReader reader2 = DirectoryReader.open(rd2); + ParallelLeafReader pr = new ParallelLeafReader(false, + getOnlyLeafReader(reader1), + getOnlyLeafReader(reader2)); // When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter) iwOut.addIndexes(SlowCodecReaderWrapper.wrap(pr)); - // ParallelReader closes any IndexReader you added to it: pr.close(); + reader1.close(); + reader2.close(); // assert subreaders were closed assertEquals(0, reader1.getRefCount()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java index c51fd2d7280c..a83c5492defc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java @@ -59,8 +59,8 @@ public void setUp() throws Exception { iw2.close(); - this.ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd1)); - this.ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd2)); + this.ir1 = getOnlyLeafReader(DirectoryReader.open(rd1)); + this.ir2 = getOnlyLeafReader(DirectoryReader.open(rd2)); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java index 3fa213109a67..eca293ed46f8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java @@ -85,7 +85,7 @@ public void testPayloadFieldBit() throws Exception { // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram)); FieldInfos fi = reader.getFieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").hasPayloads()); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").hasPayloads()); @@ -112,7 +112,7 @@ public void testPayloadFieldBit() throws Exception { // flush writer.close(); - reader = getOnlySegmentReader(DirectoryReader.open(ram)); + reader = getOnlyLeafReader(DirectoryReader.open(ram)); fi = reader.getFieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").hasPayloads()); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").hasPayloads()); @@ -603,8 +603,9 @@ public void testMixupDocs() throws Exception { field.setTokenStream(ts); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - LeafReader sr = SlowCompositeReaderWrapper.wrap(reader); - PostingsEnum de = sr.postings(new Term("field", "withPayload"), PostingsEnum.PAYLOADS); + TermsEnum te = MultiFields.getFields(reader).terms("field").iterator(); + assertTrue(te.seekExact(new BytesRef("withPayload"))); + PostingsEnum de = te.postings(null, PostingsEnum.PAYLOADS); de.nextDoc(); de.nextPosition(); assertEquals(new BytesRef("test"), de.getPayload()); @@ -637,7 +638,7 @@ public void testMixupMultiValued() throws Exception { doc.add(field3); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - SegmentReader sr = getOnlySegmentReader(reader); + LeafReader sr = getOnlyLeafReader(reader); PostingsEnum de = sr.postings(new Term("field", "withPayload"), PostingsEnum.PAYLOADS); de.nextDoc(); de.nextPosition(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 9b18f0298ee3..094623491152 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -182,7 +182,7 @@ public void testIllegalDimChangeViaAddIndexesCodecReader() throws Exception { w2.addDocument(doc); DirectoryReader r = DirectoryReader.open(dir); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - w2.addIndexes(new CodecReader[] {getOnlySegmentReader(r)}); + w2.addIndexes(new CodecReader[] {(CodecReader) getOnlyLeafReader(r)}); }); assertEquals("cannot change point dimension count from 2 to 1 for field=\"dim\"", expected.getMessage()); @@ -331,7 +331,7 @@ public void testIllegalNumBytesChangeViaAddIndexesCodecReader() throws Exception w2.addDocument(doc); DirectoryReader r = DirectoryReader.open(dir); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - w2.addIndexes(new CodecReader[] {getOnlySegmentReader(r)}); + w2.addIndexes(new CodecReader[] {(CodecReader) getOnlyLeafReader(r)}); }); assertEquals("cannot change point numBytes from 6 to 4 for field=\"dim\"", expected.getMessage()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java index aff0bd9f06ff..b21cb2384f0f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java @@ -375,10 +375,9 @@ public void testWithUnindexedFields() throws Exception { riw.addDocument(doc); } CompositeReader ir = riw.getReader(); - LeafReader slow = SlowCompositeReaderWrapper.wrap(ir); - FieldInfos fis = slow.getFieldInfos(); + FieldInfos fis = MultiFields.getMergedFieldInfos(ir); assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, fis.fieldInfo("foo").getIndexOptions()); - slow.close(); + ir.close(); ir.close(); riw.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java index 6a1ab3a4018f..401254b8d66b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderClosed.java @@ -32,7 +32,7 @@ import org.apache.lucene.util.TestUtil; public class TestReaderClosed extends LuceneTestCase { - private IndexReader reader; + private DirectoryReader reader; private Directory dir; @Override @@ -54,6 +54,7 @@ public void setUp() throws Exception { field.setStringValue(TestUtil.randomUnicodeString(random(), 10)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); } @@ -77,8 +78,7 @@ public void test() throws Exception { // LUCENE-3800 public void testReaderChaining() throws Exception { assertTrue(reader.getRefCount() > 0); - IndexReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader); - wrappedReader = new ParallelLeafReader((LeafReader) wrappedReader); + LeafReader wrappedReader = new ParallelLeafReader(getOnlyLeafReader(reader)); IndexSearcher searcher = newSearcher(wrappedReader); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderWrapperDVTypeCheck.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderWrapperDVTypeCheck.java index 308c48ee77d9..88b177d68dec 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestReaderWrapperDVTypeCheck.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderWrapperDVTypeCheck.java @@ -45,7 +45,7 @@ public void testNoDVFieldOnSegment() throws IOException{ { final Random indexRandom = new Random(seed); final int docs; - docs = indexRandom.nextInt(4); + docs = TestUtil.nextInt(indexRandom, 1, 4); // System.out.println("docs:"+docs); for(int i=0; i< docs; i++){ @@ -68,12 +68,13 @@ public void testNoDVFieldOnSegment() throws IOException{ iw.commit(); } } + iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); // System.out.println("sdv:"+ sdvExist+ " ssdv:"+ssdvExist+", segs: "+reader.leaves().size() +", "+reader.leaves()); iw.close(); - final LeafReader wrapper = SlowCompositeReaderWrapper.wrap(reader); + final LeafReader wrapper = getOnlyLeafReader(reader); { //final Random indexRandom = new Random(seed); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java index fc708c2e7862..1e85e14c9de2 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermEnum.java @@ -79,7 +79,7 @@ public void testPrevTermAtEnd() throws IOException .setCodec(TestUtil.alwaysPostingsFormat(TestUtil.getDefaultPostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); - SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(dir)); + LeafReader reader = getOnlyLeafReader(DirectoryReader.open(dir)); TermsEnum terms = reader.fields().terms("content").iterator(); assertNotNull(terms.next()); assertEquals("aaa", terms.term().utf8ToString()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java index d423616c0980..606a11aff65a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressAdvance.java @@ -74,7 +74,7 @@ public void testStressAdvance() throws Exception { bDocIDs.add(docID); } } - final TermsEnum te = getOnlySegmentReader(r).fields().terms("field").iterator(); + final TermsEnum te = getOnlyLeafReader(r).fields().terms("field").iterator(); PostingsEnum de = null; for(int iter2=0;iter2<10;iter2++) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java index aa2ca247fd87..b074f815f611 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -732,7 +732,7 @@ public void testIntersectBasic() throws Exception { w.forceMerge(1); DirectoryReader r = w.getReader(); w.close(); - LeafReader sub = getOnlySegmentReader(r); + LeafReader sub = getOnlyLeafReader(r); Terms terms = sub.fields().terms("field"); Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); @@ -786,7 +786,7 @@ public void testIntersectStartTerm() throws Exception { w.forceMerge(1); DirectoryReader r = w.getReader(); w.close(); - LeafReader sub = getOnlySegmentReader(r); + LeafReader sub = getOnlyLeafReader(r); Terms terms = sub.fields().terms("field"); Automaton automaton = new RegExp(".*d", RegExp.NONE).toAutomaton(); @@ -840,7 +840,7 @@ public void testIntersectEmptyString() throws Exception { w.forceMerge(1); DirectoryReader r = w.getReader(); w.close(); - LeafReader sub = getOnlySegmentReader(r); + LeafReader sub = getOnlyLeafReader(r); Terms terms = sub.fields().terms("field"); Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); // accept ALL diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java index 7a9bb4e6e19d..79c32d38bc93 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java @@ -36,7 +36,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.Similarity; @@ -157,9 +156,10 @@ public void setUp() throws Exception { writer.addDocument(d4); } - r = SlowCompositeReaderWrapper.wrap(writer.getReader()); + writer.forceMerge(1); + r = getOnlyLeafReader(writer.getReader()); writer.close(); - s = newSearcher(r); + s = new IndexSearcher(r); s.setSimilarity(sim); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java index f28997562ea9..cc6606f038f6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java @@ -87,7 +87,7 @@ public static void beforeClass() throws Exception { iw.forceMerge(1); iw.close(); r = DirectoryReader.open(dir); - reader = getOnlySegmentReader(r); + reader = getOnlyLeafReader(r); searcher = new IndexSearcher(reader); searcher.setSimilarity(new ClassicSimilarity() { @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseEnum.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseEnum.java index aa0c86dde9ea..5d0096dd5bc5 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseEnum.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseEnum.java @@ -48,8 +48,8 @@ public void testOneDocument() throws IOException { DirectoryReader ir = DirectoryReader.open(writer); writer.close(); - PostingsEnum p1 = getOnlySegmentReader(ir).postings(new Term("field", "foo"), PostingsEnum.POSITIONS); - PostingsEnum p2 = getOnlySegmentReader(ir).postings(new Term("field", "bar"), PostingsEnum.POSITIONS); + PostingsEnum p1 = getOnlyLeafReader(ir).postings(new Term("field", "foo"), PostingsEnum.POSITIONS); + PostingsEnum p2 = getOnlyLeafReader(ir).postings(new Term("field", "bar"), PostingsEnum.POSITIONS); PostingsEnum union = new MultiPhraseQuery.UnionPostingsEnum(Arrays.asList(p1, p2)); assertEquals(-1, union.docID()); @@ -90,8 +90,8 @@ public void testSomeDocuments() throws IOException { DirectoryReader ir = DirectoryReader.open(writer); writer.close(); - PostingsEnum p1 = getOnlySegmentReader(ir).postings(new Term("field", "foo"), PostingsEnum.POSITIONS); - PostingsEnum p2 = getOnlySegmentReader(ir).postings(new Term("field", "bar"), PostingsEnum.POSITIONS); + PostingsEnum p1 = getOnlyLeafReader(ir).postings(new Term("field", "foo"), PostingsEnum.POSITIONS); + PostingsEnum p2 = getOnlyLeafReader(ir).postings(new Term("field", "bar"), PostingsEnum.POSITIONS); PostingsEnum union = new MultiPhraseQuery.UnionPostingsEnum(Arrays.asList(p1, p2)); assertEquals(-1, union.docID()); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java index cf779034d811..0d6cb36f0ac3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java @@ -95,7 +95,7 @@ public int getPositionIncrementGap(String fieldName) { reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = new IndexSearcher(reader); } @Override @@ -123,7 +123,7 @@ public void testBarelyCloseEnough() throws Exception { query = new PhraseQuery(3, "field", "one", "five"); ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs; assertEquals(1, hits.length); - QueryUtils.check(random(), query,searcher); + QueryUtils.check(random(), query, searcher); } /** diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java index b7ae42a54bca..227b15d91d4a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java @@ -36,9 +36,7 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; -import org.apache.lucene.search.spans.MultiSpansWrapper; import org.apache.lucene.search.spans.SpanCollector; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanQuery; @@ -225,7 +223,7 @@ public void testPayloadsPos0() throws Exception { writer.addDocument(doc); final IndexReader readerFromWriter = writer.getReader(); - LeafReader r = SlowCompositeReaderWrapper.wrap(readerFromWriter); + LeafReader r = getOnlyLeafReader(readerFromWriter); PostingsEnum tp = r.postings(new Term("content", "a"), PostingsEnum.ALL); @@ -241,7 +239,7 @@ public void testPayloadsPos0() throws Exception { // only one doc has "a" assertEquals(DocIdSetIterator.NO_MORE_DOCS, tp.nextDoc()); - IndexSearcher is = newSearcher(readerFromWriter); + IndexSearcher is = newSearcher(getOnlyLeafReader(readerFromWriter)); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k")); @@ -254,7 +252,7 @@ public void testPayloadsPos0() throws Exception { System.out.println("\ngetPayloadSpans test"); } PayloadSpanCollector collector = new PayloadSpanCollector(); - Spans pspans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, SpanWeight.Postings.PAYLOADS); + Spans pspans = snq.createWeight(is, false).getSpans(is.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); while (pspans.nextDoc() != Spans.NO_MORE_DOCS) { while (pspans.nextStartPosition() != Spans.NO_MORE_POSITIONS) { if (VERBOSE) { @@ -276,7 +274,7 @@ public void testPayloadsPos0() throws Exception { assertEquals(8, count); // System.out.println("\ngetSpans test"); - Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq); + Spans spans = snq.createWeight(is, false).getSpans(is.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); count = 0; sawZero = false; while (spans.nextDoc() != Spans.NO_MORE_DOCS) { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java index 9278934a4929..c332c10406c1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java @@ -20,13 +20,13 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.Similarity; @@ -75,10 +75,9 @@ public void tearDown() throws Exception { public void testBasics() throws Exception { // sanity check of norms writer // TODO: generalize - LeafReader slow = SlowCompositeReaderWrapper.wrap(reader); - NumericDocValues fooNorms = slow.getNormValues("foo"); - NumericDocValues barNorms = slow.getNormValues("bar"); - for (int i = 0; i < slow.maxDoc(); i++) { + NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo"); + NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar"); + for (int i = 0; i < reader.maxDoc(); i++) { assertFalse(fooNorms.get(i) == barNorms.get(i)); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java index 30061b5ca1f1..2cdcba4db2c6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java @@ -29,7 +29,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.store.Directory; @@ -55,11 +54,11 @@ public void setUp() throws Exception { .setSimilarity(new ClassicSimilarity())); for (int i = 0; i < values.length; i++) { Document doc = new Document(); - doc - .add(newTextField(FIELD, values[i], Field.Store.YES)); + doc.add(newTextField(FIELD, values[i], Field.Store.YES)); writer.addDocument(doc); } - indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader()); + writer.forceMerge(1); + indexReader = getOnlyLeafReader(writer.getReader()); writer.close(); indexSearcher = newSearcher(indexReader); indexSearcher.setSimilarity(new ClassicSimilarity()); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java index 9c7ada8c68f7..c656b8559305 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java @@ -16,10 +16,15 @@ */ package org.apache.lucene.search; +import org.apache.lucene.document.Document; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase { @@ -36,7 +41,15 @@ public void testNeverCacheMatchAll() throws Exception { for (int i = 0; i < 1000; ++i) { policy.onUse(q); } - assertFalse(policy.shouldCache(q, SlowCompositeReaderWrapper.wrap(new MultiReader()).getContext())); + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + w.addDocument(new Document()); + IndexReader r = DirectoryReader.open(w); + assertFalse(policy.shouldCache(q, getOnlyLeafReader(r).getContext())); + + r.close(); + w.close(); + dir.close(); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java index 052457b63ce7..b4435e7c1407 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java @@ -116,8 +116,9 @@ public static void beforeClass() throws Exception { field("first", "bubba"), field("last", "jones") })); reader = writer.getReader(); + writer.forceMerge(1); writer.close(); - searcher = newSearcher(reader); + searcher = new IndexSearcher(getOnlyLeafReader(reader)); } @AfterClass @@ -251,7 +252,7 @@ public void testSpans0() throws Exception { SpanQuery q = new SpanOrQuery(q1, new FieldMaskingSpanQuery(q2, "gender")); check(q, new int[] { 0, 1, 2, 3, 4 }); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(span, 0,0,1); assertNext(span, 1,0,1); assertNext(span, 1,1,2); @@ -273,8 +274,8 @@ public void testSpans1() throws Exception { check(qA, new int[] { 0, 1, 2, 4 }); check(qB, new int[] { 0, 1, 2, 4 }); - Spans spanA = MultiSpansWrapper.wrap(searcher.getIndexReader(), qA); - Spans spanB = MultiSpansWrapper.wrap(searcher.getIndexReader(), qB); + Spans spanA = qA.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); + Spans spanB = qB.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); while (spanA.nextDoc() != Spans.NO_MORE_DOCS) { assertNotSame("spanB not still going", Spans.NO_MORE_DOCS, spanB.nextDoc()); @@ -299,7 +300,7 @@ public void testSpans2() throws Exception { new FieldMaskingSpanQuery(qB, "id") }, -1, false ); check(q, new int[] { 0, 1, 2, 3 }); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(span, 0,0,1); assertNext(span, 1,1,2); assertNext(span, 2,0,1); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java index b101c6121624..f297f33676d1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java @@ -60,9 +60,10 @@ public void setUp() throws Exception { doc.add(newTextField(FIELD, docFields[i], Field.Store.NO)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = newSearcher(getOnlyLeafReader(reader)); } protected String[] docFields = { @@ -118,7 +119,7 @@ public String s(int doc, int start, int end) { public void testNearSpansNext() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(span,0,0,3); assertNext(span,1,0,4); assertFinished(span); @@ -131,7 +132,7 @@ public void testNearSpansNext() throws Exception { */ public void testNearSpansAdvanceLikeNext() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(0, span.advance(0)); assertEquals(0, span.nextStartPosition()); assertEquals(s(0,0,3), s(span)); @@ -143,7 +144,7 @@ public void testNearSpansAdvanceLikeNext() throws Exception { public void testNearSpansNextThenAdvance() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNotSame(Spans.NO_MORE_DOCS, span.nextDoc()); assertEquals(0, span.nextStartPosition()); assertEquals(s(0,0,3), s(span)); @@ -155,7 +156,7 @@ public void testNearSpansNextThenAdvance() throws Exception { public void testNearSpansNextThenAdvancePast() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNotSame(Spans.NO_MORE_DOCS, span.nextDoc()); assertEquals(0, span.nextStartPosition()); assertEquals(s(0,0,3), s(span)); @@ -164,13 +165,13 @@ public void testNearSpansNextThenAdvancePast() throws Exception { public void testNearSpansAdvancePast() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(Spans.NO_MORE_DOCS, span.advance(2)); } public void testNearSpansAdvanceTo0() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(0, span.advance(0)); assertEquals(0, span.nextStartPosition()); assertEquals(s(0,0,3), s(span)); @@ -178,7 +179,7 @@ public void testNearSpansAdvanceTo0() throws Exception { public void testNearSpansAdvanceTo1() throws Exception { SpanNearQuery q = makeQuery(); - Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q); + Spans span = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(1, span.advance(1)); assertEquals(0, span.nextStartPosition()); assertEquals(s(1,0,4), s(span)); @@ -217,7 +218,7 @@ public void testOrderedSpanIteration() throws Exception { new SpanOrQuery(new SpanTermQuery(new Term(FIELD, "w1")), new SpanTermQuery(new Term(FIELD, "w2"))), new SpanTermQuery(new Term(FIELD, "w4")) }, 10, true); - Spans spans = MultiSpansWrapper.wrap(reader, q); + Spans spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans,0,0,4); assertNext(spans,0,1,4); assertFinished(spans); @@ -227,7 +228,7 @@ public void testOrderedSpanIterationSameTerms1() throws Exception { SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{ new SpanTermQuery(new Term(FIELD, "t1")), new SpanTermQuery(new Term(FIELD, "t2")) }, 1, true); - Spans spans = MultiSpansWrapper.wrap(reader, q); + Spans spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans,4,0,2); assertFinished(spans); } @@ -236,7 +237,7 @@ public void testOrderedSpanIterationSameTerms2() throws Exception { SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{ new SpanTermQuery(new Term(FIELD, "t2")), new SpanTermQuery(new Term(FIELD, "t1")) }, 1, true); - Spans spans = MultiSpansWrapper.wrap(reader, q); + Spans spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans,4,1,4); assertNext(spans,4,2,4); assertFinished(spans); @@ -260,7 +261,7 @@ public void testGaps() throws Exception { .addGap(1) .addClause(new SpanTermQuery(new Term(FIELD, "w2"))) .build(); - Spans spans = MultiSpansWrapper.wrap(reader, q); + Spans spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans, 1, 0, 3); assertNext(spans, 2, 0, 3); assertFinished(spans); @@ -273,7 +274,7 @@ public void testGaps() throws Exception { .addClause(new SpanTermQuery(new Term(FIELD, "w3"))) .setSlop(1) .build(); - spans = MultiSpansWrapper.wrap(reader, q); + spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans, 2, 0, 5); assertNext(spans, 3, 0, 6); assertFinished(spans); @@ -285,7 +286,7 @@ public void testMultipleGaps() throws Exception { .addGap(2) .addClause(new SpanTermQuery(new Term(FIELD, "g"))) .build(); - Spans spans = MultiSpansWrapper.wrap(reader, q); + Spans spans = q.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans, 5, 0, 4); assertNext(spans, 5, 9, 13); assertFinished(spans); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java index 03fc40dda39a..dfc0439e32ca 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java @@ -67,9 +67,10 @@ public void setUp() throws Exception { doc.add(newField(FIELD, docFields[i], OFFSETS)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = newSearcher(getOnlyLeafReader(reader)); } private static class TermCollector implements SpanCollector { @@ -119,7 +120,7 @@ public void testNestedNearQuery() throws IOException { SpanNearQuery q7 = new SpanNearQuery(new SpanQuery[]{q1, q6}, 1, true); TermCollector collector = new TermCollector(); - Spans spans = MultiSpansWrapper.wrap(reader, q7, SpanWeight.Postings.POSITIONS); + Spans spans = q7.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(0, spans.advance(0)); spans.nextStartPosition(); checkCollectedTerms(spans, collector, new Term(FIELD, "w1"), new Term(FIELD, "w2"), new Term(FIELD, "w3")); @@ -139,7 +140,7 @@ public void testOrQuery() throws IOException { SpanOrQuery orQuery = new SpanOrQuery(q2, q3); TermCollector collector = new TermCollector(); - Spans spans = MultiSpansWrapper.wrap(reader, orQuery, SpanWeight.Postings.POSITIONS); + Spans spans = orQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(1, spans.advance(1)); spans.nextStartPosition(); @@ -169,7 +170,7 @@ public void testSpanNotQuery() throws IOException { SpanNotQuery notq = new SpanNotQuery(nq, q3); TermCollector collector = new TermCollector(); - Spans spans = MultiSpansWrapper.wrap(reader, notq, SpanWeight.Postings.POSITIONS); + Spans spans = notq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals(2, spans.advance(2)); spans.nextStartPosition(); diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanContainQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanContainQuery.java index c26070416c26..3e50183e739a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanContainQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanContainQuery.java @@ -47,9 +47,10 @@ public void setUp() throws Exception { doc.add(newTextField(field, docFields[i], Field.Store.YES)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = newSearcher(getOnlyLeafReader(reader)); } @Override @@ -71,7 +72,7 @@ void checkHits(Query query, int[] results) throws Exception { } Spans makeSpans(SpanQuery sq) throws Exception { - return MultiSpansWrapper.wrap(searcher.getIndexReader(), sq); + return sq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); } void tstEqualSpans(String mes, SpanQuery expectedQ, SpanQuery actualQ) throws Exception { @@ -144,4 +145,4 @@ public void testSpanContainPhraseSecondWord() throws Exception { assertFinished(spans); } -} \ No newline at end of file +} diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java index 9352f60b7740..29833012a456 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java @@ -70,9 +70,10 @@ public void setUp() throws Exception { doc.add(newTextField(field, docFields[i], Field.Store.YES)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = newSearcher(getOnlyLeafReader(reader)); } @Override @@ -201,7 +202,7 @@ public void testSpanNearOrderedEqual15() throws Exception { public void testSpanNearOrderedOverlap() throws Exception { final SpanQuery query = spanNearOrderedQuery(field, 1, "t1", "t2", "t3"); - Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), query); + Spans spans = query.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertEquals("first doc", 11, spans.nextDoc()); assertEquals("first start", 0, spans.nextStartPosition()); @@ -216,7 +217,7 @@ public void testSpanNearOrderedOverlap() throws Exception { public void testSpanNearUnOrdered() throws Exception { //See http://www.gossamer-threads.com/lists/lucene/java-dev/52270 for discussion about this test SpanQuery senq = spanNearUnorderedQuery(field, 0, "u1", "u2"); - Spans spans = MultiSpansWrapper.wrap(reader, senq); + Spans spans = senq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans, 4, 1, 3); assertNext(spans, 5, 2, 4); assertNext(spans, 8, 2, 4); @@ -225,7 +226,7 @@ public void testSpanNearUnOrdered() throws Exception { assertFinished(spans); senq = spanNearUnorderedQuery(1, senq, spanTermQuery(field, "u2")); - spans = MultiSpansWrapper.wrap(reader, senq); + spans = senq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertNext(spans, 4, 0, 3); assertNext(spans, 4, 1, 3); // unordered spans can be subsets assertNext(spans, 5, 0, 4); @@ -239,7 +240,7 @@ public void testSpanNearUnOrdered() throws Exception { } private Spans orSpans(String[] terms) throws Exception { - return MultiSpansWrapper.wrap(searcher.getIndexReader(), spanOrQuery(field, terms)); + return spanOrQuery(field, terms).createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); } public void testSpanOrEmpty() throws Exception { @@ -443,7 +444,7 @@ private int spanCount(String include, String exclude, int pre, int post) throws SpanQuery iq = spanTermQuery(field, include); SpanQuery eq = spanTermQuery(field, exclude); SpanQuery snq = spanNotQuery(iq, eq, pre, post); - Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), snq); + Spans spans = snq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); int i = 0; if (spans != null) { diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java index 5d374f782c7a..e052541b3604 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java @@ -23,9 +23,16 @@ import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState.OrdRange; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues; +import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues; +import org.apache.lucene.index.MultiDocValues.OrdinalMap; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; @@ -35,12 +42,13 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesReaderState { private final String field; - private final LeafReader topReader; private final int valueCount; /** {@link IndexReader} passed to the constructor. */ public final IndexReader origReader; + private final Map cachedOrdMaps = new HashMap<>(); + private final Map prefixToOrdRange = new HashMap<>(); /** Creates this, pulling doc values from the default {@link @@ -57,8 +65,7 @@ public DefaultSortedSetDocValuesReaderState(IndexReader reader, String field) th // We need this to create thread-safe MultiSortedSetDV // per collector: - topReader = SlowCompositeReaderWrapper.wrap(reader); - SortedSetDocValues dv = topReader.getSortedSetDocValues(field); + SortedSetDocValues dv = getDocValues(); if (dv == null) { throw new IllegalArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues"); } @@ -100,7 +107,43 @@ public DefaultSortedSetDocValuesReaderState(IndexReader reader, String field) th /** Return top-level doc values. */ @Override public SortedSetDocValues getDocValues() throws IOException { - return topReader.getSortedSetDocValues(field); + // TODO: this is dup'd from slow composite reader wrapper ... can we factor it out to share? + OrdinalMap map = null; + synchronized (cachedOrdMaps) { + map = cachedOrdMaps.get(field); + if (map == null) { + // uncached, or not a multi dv + SortedSetDocValues dv = MultiDocValues.getSortedSetValues(origReader, field); + if (dv instanceof MultiSortedSetDocValues) { + map = ((MultiSortedSetDocValues)dv).mapping; + if (map.owner == origReader.getCoreCacheKey()) { + cachedOrdMaps.put(field, map); + } + } + return dv; + } + } + + assert map != null; + int size = origReader.leaves().size(); + final SortedSetDocValues[] values = new SortedSetDocValues[size]; + final int[] starts = new int[size+1]; + for (int i = 0; i < size; i++) { + LeafReaderContext context = origReader.leaves().get(i); + final LeafReader reader = context.reader(); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); + if (fieldInfo != null && fieldInfo.getDocValuesType() != DocValuesType.SORTED_SET) { + return null; + } + SortedSetDocValues v = reader.getSortedSetDocValues(field); + if (v == null) { + v = DocValues.emptySortedSet(); + } + values[i] = v; + starts[i] = context.docBase; + } + starts[size] = origReader.maxDoc(); + return new MultiSortedSetDocValues(values, starts, map); } /** Returns mapping from prefix to {@link OrdRange}. */ diff --git a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java index 7659be80ec8f..60beddd88d0b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java @@ -32,7 +32,6 @@ import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -227,41 +226,6 @@ public void testSomeSegmentsMissing() throws Exception { dir.close(); } - public void testSlowCompositeReaderWrapper() throws Exception { - Directory dir = newDirectory(); - - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - - FacetsConfig config = new FacetsConfig(); - - Document doc = new Document(); - doc.add(new SortedSetDocValuesFacetField("a", "foo1")); - writer.addDocument(config.build(doc)); - - writer.commit(); - - doc = new Document(); - doc.add(new SortedSetDocValuesFacetField("a", "foo2")); - writer.addDocument(config.build(doc)); - - // NRT open - IndexSearcher searcher = new IndexSearcher(SlowCompositeReaderWrapper.wrap(writer.getReader())); - - // Per-top-reader state: - SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader()); - - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); - Facets facets = new SortedSetDocValuesFacetCounts(state, c); - - // Ask for top 10 labels for any dims that have counts: - assertEquals("dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n", facets.getTopChildren(10, "a").toString()); - - writer.close(); - IOUtils.close(searcher.getIndexReader(), dir); - } - - public void testRandom() throws Exception { String[] tokens = getRandomTokens(10); Directory indexDir = newDirectory(); diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java index a3bda0b3a6f1..2f51c9433c50 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java @@ -44,7 +44,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.ReaderUtil; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource; @@ -1164,16 +1163,7 @@ private TopGroups searchShards(IndexSearcher topSearcher, ShardSearche final List>> shardGroups = new ArrayList<>(); List> firstPassGroupingCollectors = new ArrayList<>(); AbstractFirstPassGroupingCollector firstPassCollector = null; - boolean shardsCanUseIDV; - if (canUseIDV) { - if (SlowCompositeReaderWrapper.class.isAssignableFrom(subSearchers[0].getIndexReader().getClass())) { - shardsCanUseIDV = false; - } else { - shardsCanUseIDV = !preFlex; - } - } else { - shardsCanUseIDV = false; - } + boolean shardsCanUseIDV = canUseIDV; String groupField = "group"; diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index af507cdc2fb6..2796e015b517 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -58,7 +58,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; @@ -558,7 +557,7 @@ public String toString(String field) { assertEquals(numParents, topDocs.totalHits); for (int i = 0; i < topDocs.scoreDocs.length; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; - String id = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()).document(scoreDoc.doc).get("id"); + String id = searcher.doc(scoreDoc.doc).get("id"); assertEquals(lowestScoresPerParent.get(id), scoreDoc.score, 0f); } @@ -567,7 +566,7 @@ public String toString(String field) { assertEquals(numParents, topDocs.totalHits); for (int i = 0; i < topDocs.scoreDocs.length; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; - String id = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()).document(scoreDoc.doc).get("id"); + String id = searcher.doc(scoreDoc.doc).get("id"); assertEquals(highestScoresPerParent.get(id), scoreDoc.score, 0f); } @@ -1229,8 +1228,7 @@ public boolean needsScores() { final Map docToJoinScore = new HashMap<>(); if (multipleValuesPerDocument) { - LeafReader slowCompositeReader = SlowCompositeReaderWrapper.wrap(topLevelReader); - Terms terms = slowCompositeReader.terms(toField); + Terms terms = MultiFields.getTerms(topLevelReader, toField); if (terms != null) { PostingsEnum postingsEnum = null; SortedSet joinValues = new TreeSet<>(); diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index e29e8c805e59..57e25fe0b0b6 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -41,7 +41,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; @@ -49,8 +48,10 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -66,8 +67,8 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.ByteBlockPool.Allocator; +import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; @@ -170,15 +171,14 @@ public void assertAgainstRAMDirectory(MemoryIndex memory) throws Exception { private void duellReaders(CompositeReader other, LeafReader memIndexReader) throws IOException { - LeafReader competitor = SlowCompositeReaderWrapper.wrap(other); Fields memFields = memIndexReader.fields(); - for (String field : competitor.fields()) { + for (String field : MultiFields.getFields(other)) { Terms memTerms = memFields.terms(field); Terms iwTerms = memIndexReader.terms(field); if (iwTerms == null) { assertNull(memTerms); } else { - NumericDocValues normValues = competitor.getNormValues(field); + NumericDocValues normValues = MultiDocValues.getNormValues(other, field); NumericDocValues memNormValues = memIndexReader.getNormValues(field); if (normValues != null) { // mem idx always computes norms on the fly diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java b/lucene/misc/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java rename to lucene/misc/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java index e44c53cb54ea..de79ab07f764 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java +++ b/lucene/misc/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java @@ -41,6 +41,7 @@ * leaves and then operate per-LeafReader, * instead of using this class. */ + public final class SlowCompositeReaderWrapper extends LeafReader { private final CompositeReader in; @@ -63,6 +64,9 @@ public static LeafReader wrap(IndexReader reader) throws IOException { SlowCompositeReaderWrapper(CompositeReader reader, boolean merging) throws IOException { super(); in = reader; + if (getFieldInfos().hasPointValues()) { + throw new IllegalArgumentException("cannot wrap points"); + } fields = MultiFields.getFields(in); in.registerParentReader(this); this.merging = merging; diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestSlowCompositeReaderWrapper.java b/lucene/misc/src/test/org/apache/lucene/index/TestSlowCompositeReaderWrapper.java new file mode 100644 index 000000000000..d37e45abebfb --- /dev/null +++ b/lucene/misc/src/test/org/apache/lucene/index/TestSlowCompositeReaderWrapper.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.document.Document; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +public class TestSlowCompositeReaderWrapper extends LuceneTestCase { + + public void testCoreListenerOnSlowCompositeReaderWrapper() throws IOException { + RandomIndexWriter w = new RandomIndexWriter(random(), newDirectory()); + final int numDocs = TestUtil.nextInt(random(), 1, 5); + for (int i = 0; i < numDocs; ++i) { + w.addDocument(new Document()); + if (random().nextBoolean()) { + w.commit(); + } + } + w.commit(); + w.close(); + + final IndexReader reader = DirectoryReader.open(w.w.getDirectory()); + final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader); + + final int numListeners = TestUtil.nextInt(random(), 1, 10); + final List listeners = new ArrayList<>(); + AtomicInteger counter = new AtomicInteger(numListeners); + + for (int i = 0; i < numListeners; ++i) { + CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheKey()); + listeners.add(listener); + leafReader.addCoreClosedListener(listener); + } + for (int i = 0; i < 100; ++i) { + leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size()))); + } + final int removed = random().nextInt(numListeners); + Collections.shuffle(listeners, random()); + for (int i = 0; i < removed; ++i) { + leafReader.removeCoreClosedListener(listeners.get(i)); + } + assertEquals(numListeners, counter.get()); + // make sure listeners are registered on the wrapped reader and that closing any of them has the same effect + if (random().nextBoolean()) { + reader.close(); + } else { + leafReader.close(); + } + assertEquals(removed, counter.get()); + w.w.getDirectory().close(); + } + + private static final class CountCoreListener implements LeafReader.CoreClosedListener { + + private final AtomicInteger count; + private final Object coreCacheKey; + + public CountCoreListener(AtomicInteger count, Object coreCacheKey) { + this.count = count; + this.coreCacheKey = coreCacheKey; + } + + @Override + public void onClose(Object coreCacheKey) { + assertSame(this.coreCacheKey, coreCacheKey); + count.decrementAndGet(); + } + + } +} diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java index 4861cd35ca2f..0c5d765b8cd3 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java @@ -443,9 +443,9 @@ public void testBackToTheFuture() throws Exception { iw.deleteDocuments(new Term("foo", "baz")); DirectoryReader r2 = DirectoryReader.open(iw); - FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r2), "foo", null); + FieldCache.DEFAULT.getDocTermOrds(getOnlyLeafReader(r2), "foo", null); - SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(getOnlySegmentReader(r1), "foo", null); + SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(getOnlyLeafReader(r1), "foo", null); assertEquals(3, v.getValueCount()); v.setDocument(1); assertEquals(1, v.nextOrd()); @@ -473,7 +473,7 @@ public void testNumericEncoded32() throws IOException { iw.close(); DirectoryReader ir = DirectoryReader.open(dir); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(ar, "foo", FieldCache.INT32_TERM_PREFIX); assertEquals(2, v.getValueCount()); @@ -514,7 +514,7 @@ public void testNumericEncoded64() throws IOException { iw.close(); DirectoryReader ir = DirectoryReader.open(dir); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(ar, "foo", FieldCache.INT64_TERM_PREFIX); assertEquals(2, v.getValueCount()); @@ -563,7 +563,7 @@ public void testSortedTermsEnum() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - LeafReader ar = getOnlySegmentReader(ireader); + LeafReader ar = getOnlyLeafReader(ireader); SortedSetDocValues dv = FieldCache.DEFAULT.getDocTermOrds(ar, "field", null); assertEquals(3, dv.getValueCount()); @@ -648,7 +648,7 @@ public void testActuallySingleValued() throws IOException { iw.close(); DirectoryReader ir = DirectoryReader.open(dir); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); SortedSetDocValues v = FieldCache.DEFAULT.getDocTermOrds(ar, "foo", null); assertNotNull(DocValues.unwrapSingleton(v)); // actually a single-valued field diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java index 1b322d936008..93c302c51e77 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java @@ -419,7 +419,7 @@ public void testDocValuesIntegration() throws Exception { iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); // Binary type: can be retrieved via getTerms() expectThrows(IllegalStateException.class, () -> { @@ -535,7 +535,7 @@ public void testNonexistantFields() throws Exception { DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); @@ -593,7 +593,7 @@ public void testNonIndexedFields() throws Exception { DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); @@ -673,7 +673,7 @@ public void testLongFieldCache() throws IOException { } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LONG_POINT_PARSER, false); + final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LONG_POINT_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], longs.get(i)); } @@ -719,7 +719,7 @@ public void testIntFieldCache() throws IOException { } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.INT_POINT_PARSER, false); + final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.INT_POINT_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], ints.get(i)); } diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java index 0d5584e4544e..de8eab1db77f 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java @@ -47,7 +47,7 @@ public void testFieldCacheReuseAfterReopen() throws Exception { // Open reader1 DirectoryReader r = DirectoryReader.open(dir); - LeafReader r1 = getOnlySegmentReader(r); + LeafReader r1 = getOnlyLeafReader(r); final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER, false); assertEquals(17, ints.get(0)); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java index 23b7d0c7a912..9809324e0593 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java @@ -399,7 +399,7 @@ private void doTestSortedSetVsUninvertedField(int minLength, int maxLength) thro // now compare again after the merge ir = writer.getReader(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); SortedSetDocValues expected = FieldCache.DEFAULT.getDocTermOrds(ar, "indexed", null); SortedSetDocValues actual = ar.getSortedSetDocValues("dv"); assertEquals(ir.maxDoc(), expected, actual); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java index e716419de7ca..3f1f450d01f5 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java @@ -181,7 +181,7 @@ public void test2() throws Exception { final DirectoryReader r = writer.getReader(); writer.close(); - final LeafReader sr = getOnlySegmentReader(r); + final LeafReader sr = getOnlyLeafReader(r); final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1); diff --git a/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java b/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java index c4ef1c4f4b76..4fd66ad05ea2 100644 --- a/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java +++ b/lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java @@ -307,7 +307,7 @@ public void testDocValuesIntegration() throws Exception { iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); // Binary type: can be retrieved via getTerms() expectThrows(IllegalStateException.class, () -> { @@ -340,7 +340,7 @@ public void testNonexistantFields() throws Exception { DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); @@ -379,7 +379,7 @@ public void testNonIndexedFields() throws Exception { DirectoryReader ir = iw.getReader(); iw.close(); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); @@ -440,7 +440,7 @@ public void testLongFieldCache() throws IOException { } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LEGACY_LONG_PARSER, false); + final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_LONG_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], longs.get(i)); } @@ -486,7 +486,7 @@ public void testIntFieldCache() throws IOException { } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); - final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.LEGACY_INT_PARSER, false); + final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LEGACY_INT_PARSER, false); for (int i = 0; i < values.length; ++i) { assertEquals(values[i], ints.get(i)); } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java index a7bfffdfc61c..e991b0cb61dd 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java @@ -32,7 +32,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; import org.apache.lucene.index.Terms; @@ -399,8 +398,9 @@ public void testRandomIndex() throws IOException { analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH)); RandomIndexWriter w = new RandomIndexWriter(random(), dir, analyzer); createRandomIndex(atLeast(50), w, random().nextLong()); + w.forceMerge(1); DirectoryReader reader = w.getReader(); - LeafReader wrapper = SlowCompositeReaderWrapper.wrap(reader); + LeafReader wrapper = getOnlyLeafReader(reader); String field = "body"; Terms terms = wrapper.terms(field); PriorityQueue lowFreqQueue = new PriorityQueue( @@ -489,7 +489,7 @@ protected boolean lessThan(TermAndFreq a, TermAndFreq b) { QueryUtils.check(random(), cq, newSearcher(reader2)); reader2.close(); } finally { - IOUtils.close(reader, wrapper, w, dir, analyzer); + IOUtils.close(wrapper, w, dir, analyzer); } } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java index c14d5438d275..a87e45d23f08 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java @@ -342,11 +342,11 @@ public void testIsConsideredCostlyByQueryCache() throws IOException { w.close(); TermsQuery query = new TermsQuery(new Term("foo", "bar"), new Term("foo", "baz")); UsageTrackingQueryCachingPolicy policy = new UsageTrackingQueryCachingPolicy(); - assertFalse(policy.shouldCache(query, getOnlySegmentReader(reader).getContext())); + assertFalse(policy.shouldCache(query, getOnlyLeafReader(reader).getContext())); policy.onUse(query); policy.onUse(query); // cached after two uses - assertTrue(policy.shouldCache(query, getOnlySegmentReader(reader).getContext())); + assertTrue(policy.shouldCache(query, getOnlyLeafReader(reader).getContext())); reader.close(); dir.close(); } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestSortedSetFieldSource.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestSortedSetFieldSource.java index d028dce6db43..b72bfebfa44a 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestSortedSetFieldSource.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestSortedSetFieldSource.java @@ -50,7 +50,7 @@ public void testSimple() throws Exception { DirectoryReader ir = DirectoryReader.open(dir); IndexSearcher searcher = newSearcher(ir); - LeafReader ar = getOnlySegmentReader(ir); + LeafReader ar = getOnlyLeafReader(ir); ValueSource vs = new SortedSetFieldSource("value"); FunctionValues values = vs.getValues(Collections.emptyMap(), ar.getContext()); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/PayloadHelper.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/PayloadHelper.java index 484c1bd4f649..cfd5156aa6ff 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/PayloadHelper.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/PayloadHelper.java @@ -126,10 +126,11 @@ public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) th doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES)); writer.addDocument(doc); } + writer.forceMerge(1); reader = DirectoryReader.open(writer); writer.close(); - IndexSearcher searcher = LuceneTestCase.newSearcher(reader); + IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader)); searcher.setSimilarity(similarity); return searcher; } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java index 3f168bba4aff..179b971fbcaf 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java @@ -42,7 +42,6 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.spans.MultiSpansWrapper; import org.apache.lucene.search.spans.SpanCollector; import org.apache.lucene.search.spans.SpanFirstQuery; import org.apache.lucene.search.spans.SpanNearQuery; @@ -75,12 +74,12 @@ public void testSpanTermQuery() throws Exception { Spans spans; stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "seventy")); - spans = MultiSpansWrapper.wrap(indexReader, stq, SpanWeight.Postings.PAYLOADS); + spans = stq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 100, 1, 1, 1); stq = new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "seventy")); - spans = MultiSpansWrapper.wrap(indexReader, stq, SpanWeight.Postings.PAYLOADS); + spans = stq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 100, 0, 0, 0); } @@ -91,7 +90,7 @@ public void testSpanFirst() throws IOException { SpanFirstQuery sfq; match = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one")); sfq = new SpanFirstQuery(match, 2); - Spans spans = MultiSpansWrapper.wrap(indexReader, sfq, SpanWeight.Postings.PAYLOADS); + Spans spans = sfq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); checkSpans(spans, 109, 1, 1, 1); //Test more complicated subclause SpanQuery[] clauses = new SpanQuery[2]; @@ -99,11 +98,11 @@ public void testSpanFirst() throws IOException { clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "hundred")); match = new SpanNearQuery(clauses, 0, true); sfq = new SpanFirstQuery(match, 2); - checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, SpanWeight.Postings.PAYLOADS), 100, 2, 1, 1); + checkSpans(sfq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS), 100, 2, 1, 1); match = new SpanNearQuery(clauses, 0, false); sfq = new SpanFirstQuery(match, 2); - checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, SpanWeight.Postings.PAYLOADS), 100, 2, 1, 1); + checkSpans(sfq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS), 100, 2, 1, 1); } @@ -123,10 +122,10 @@ public void testSpanNot() throws Exception { Document doc = new Document(); doc.add(newTextField(PayloadHelper.FIELD, "one two three one four three", Field.Store.YES)); writer.addDocument(doc); - IndexReader reader = writer.getReader(); + IndexReader reader = getOnlyLeafReader(writer.getReader()); writer.close(); - checkSpans(MultiSpansWrapper.wrap(reader, snq, SpanWeight.Postings.PAYLOADS), 1, new int[]{2}); + checkSpans(snq.createWeight(newSearcher(reader), false).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2}); reader.close(); directory.close(); } @@ -137,7 +136,7 @@ public void testNestedSpans() throws Exception { IndexSearcher searcher = getSearcher(); stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "mark")); - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq, SpanWeight.Postings.PAYLOADS); + spans = stq.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertNull(spans); SpanQuery[] clauses = new SpanQuery[3]; @@ -146,7 +145,7 @@ public void testNestedSpans() throws Exception { clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx")); SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 12, false); - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, SpanWeight.Postings.PAYLOADS); + spans = spanNearQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 2, new int[]{3,3}); @@ -157,7 +156,7 @@ public void testNestedSpans() throws Exception { spanNearQuery = new SpanNearQuery(clauses, 6, true); - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, SpanWeight.Postings.PAYLOADS); + spans = spanNearQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 1, new int[]{3}); @@ -179,7 +178,7 @@ public void testNestedSpans() throws Exception { SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses2, 6, false); // yy within 6 of xx within 6 of rr - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, SpanWeight.Postings.PAYLOADS); + spans = nestedSpanNearQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 2, new int[]{3,3}); closeIndexReader.close(); @@ -210,7 +209,7 @@ public void testFirstClauseWithoutPayload() throws Exception { clauses3[1] = snq; SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false); - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, SpanWeight.Postings.PAYLOADS); + spans = nestedSpanNearQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 1, new int[]{3}); @@ -248,7 +247,7 @@ public void testHeavilyNestedSpanQuery() throws Exception { SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false); - spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, SpanWeight.Postings.PAYLOADS); + spans = nestedSpanNearQuery.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); assertTrue("spans is null and it shouldn't be", spans != null); checkSpans(spans, 2, new int[]{8, 8}); closeIndexReader.close(); @@ -265,7 +264,7 @@ public void testShrinkToAfterShortestMatch() throws IOException { writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(reader); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); @@ -273,7 +272,7 @@ public void testShrinkToAfterShortestMatch() throws IOException { SpanQuery[] sqs = { stq1, stq2 }; SpanNearQuery snq = new SpanNearQuery(sqs, 1, true); VerifyingCollector collector = new VerifyingCollector(); - Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, SpanWeight.Postings.PAYLOADS); + Spans spans = snq.createWeight(is, false).getSpans(is.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); TopDocs topDocs = is.search(snq, 1); Set payloadSet = new HashSet<>(); @@ -304,7 +303,7 @@ public void testShrinkToAfterShortestMatch2() throws IOException { doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(reader); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); @@ -312,7 +311,7 @@ public void testShrinkToAfterShortestMatch2() throws IOException { SpanQuery[] sqs = { stq1, stq2 }; SpanNearQuery snq = new SpanNearQuery(sqs, 0, true); VerifyingCollector collector = new VerifyingCollector(); - Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, SpanWeight.Postings.PAYLOADS); + Spans spans = snq.createWeight(is, false).getSpans(is.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); TopDocs topDocs = is.search(snq, 1); Set payloadSet = new HashSet<>(); @@ -343,14 +342,14 @@ public void testShrinkToAfterShortestMatch3() throws IOException { doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(reader); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k")); SpanQuery[] sqs = { stq1, stq2 }; SpanNearQuery snq = new SpanNearQuery(sqs, 0, true); - Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, SpanWeight.Postings.PAYLOADS); + Spans spans = snq.createWeight(is, false).getSpans(is.getIndexReader().leaves().get(0), SpanWeight.Postings.PAYLOADS); TopDocs topDocs = is.search(snq, 1); Set payloadSet = new HashSet<>(); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java index f453b0d0af72..dfa0191efa7b 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadTermQuery.java @@ -38,10 +38,10 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.spans.MultiSpansWrapper; import org.apache.lucene.search.spans.SpanQuery; -import org.apache.lucene.search.spans.Spans; import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.search.spans.SpanWeight; +import org.apache.lucene.search.spans.Spans; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; @@ -131,10 +131,11 @@ public static void beforeClass() throws Exception { doc.add(newTextField("multiField", English.intToEnglish(i) + " " + English.intToEnglish(i), Field.Store.YES)); writer.addDocument(doc); } + writer.forceMerge(1); reader = writer.getReader(); writer.close(); - searcher = newSearcher(reader); + searcher = newSearcher(getOnlyLeafReader(reader)); searcher.setSimilarity(similarity); } @@ -163,7 +164,7 @@ public void test() throws IOException { assertTrue(doc.score + " does not equal: " + 1, doc.score == 1); } CheckHits.checkExplanations(query, PayloadHelper.FIELD, searcher, true); - Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), query); + Spans spans = query.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertTrue("spans is null and it shouldn't be", spans != null); /*float score = hits.score(0); for (int i =1; i < hits.length(); i++) @@ -214,7 +215,7 @@ public void testMultipleMatchesPerDoc() throws Exception { } assertTrue(numTens + " does not equal: " + 10, numTens == 10); CheckHits.checkExplanations(query, "field", searcher, true); - Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), query); + Spans spans = query.createWeight(searcher, false).getSpans(searcher.getIndexReader().leaves().get(0), SpanWeight.Postings.POSITIONS); assertTrue("spans is null and it shouldn't be", spans != null); //should be two matches per document int count = 0; diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java index 5a8a99f4d154..85ac12f7ceab 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java @@ -568,7 +568,7 @@ public void testBytesMergeAwayAllValues() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - BinaryDocValues dv = getOnlySegmentReader(ireader).getBinaryDocValues("field"); + BinaryDocValues dv = getOnlyLeafReader(ireader).getBinaryDocValues("field"); assertEquals(new BytesRef(), dv.get(0)); ireader.close(); @@ -743,7 +743,7 @@ public void testSortedMergeAwayAllValues() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field"); + SortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field"); if (codecSupportsDocsWithField()) { assertEquals(-1, dv.getOrd(0)); assertEquals(0, dv.getValueCount()); @@ -833,7 +833,7 @@ public void testSortedTermsEnum() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field"); + SortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field"); assertEquals(3, dv.getValueCount()); TermsEnum termsEnum = dv.termsEnum(); @@ -1077,7 +1077,7 @@ public void testDocValuesSimple() throws IOException { TopDocs search = searcher.search(query.build(), 10); assertEquals(5, search.totalHits); ScoreDoc[] scoreDocs = search.scoreDocs; - NumericDocValues docValues = getOnlySegmentReader(reader).getNumericDocValues("docId"); + NumericDocValues docValues = getOnlyLeafReader(reader).getNumericDocValues("docId"); for (int i = 0; i < scoreDocs.length; i++) { assertEquals(i, scoreDocs[i].doc); assertEquals(i, docValues.get(scoreDocs[i].doc)); @@ -1154,12 +1154,11 @@ public void testRandomSortedBytes() throws IOException { int ord = docValues.lookupTerm(expected); assertEquals(i, ord); } - LeafReader slowR = SlowCompositeReaderWrapper.wrap(reader); Set> entrySet = docToString.entrySet(); for (Entry entry : entrySet) { // pk lookup - PostingsEnum termPostingsEnum = slowR.postings(new Term("id", entry.getKey())); + PostingsEnum termPostingsEnum = TestUtil.docs(random(), reader, "id", new BytesRef(entry.getKey()), null, 0); int docId = termPostingsEnum.nextDoc(); expected = new BytesRef(entry.getValue()); final BytesRef actual = docValues.get(docId); @@ -1516,7 +1515,7 @@ public void testSortedSetOneValue() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); dv.setDocument(0); assertEquals(0, dv.nextOrd()); @@ -1542,7 +1541,7 @@ public void testSortedSetTwoFields() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); dv.setDocument(0); assertEquals(0, dv.nextOrd()); @@ -1551,7 +1550,7 @@ public void testSortedSetTwoFields() throws IOException { BytesRef bytes = dv.lookupOrd(0); assertEquals(new BytesRef("hello"), bytes); - dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field2"); + dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field2"); dv.setDocument(0); assertEquals(0, dv.nextOrd()); @@ -1585,7 +1584,7 @@ public void testSortedSetTwoDocumentsMerged() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(2, dv.getValueCount()); dv.setDocument(0); @@ -1619,7 +1618,7 @@ public void testSortedSetTwoValues() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); dv.setDocument(0); assertEquals(0, dv.nextOrd()); @@ -1649,7 +1648,7 @@ public void testSortedSetTwoValuesUnordered() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); dv.setDocument(0); assertEquals(0, dv.nextOrd()); @@ -1689,7 +1688,7 @@ public void testSortedSetThreeValuesTwoDocs() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(3, dv.getValueCount()); dv.setDocument(0); @@ -1733,7 +1732,7 @@ public void testSortedSetTwoDocumentsLastMissing() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(1, dv.getValueCount()); dv.setDocument(0); @@ -1767,7 +1766,7 @@ public void testSortedSetTwoDocumentsLastMissingMerge() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(1, dv.getValueCount()); dv.setDocument(0); @@ -1800,7 +1799,7 @@ public void testSortedSetTwoDocumentsFirstMissing() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(1, dv.getValueCount()); dv.setDocument(1); @@ -1834,7 +1833,7 @@ public void testSortedSetTwoDocumentsFirstMissingMerge() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(1, dv.getValueCount()); dv.setDocument(1); @@ -1870,7 +1869,7 @@ public void testSortedSetMergeAwayAllValues() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(0, dv.getValueCount()); ireader.close(); @@ -1894,7 +1893,7 @@ public void testSortedSetTermsEnum() throws IOException { DirectoryReader ireader = iwriter.getReader(); iwriter.close(); - SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field"); + SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field"); assertEquals(3, dv.getValueCount()); TermsEnum termsEnum = dv.termsEnum(); @@ -2784,13 +2783,12 @@ public void testEmptyBinaryValueOnPageSizes() throws Exception { IndexReader r = w.getReader(); w.close(); - LeafReader ar = SlowCompositeReaderWrapper.wrap(r); - BinaryDocValues values = ar.getBinaryDocValues("field"); + BinaryDocValues values = MultiDocValues.getBinaryValues(r, "field"); for(int j=0;j<5;j++) { BytesRef result = values.get(0); assertTrue(result.length == 0 || result.length == 1< - * NOTE: This should be used for testing purposes only - * @lucene.internal - */ -public class MultiSpansWrapper { - - public static Spans wrap(IndexReader reader, SpanQuery spanQuery) throws IOException { - return wrap(reader, spanQuery, SpanWeight.Postings.POSITIONS); - } - - public static Spans wrap(IndexReader reader, SpanQuery spanQuery, SpanWeight.Postings requiredPostings) throws IOException { - - LeafReader lr = SlowCompositeReaderWrapper.wrap(reader); // slow, but ok for testing - LeafReaderContext lrContext = lr.getContext(); - IndexSearcher searcher = new IndexSearcher(lr); - searcher.setQueryCache(null); - - SpanWeight w = spanQuery.createWeight(searcher, false); - - return w.getSpans(lrContext, requiredPostings); - } -} diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 79eb62803eb3..e5aa7a2e4efb 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -764,15 +764,29 @@ public String getTestName() { * Some tests expect the directory to contain a single segment, and want to * do tests on that segment's reader. This is an utility method to help them. */ + /* public static SegmentReader getOnlySegmentReader(DirectoryReader reader) { List subReaders = reader.leaves(); if (subReaders.size() != 1) { throw new IllegalArgumentException(reader + " has " + subReaders.size() + " segments instead of exactly one"); } final LeafReader r = subReaders.get(0).reader(); - assertTrue(r instanceof SegmentReader); + assertTrue("expected a SegmentReader but got " + r, r instanceof SegmentReader); return (SegmentReader) r; } + */ + + /** + * Some tests expect the directory to contain a single segment, and want to + * do tests on that segment's reader. This is an utility method to help them. + */ + public static LeafReader getOnlyLeafReader(IndexReader reader) { + List subReaders = reader.leaves(); + if (subReaders.size() != 1) { + throw new IllegalArgumentException(reader + " has " + subReaders.size() + " segments instead of exactly one"); + } + return subReaders.get(0).reader(); + } /** * Returns true if and only if the calling thread is the primary thread @@ -1625,25 +1639,11 @@ static Directory newDirectoryImpl(Random random, String clazzName, LockFactory l } public static IndexReader wrapReader(IndexReader r) throws IOException { - return wrapReader(r, true); - } - - public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeReader) throws IOException { Random random = random(); - // TODO: remove this, and fix those tests to wrap before putting slow around: - final boolean wasOriginallyAtomic = r instanceof LeafReader; for (int i = 0, c = random.nextInt(6)+1; i < c; i++) { - switch(random.nextInt(6)) { + switch(random.nextInt(5)) { case 0: - if (allowSlowCompositeReader) { - if (VERBOSE) { - System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with SlowCompositeReaderWrapper.wrap"); - } - r = SlowCompositeReaderWrapper.wrap(r); - } - break; - case 1: // will create no FC insanity in atomic case, as ParallelLeafReader has own cache key: if (VERBOSE) { System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with ParallelLeaf/CompositeReader"); @@ -1652,7 +1652,7 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe new ParallelLeafReader((LeafReader) r) : new ParallelCompositeReader((CompositeReader) r); break; - case 2: + case 1: // Häckidy-Hick-Hack: a standard MultiReader will cause FC insanity, so we use // QueryUtils' reader with a fake cache key, so insanity checker cannot walk // along our reader: @@ -1661,9 +1661,9 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe } r = new FCInvisibleMultiReader(r); break; - case 3: - if (allowSlowCompositeReader) { - final LeafReader ar = SlowCompositeReaderWrapper.wrap(r); + case 2: + if (r instanceof LeafReader) { + final LeafReader ar = (LeafReader) r; final List allFields = new ArrayList<>(); for (FieldInfo fi : ar.getFieldInfos()) { allFields.add(fi.name); @@ -1673,7 +1673,7 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe final Set fields = new HashSet<>(allFields.subList(0, end)); // will create no FC insanity as ParallelLeafReader has own cache key: if (VERBOSE) { - System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with ParallelLeafReader(SlowCompositeReaderWapper)"); + System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with ParallelLeafReader"); } r = new ParallelLeafReader( new FieldFilterLeafReader(ar, fields, false), @@ -1681,7 +1681,7 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe ); } break; - case 4: + case 3: // Häckidy-Hick-Hack: a standard Reader will cause FC insanity, so we use // QueryUtils' reader with a fake cache key, so insanity checker cannot walk // along our reader: @@ -1694,7 +1694,7 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe r = new AssertingDirectoryReader((DirectoryReader)r); } break; - case 5: + case 4: if (VERBOSE) { System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with MismatchedLeaf/DirectoryReader"); } @@ -1708,11 +1708,8 @@ public static IndexReader wrapReader(IndexReader r, boolean allowSlowCompositeRe fail("should not get here"); } } - if (wasOriginallyAtomic) { - if (allowSlowCompositeReader) { - r = SlowCompositeReaderWrapper.wrap(r); - } - } else if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) { + + if ((r instanceof CompositeReader) && !(r instanceof FCInvisibleMultiReader)) { // prevent cache insanity caused by e.g. ParallelCompositeReader, to fix we wrap one more time: r = new FCInvisibleMultiReader(r); } diff --git a/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java b/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java index 22cd46701a54..52953490190d 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java +++ b/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java @@ -315,7 +315,7 @@ public int getOffsetGap(String fieldName) { doc.add(new Field("f", "a", ft)); doc.add(new Field("f", "a", ft)); writer.addDocument(doc); - final LeafReader reader = getOnlySegmentReader(writer.getReader()); + final LeafReader reader = getOnlyLeafReader(writer.getReader()); final Fields fields = reader.getTermVectors(0); final Terms terms = fields.terms("f"); final TermsEnum te = terms.iterator(); diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java index d84bed99ee33..c6396ae551d5 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java +++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java @@ -21,17 +21,18 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.BaseStoredFieldsFormatTestCase; import org.apache.lucene.index.CodecReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.ByteArrayDataInput; @@ -306,7 +307,7 @@ public void testChunkCleanup() throws IOException { assertNotNull(ir2); ir.close(); ir = ir2; - CodecReader sr = getOnlySegmentReader(ir); + CodecReader sr = (CodecReader) getOnlyLeafReader(ir); CompressingStoredFieldsReader reader = (CompressingStoredFieldsReader)sr.getFieldsReader(); // we could get lucky, and have zero, but typically one. assertTrue(reader.getNumDirtyChunks() <= 1); diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java index 4fa02786c703..f4858d16cd7e 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java +++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java @@ -25,18 +25,18 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.BaseTermVectorsFormatTestCase; import org.apache.lucene.index.CodecReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.BaseTermVectorsFormatTestCase; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -56,7 +56,7 @@ public void testNoOrds() throws Exception { ft.setStoreTermVectors(true); doc.add(new Field("foo", "this is a test", ft)); iw.addDocument(doc); - LeafReader ir = getOnlySegmentReader(iw.getReader()); + LeafReader ir = getOnlyLeafReader(iw.getReader()); Terms terms = ir.getTermVector(0, "foo"); assertNotNull(terms); TermsEnum termsEnum = terms.iterator(); @@ -118,7 +118,7 @@ public void testChunkCleanup() throws IOException { assertNotNull(ir2); ir.close(); ir = ir2; - CodecReader sr = getOnlySegmentReader(ir); + CodecReader sr = (CodecReader) getOnlyLeafReader(ir); CompressingTermVectorsReader reader = (CompressingTermVectorsReader)sr.getTermVectorsReader(); // we could get lucky, and have zero, but typically one. assertTrue(reader.getNumDirtyChunks() <= 1); diff --git a/lucene/test-framework/src/test/org/apache/lucene/index/TestAssertingLeafReader.java b/lucene/test-framework/src/test/org/apache/lucene/index/TestAssertingLeafReader.java index b572289e9144..5276301ba481 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/index/TestAssertingLeafReader.java +++ b/lucene/test-framework/src/test/org/apache/lucene/index/TestAssertingLeafReader.java @@ -54,9 +54,8 @@ public void testAssertBits() throws Exception { assertEquals(1, r.numDocs()); r = new AssertingDirectoryReader((DirectoryReader) r); + final IndexReader r2 = r; - final IndexReader r2 = SlowCompositeReaderWrapper.wrap(r); - Thread thread = new Thread() { @Override public void run() { @@ -68,6 +67,6 @@ public void run() { thread.start(); thread.join(); - IOUtils.close(r2, dir); + IOUtils.close(r, dir); } } From 77d233bc01894c33f02178898dd3368b51902a11 Mon Sep 17 00:00:00 2001 From: Upayavira Date: Thu, 10 Mar 2016 23:48:48 +0000 Subject: [PATCH 0099/1113] SOLR-7858 - update links between new/old UIs for 6.x release --- solr/webapp/web/css/angular/common.css | 13 ++----------- solr/webapp/web/css/styles/common.css | 16 +++++++++++++++- solr/webapp/web/index.html | 6 ++---- solr/webapp/web/old.html | 5 +++-- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/solr/webapp/web/css/angular/common.css b/solr/webapp/web/css/angular/common.css index 1a3b087db5b0..d9604464d6cc 100644 --- a/solr/webapp/web/css/angular/common.css +++ b/solr/webapp/web/css/angular/common.css @@ -762,16 +762,7 @@ pre.syntax .tex .formula padding-left: 16px; } -.new-ui-warning { - position: absolute; - left: 150px; - top: -20px; - align: center; - color: red; - font-weight: bold; -} -.new-ui-warning a.ul { - color: red; - font-weight: bold; +.other-ui-link a.ul { text-decoration: underline; } + diff --git a/solr/webapp/web/css/styles/common.css b/solr/webapp/web/css/styles/common.css index f0e0652fa17e..6c0a9fbcba34 100644 --- a/solr/webapp/web/css/styles/common.css +++ b/solr/webapp/web/css/styles/common.css @@ -714,4 +714,18 @@ pre.syntax .tex .formula } .other-ui-link a.ul { text-decoration: underline; -} \ No newline at end of file +} + +.old-ui-warning { + position: absolute; + right: 0px; + top: -20px; + align: center; + color: red; + font-weight: bold; +} +.old-ui-warning a.ul { + color: red; + font-weight: bold; + text-decoration: underline; +} diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html index 3c230478ada3..6c4df6afe86c 100644 --- a/solr/webapp/web/index.html +++ b/solr/webapp/web/index.html @@ -121,10 +121,8 @@

    Connection recovered...

    {{exception.msg}}
    -
    - This is an experimental UI. Report bugs here. - For the old UI click here -   +
    + Use original UI 
    diff --git a/solr/webapp/web/old.html b/solr/webapp/web/old.html index a25504e21ba3..d688a0a56e3c 100644 --- a/solr/webapp/web/old.html +++ b/solr/webapp/web/old.html @@ -79,8 +79,9 @@

    SolrCore Initialization Failures

    -
    - Try New UI  +
    + THIS USER INTERFACE IS DEPRECATED. Please use the current UI here +  
    From 8fe11a06ce9e8bfef99184b1849e459b15691bf2 Mon Sep 17 00:00:00 2001 From: anshum Date: Thu, 10 Mar 2016 12:22:23 -0800 Subject: [PATCH 0100/1113] SOLR-8790: Add the node name to core responses in calls from the Overseer --- solr/CHANGES.txt | 3 +++ .../apache/solr/cloud/OverseerCollectionMessageHandler.java | 1 + .../test/org/apache/solr/cloud/CollectionsAPISolrJTest.java | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a1b14d6afdc8..001138e6ab23 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -285,6 +285,9 @@ Bug Fixes * SOLR-8135: If a core reload is fired after a core close, it is not a non-recoverable error (noble) +* SOLR-8790: Collections API responses contain node name in the core-level responses that are + returned. (Anshum Gupta) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index d7d894bc69b7..78db47332f0a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -1743,6 +1743,7 @@ public static void sendShardRequest(String nodeName, ModifiableSolrParams params String replica = zkStateReader.getBaseUrlForNodeName(nodeName); sreq.shards = new String[]{replica}; sreq.actualShards = sreq.shards; + sreq.nodeName = nodeName; sreq.params = params; shardHandler.submit(sreq, replica, sreq.params); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index 9f41c0df807c..6b0db0f151f9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -54,7 +54,6 @@ public class CollectionsAPISolrJTest extends AbstractFullDistribZkTestBase { @Test - @Ignore public void test() throws Exception { testCreateAndDeleteCollection(); testCreateAndDeleteShard(); @@ -80,6 +79,7 @@ protected void testCreateAndDeleteCollection() throws Exception { .setConfigName("conf1") .setRouterField("myOwnField") .setStateFormat(1); + CollectionAdminResponse response = createCollectionRequest.process(cloudClient); assertEquals(0, response.getStatus()); From bc41158648fc031cc1fe7b07ed62f5f140854f59 Mon Sep 17 00:00:00 2001 From: anshum Date: Thu, 10 Mar 2016 13:59:37 -0800 Subject: [PATCH 0101/1113] SOLR-8765: Fix CollectionAdminRequest.SplitShard to accept requests without the 'shard' parameter --- .../handler/admin/CollectionsHandler.java | 2 +- .../solrj/request/CollectionAdminRequest.java | 36 ++++++++++++------- .../common/params/CollectionAdminParams.java | 2 ++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 06968c381484..b4d0a1d06d91 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -454,7 +454,7 @@ Map call(SolrQueryRequest req, SolrQueryResponse rsp, Collection String splitKey = req.getParams().get("split.key"); if (splitKey == null && shard == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Missing required parameter: shard"); + throw new SolrException(ErrorCode.BAD_REQUEST, "At least one of shard, or split.key should be specified."); } if (splitKey != null && shard != null) { throw new SolrException(ErrorCode.BAD_REQUEST, diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 76eb19f76409..0c25e09b5cd4 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -616,27 +616,30 @@ public SolrParams getParams() { /** * Returns a SolrRequest to split a shard in a collection */ - public static SplitShard splitShard(String collection, String shard) { - return new SplitShard(collection, shard); + public static SplitShard splitShard(String collection) { + return new SplitShard(collection); } // SPLITSHARD request - public static class SplitShard extends AsyncShardSpecificAdminRequest { + public static class SplitShard extends AsyncCollectionAdminRequest { + protected String collection; protected String ranges; protected String splitKey; + protected String shard; private Properties properties; - private SplitShard(String collection, String shard) { - super(CollectionAction.SPLITSHARD, collection, shard); + private SplitShard(String collection) { + super(CollectionAction.SPLITSHARD); + this.collection = collection; } /** - * @deprecated Use {@link #splitShard(String, String)} + * @deprecated Use {@link #splitShard(String)} */ @Deprecated public SplitShard() { - super(CollectionAction.SPLITSHARD, null, null); + super(CollectionAction.SPLITSHARD); } public SplitShard setRanges(String ranges) { this.ranges = ranges; return this; } @@ -660,15 +663,12 @@ public SplitShard setProperties(Properties properties) { return this; } - @Override @Deprecated public SplitShard setCollectionName(String collection) { this.collection = collection; return this; } - @Override - @Deprecated public SplitShard setShardName(String shard) { this.shard = shard; return this; @@ -684,10 +684,20 @@ public SplitShard setAsyncId(String id) { @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set( "ranges", ranges); - if(splitKey != null) - params.set("split.key", this.splitKey); + if(this.collection == null) { + throw new IllegalArgumentException("You must set collection name for this request."); + } + + params.set(CollectionAdminParams.COLLECTION, collection); + + if (this.shard == null && this.splitKey == null) { + throw new IllegalArgumentException("You must set shardname OR splitkey for this request."); + } + + params.set("shard", shard); + params.set("split.key", this.splitKey); + params.set( "ranges", ranges); if(properties != null) { addProperties(params, properties); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index b9bf717663f2..e817dd9664d2 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -21,4 +21,6 @@ public abstract class CollectionAdminParams { /* Param used by DELETESTATUS call to clear all stored responses */ public static final String FLUSH = "flush"; + public static final String COLLECTION = "collection"; + } From b8e454c1daf52b4f6126c0997e4e0b75eae9e34d Mon Sep 17 00:00:00 2001 From: Varun Thacker Date: Fri, 11 Mar 2016 14:01:01 +0530 Subject: [PATCH 0102/1113] SOLR-8804: Fix a race condition in the ClusterStatus API call --- solr/CHANGES.txt | 3 +++ .../solr/handler/admin/ClusterStatus.java | 16 ++++++++++++---- .../apache/solr/cloud/TestCollectionAPI.java | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 001138e6ab23..d43d35cf2bee 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -288,6 +288,9 @@ Bug Fixes * SOLR-8790: Collections API responses contain node name in the core-level responses that are returned. (Anshum Gupta) +* SOLR-8804: Fix a race condition in the ClusterStatus API call whereby the call would fail when a concurrent delete + collection api command was executed (Alexey Serba, Varun Thacker) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java index ff60adc465b3..63044fa338f1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java @@ -89,7 +89,7 @@ public void getClusterStatus(NamedList results) byte[] bytes = Utils.toJSON(clusterState); Map stateMap = (Map) Utils.fromJSON(bytes); - Set collections = new HashSet<>(); + Set collections; String routeKey = message.getStr(ShardParams._ROUTE_); String shard = message.getStr(ZkStateReader.SHARD_ID_PROP); if (collection == null) { @@ -98,11 +98,19 @@ public void getClusterStatus(NamedList results) collections = Collections.singleton(collection); } - NamedList collectionProps = new SimpleOrderedMap(); + NamedList collectionProps = new SimpleOrderedMap<>(); for (String name : collections) { - Map collectionStatus = null; - DocCollection clusterStateCollection = clusterState.getCollection(name); + Map collectionStatus; + DocCollection clusterStateCollection = clusterState.getCollectionOrNull(name); + if (clusterStateCollection == null) { + if (collection != null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + name + " not found"); + } else { + //collection might have got deleted at the same time + continue; + } + } Set requestedShards = new HashSet<>(); if (routeKey != null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java index 45b6f733becf..dec2bf580319 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java @@ -77,6 +77,7 @@ public void test() throws Exception { clusterStatusWithRouteKey(); clusterStatusAliasTest(); clusterStatusRolesTest(); + clusterStatusBadCollectionTest(); replicaPropTest(); clusterStatusZNodeVersion(); testClusterStateMigration(); @@ -318,6 +319,24 @@ private void clusterStatusRolesTest() throws Exception { } } + private void clusterStatusBadCollectionTest() throws Exception { + try (CloudSolrClient client = createCloudClient(null)) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString()); + params.set("collection", "bad_collection_name"); + SolrRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + + try { + client.request(request); + fail("Collection does not exist. An exception should be thrown"); + } catch (SolrException e) { + //expected + assertTrue(e.getMessage().contains("Collection: bad_collection_name not found")); + } + } + } + private void replicaPropTest() throws Exception { try (CloudSolrClient client = createCloudClient(null)) { client.connect(); From 030caba6cb9fe20fa41413aaf59b6ad6b58bd7d2 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 11 Mar 2016 04:54:01 -0500 Subject: [PATCH 0103/1113] remove TODO that was in fact done --- lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 5388bb3c97b2..aa7e5dcbac70 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -1059,9 +1059,6 @@ private void build(int nodeID, int leafNodeOffset, assert count > 0: "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset; writeLeafBlockDocs(out, heapSource.docIDs, Math.toIntExact(source.start), count); - // TODO: we should delta compress / only write suffix bytes, like terms dict (the values will all be "close together" since we are at - // a leaf cell): - // First pass: find the per-dim common prefix for all values in this block: Arrays.fill(commonPrefixLengths, bytesPerDim); for (int i=0;i Date: Fri, 11 Mar 2016 04:53:43 -0500 Subject: [PATCH 0104/1113] test bug: close the right reader (the one we passed to newSearcher) --- .../test/org/apache/lucene/queries/CommonTermsQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java index e991b0cb61dd..fa7449784a1b 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java @@ -489,7 +489,7 @@ protected boolean lessThan(TermAndFreq a, TermAndFreq b) { QueryUtils.check(random(), cq, newSearcher(reader2)); reader2.close(); } finally { - IOUtils.close(wrapper, w, dir, analyzer); + IOUtils.close(reader, w, dir, analyzer); } } From bb51219594375365cc1315568f4f8a76da130676 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 11 Mar 2016 05:34:19 -0500 Subject: [PATCH 0105/1113] remove dead code --- .../java/org/apache/lucene/util/bkd/OfflinePointReader.java | 3 +-- .../java/org/apache/lucene/util/bkd/OfflinePointWriter.java | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 14d714cab1ff..83d863ba3565 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -30,7 +30,6 @@ final class OfflinePointReader implements PointReader { private final byte[] packedValue; private long ord; private int docID; - final int bytesPerDoc; OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length) throws IOException { this(tempDir.openInput(tempFileName, IOContext.READONCE), packedBytesLength, start, length); @@ -38,7 +37,7 @@ final class OfflinePointReader implements PointReader { private OfflinePointReader(IndexInput in, int packedBytesLength, long start, long length) throws IOException { this.in = in; - bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; + int bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; long seekFP = start * bytesPerDoc; in.seek(seekFP); this.countLeft = length; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index f47f5675538d..625e6fa2b13b 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -28,7 +28,6 @@ final class OfflinePointWriter implements PointWriter { final Directory tempDir; final IndexOutput out; final int packedBytesLength; - final int bytesPerDoc; private long count; private boolean closed; @@ -36,7 +35,6 @@ public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int pack this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd", IOContext.DEFAULT); this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; - bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; } /** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */ @@ -44,7 +42,6 @@ public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLen this.out = out; this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; - bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; this.count = count; closed = true; } From 11c8d33bc7d08cb6166c71d241ccab872b93711f Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 11 Mar 2016 06:48:30 -0500 Subject: [PATCH 0106/1113] fix int overflow bug in BKDWriter that prevented it from indexing > 2.1B points; try to improve runtime of Test2BPoints --- .../org/apache/lucene/util/bkd/BKDWriter.java | 4 +- .../org/apache/lucene/index/Test2BPoints.java | 62 ++++++++++++++----- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index aa7e5dcbac70..f5a2d8107a8d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -1082,7 +1082,7 @@ private void build(int nodeID, int leafNodeOffset, // Second pass: write the full values: byte[] lastPackedValue = new byte[bytesPerDim]; - for (int i=0;i Date: Fri, 11 Mar 2016 13:26:01 +0000 Subject: [PATCH 0107/1113] SOLR-8730: Fix highlighting in new UI query pane --- solr/webapp/web/partials/query.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/webapp/web/partials/query.html b/solr/webapp/web/partials/query.html index 87f2f4e7d613..31bc24200663 100644 --- a/solr/webapp/web/partials/query.html +++ b/solr/webapp/web/partials/query.html @@ -215,26 +215,26 @@
    - + - + - + From 164d6ccd2d4e0a18603bd987ba585fe856c9e0ee Mon Sep 17 00:00:00 2001 From: yonik Date: Fri, 11 Mar 2016 14:10:37 -0500 Subject: [PATCH 0108/1113] SOLR-8831: allow _version_ field to be retrievable via docValues --- solr/CHANGES.txt | 2 ++ .../core/src/java/org/apache/solr/update/VersionInfo.java | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d43d35cf2bee..044cfae48239 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -192,6 +192,8 @@ New Features * SOLR-8698: params.json can now specify 'appends' and 'invariants' (noble) +* SOLR-8831: allow _version_ field to be retrievable via docValues (yonik) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index d5eebec883d3..5fe415c61101 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -61,7 +61,7 @@ public class VersionInfo { */ public static SchemaField getAndCheckVersionField(IndexSchema schema) throws SolrException { - final String errPrefix = VERSION_FIELD + " field must exist in schema, using indexed=\"true\" or docValues=\"true\", stored=\"true\" and multiValued=\"false\""; + final String errPrefix = VERSION_FIELD + " field must exist in schema and be searchable (indexed or docValues) and retrievable(stored or docValues) and not multiValued"; SchemaField sf = schema.getFieldOrNull(VERSION_FIELD); if (null == sf) { @@ -72,12 +72,12 @@ public static SchemaField getAndCheckVersionField(IndexSchema schema) if ( !sf.indexed() && !sf.hasDocValues()) { throw new SolrException (SolrException.ErrorCode.SERVER_ERROR, - errPrefix + " (" + VERSION_FIELD + " must be either indexed or have docValues"); + errPrefix + " (" + VERSION_FIELD + " not searchable"); } - if ( !sf.stored() ) { + if ( !sf.stored() && !sf.hasDocValues()) { throw new SolrException (SolrException.ErrorCode.SERVER_ERROR, - errPrefix + " (" + VERSION_FIELD + " is not stored"); + errPrefix + " (" + VERSION_FIELD + " not retrievable"); } if ( sf.multiValued() ) { throw new SolrException From 26f230a4740e281ee9b43ed60bb8d24c4ed8dbdc Mon Sep 17 00:00:00 2001 From: jbernste Date: Fri, 11 Mar 2016 15:34:31 -0500 Subject: [PATCH 0109/1113] SOLR-8832: Faulty DaemonStream shutdown procedures --- .../apache/solr/client/solrj/io/stream/DaemonStream.java | 6 +++++- .../solr/client/solrj/io/stream/StreamExpressionTest.java | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java index 2f6539480d11..edafd7e92a7d 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java @@ -171,11 +171,16 @@ public void setStreamContext(StreamContext streamContext) { this.tupleStream.setStreamContext(streamContext); } + public void shutdown() { + streamRunner.setShutdown(true); + } + public void close() { if(closed) { return; } streamRunner.setShutdown(true); + this.closed = true; } public List children() { @@ -226,7 +231,6 @@ public StreamRunner(long runInterval, String id) { public synchronized void setShutdown(boolean shutdown) { this.shutdown = shutdown; - interrupt(); //We could be blocked on the queue or sleeping } public synchronized boolean getShutdown() { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index 465369b15279..e7f57c102b68 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -623,7 +623,7 @@ private void testDaemonStream() throws Exception { DaemonStream daemonStream; expression = StreamExpressionParser.parse("daemon(rollup(" - + "search(collection1, q=*:*, fl=\"a_i,a_s\", sort=\"a_s asc\")," + + "search(collection1, q=\"*:*\", fl=\"a_i,a_s\", sort=\"a_s asc\")," + "over=\"a_s\"," + "sum(a_i)" + "), id=\"test\", runInterval=\"1000\", queueSize=\"9\")"); @@ -2366,13 +2366,17 @@ private void testTopicStream() throws Exception{ assertEquals(14, (long) tuple.getLong(id)); tuple = dstream.read(); // This should trigger a checkpoint as it's the 4th read from the stream. assertEquals(15, (long) tuple.getLong(id)); + + dstream.shutdown(); + tuple = dstream.read(); + assertTrue(tuple.EOF); } finally { dstream.close(); } } finally { - cache.close(); del("*:*"); commit(); + cache.close(); } } From e11182c6281e1d77bb6280135e846c97f41198d4 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 11 Mar 2016 18:43:34 -0500 Subject: [PATCH 0110/1113] Replace O(N^2) cost bitset clearing with O(N) --- .../org/apache/lucene/util/bkd/BKDWriter.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index f5a2d8107a8d..f1aba9d12197 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -1126,6 +1126,14 @@ private void build(int nodeID, int leafNodeOffset, byte[] maxSplitPackedValue = new byte[packedBytesLength]; System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedBytesLength); + // When we are on this dim, below, we clear the ordBitSet: + int dimToClear; + if (numDims - 1 == splitDim) { + dimToClear = numDims - 2; + } else { + dimToClear = numDims - 1; + } + for(int dim=0;dim 1) { - ordBitSet.clear(0, pointCount); - } - // Recurse on left tree: build(2*nodeID, leafNodeOffset, leftSlices, ordBitSet, out, From 51d02687a6be773ca68320c9a0b8d525170b127c Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 11 Mar 2016 19:29:22 -0500 Subject: [PATCH 0111/1113] LUCENE-7099: add newDistanceSort to sandbox LatLonPoint --- lucene/CHANGES.txt | 6 +- .../apache/lucene/document/LatLonPoint.java | 69 ++++++- .../LatLonPointDistanceComparator.java | 119 +++++++++++ .../lucene/document/LatLonPointSortField.java | 108 ++++++++++ .../lucene/document/TestLatLonPoint.java | 3 + .../document/TestLatLonPointDistanceSort.java | 190 ++++++++++++++++++ 6 files changed, 485 insertions(+), 10 deletions(-) create mode 100644 lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java create mode 100644 lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java create mode 100644 lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 832f81971246..0458262b3cd4 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -4,7 +4,11 @@ For more information on past and future Lucene versions, please see: http://s.apache.org/luceneversions ======================= Lucene 6.1.0 ======================= -(No Changes) + +New Features + +* LUCENE-7099: Add LatLonPoint.newDistanceSort to the sandbox's + LatLonPoint. (Robert Muir) Optimizations diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index fd3284b6e29b..e8c2f177849f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -18,14 +18,16 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; - +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; import org.apache.lucene.spatial.util.GeoUtils; /** @@ -35,10 +37,11 @@ * efficient. Multiple values for the same field in one document * is allowed. *

    - * This field defines static factory methods for creating common queries: + * This field defines static factory methods for common operations: *

      *
    • {@link #newBoxQuery newBoxQuery()} for matching points within a bounding box. *
    • {@link #newDistanceQuery newDistanceQuery()} for matching points within a specified distance. + *
    • {@link #newDistanceSort newDistanceSort()} for ordering documents by distance from a specified location. *
    • {@link #newPolygonQuery newPolygonQuery()} for matching points within an arbitrary polygon. *
    *

    @@ -50,6 +53,8 @@ // to the field is not actually what gets indexed. Float would be 1E-5 error vs 1E-7, but it might be // a better tradeoff? then it would be completely transparent to the user and lucene would be "lossless". public class LatLonPoint extends Field { + private long currentValue; + /** * Type for an indexed LatLonPoint *

    @@ -58,6 +63,7 @@ public class LatLonPoint extends Field { public static final FieldType TYPE = new FieldType(); static { TYPE.setDimensions(2, Integer.BYTES); + TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC); TYPE.freeze(); } @@ -69,9 +75,12 @@ public class LatLonPoint extends Field { */ public void setLocationValue(double latitude, double longitude) { byte[] bytes = new byte[8]; - NumericUtils.intToSortableBytes(encodeLatitude(latitude), bytes, 0); - NumericUtils.intToSortableBytes(encodeLongitude(longitude), bytes, Integer.BYTES); + int latitudeEncoded = encodeLatitude(latitude); + int longitudeEncoded = encodeLongitude(longitude); + NumericUtils.intToSortableBytes(latitudeEncoded, bytes, 0); + NumericUtils.intToSortableBytes(longitudeEncoded, bytes, Integer.BYTES); fieldsData = new BytesRef(bytes); + currentValue = (((long)latitudeEncoded) << 32) | (longitudeEncoded & 0xFFFFFFFFL); } /** @@ -98,15 +107,25 @@ public String toString() { result.append(name); result.append(':'); - BytesRef bytes = (BytesRef) fieldsData; - result.append(decodeLatitude(BytesRef.deepCopyOf(bytes).bytes, 0)); + result.append(decodeLatitude((int)(currentValue >> 32))); result.append(','); - result.append(decodeLongitude(BytesRef.deepCopyOf(bytes).bytes, Integer.BYTES)); + result.append(decodeLongitude((int)(currentValue & 0xFFFFFFFF))); result.append('>'); return result.toString(); } + /** + * Returns a 64-bit long, where the upper 32 bits are the encoded latitude, + * and the lower 32 bits are the encoded longitude. + * @see #decodeLatitude(int) + * @see #decodeLongitude(int) + */ + @Override + public Number numericValue() { + return currentValue; + } + // public helper methods (e.g. for queries) /** @@ -197,16 +216,22 @@ private static byte[][] encode(double latitude, double longitude) { /** helper: checks a fieldinfo and throws exception if its definitely not a LatLonPoint */ static void checkCompatible(FieldInfo fieldInfo) { - if (fieldInfo.getPointDimensionCount() != TYPE.pointDimensionCount()) { + // point/dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. + if (fieldInfo.getPointDimensionCount() != 0 && fieldInfo.getPointDimensionCount() != TYPE.pointDimensionCount()) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with numDims=" + fieldInfo.getPointDimensionCount() + " but this point type has numDims=" + TYPE.pointDimensionCount() + ", is the field really a LatLonPoint?"); } - if (fieldInfo.getPointNumBytes() != TYPE.pointNumBytes()) { + if (fieldInfo.getPointNumBytes() != 0 && fieldInfo.getPointNumBytes() != TYPE.pointNumBytes()) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with bytesPerDim=" + fieldInfo.getPointNumBytes() + " but this point type has bytesPerDim=" + TYPE.pointNumBytes() + ", is the field really a LatLonPoint?"); } + if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) { + throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + + " but this point type has docValuesType=" + TYPE.docValuesType() + + ", is the field really a LatLonPoint?"); + } } // static methods for generating queries @@ -298,4 +323,30 @@ public static Query newDistanceQuery(String field, double latitude, double longi public static Query newPolygonQuery(String field, double[] polyLats, double[] polyLons) { return new LatLonPointInPolygonQuery(field, polyLats, polyLons); } + + /** + * Creates a SortField for sorting by distance from a location. + *

    + * This sort orders documents by ascending distance from the location. The value returned in {@link FieldDoc} for + * the hits contains a Double instance with the distance in meters. + *

    + * If a document is missing the field, then by default it is treated as having {@link Double#POSITIVE_INFINITY} distance + * (missing last). You can change this by calling {@link SortField#setMissingValue(Object)} on the returned SortField + * to a different Double value. + *

    + * If a document contains multiple values for the field, the closest distance to the location is used. + *

    + * NOTE: distance sorting might be expensive for many documents. Consider restricting the document + * set with a {@link #newBoxQuery box}, {@link #newDistanceQuery radius} radius, or {@link #newPolygonQuery polygon} + * query for better performance + * + * @param field field name. cannot be null. + * @param latitude latitude at the center: must be within standard +/-90 coordinate bounds. + * @param longitude longitude at the center: must be within standard +/-180 coordinate bounds. + * @return SortField ordering documents by distance + * @throws IllegalArgumentException if {@code field} is null or location has invalid coordinates. + */ + public static SortField newDistanceSort(String field, double latitude, double longitude) { + return new LatLonPointSortField(field, latitude, longitude); + } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java new file mode 100644 index 000000000000..a5d85344d84c --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import java.io.IOException; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.spatial.util.GeoDistanceUtils; + +/** Compares docs by distance from an origin */ +class LatLonPointDistanceComparator extends FieldComparator implements LeafFieldComparator { + final String field; + final double latitude; + final double longitude; + final double missingValue; + + final double[] values; + double bottom; + double topValue; + SortedNumericDocValues currentDocs; + + public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits, double missingValue) { + this.field = field; + this.latitude = latitude; + this.longitude = longitude; + this.values = new double[numHits]; + this.missingValue = missingValue; + } + + @Override + public void setScorer(Scorer scorer) {} + + @Override + public int compare(int slot1, int slot2) { + return Double.compare(values[slot1], values[slot2]); + } + + @Override + public void setBottom(int slot) { + bottom = values[slot]; + } + + @Override + public void setTopValue(Double value) { + topValue = value.doubleValue(); + } + + @Override + public int compareBottom(int doc) throws IOException { + return Double.compare(bottom, distance(doc)); + } + + @Override + public void copy(int slot, int doc) throws IOException { + values[slot] = distance(doc); + } + + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + LeafReader reader = context.reader(); + FieldInfo info = reader.getFieldInfos().fieldInfo(field); + if (info != null) { + LatLonPoint.checkCompatible(info); + } + currentDocs = DocValues.getSortedNumeric(reader, field); + return this; + } + + @Override + public Double value(int slot) { + return Double.valueOf(values[slot]); + } + + @Override + public int compareTop(int doc) throws IOException { + return Double.compare(topValue, distance(doc)); + } + + // TODO: optimize for single-valued case? + // TODO: do all kinds of other optimizations! + double distance(int doc) { + currentDocs.setDocument(doc); + + int numValues = currentDocs.count(); + if (numValues == 0) { + return missingValue; + } + + double minValue = Double.POSITIVE_INFINITY; + for (int i = 0; i < numValues; i++) { + long encoded = currentDocs.valueAt(i); + double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); + double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); + minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude)); + } + return minValue; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java new file mode 100644 index 000000000000..9d3928ea8c37 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import java.io.IOException; + +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.SortField; +import org.apache.lucene.spatial.util.GeoUtils; + +/** + * Sorts by distance from an origin location. + */ +final class LatLonPointSortField extends SortField { + final double latitude; + final double longitude; + + LatLonPointSortField(String field, double latitude, double longitude) { + super(field, SortField.Type.CUSTOM); + if (field == null) { + throw new IllegalArgumentException("field cannot be null"); + } + if (GeoUtils.isValidLat(latitude) == false) { + throw new IllegalArgumentException("latitude: '" + latitude + "' is invalid"); + } + if (GeoUtils.isValidLon(longitude) == false) { + throw new IllegalArgumentException("longitude: '" + longitude + "' is invalid"); + } + this.latitude = latitude; + this.longitude = longitude; + setMissingValue(Double.POSITIVE_INFINITY); + } + + @Override + public FieldComparator getComparator(int numHits, int sortPos) throws IOException { + return new LatLonPointDistanceComparator(getField(), latitude, longitude, numHits, getMissingValue()); + } + + @Override + public Double getMissingValue() { + return (Double) super.getMissingValue(); + } + + @Override + public void setMissingValue(Object missingValue) { + if (missingValue == null) { + throw new IllegalArgumentException("Missing value cannot be null"); + } + if (missingValue.getClass() != Double.class) + throw new IllegalArgumentException("Missing value can only be of type java.lang.Double, but got " + missingValue.getClass()); + this.missingValue = missingValue; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + long temp; + temp = Double.doubleToLongBits(latitude); + result = prime * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(longitude); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + LatLonPointSortField other = (LatLonPointSortField) obj; + if (Double.doubleToLongBits(latitude) != Double.doubleToLongBits(other.latitude)) return false; + if (Double.doubleToLongBits(longitude) != Double.doubleToLongBits(other.longitude)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("", LatLonPoint.newDistanceSort("field", 18.0, 19.0).toString()); } /** Valid values that should not cause exception */ diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java new file mode 100644 index 000000000000..a0ee83f3de0a --- /dev/null +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.spatial.util.GeoDistanceUtils; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** Simple tests for {@link LatLonPoint#newDistanceSort} */ +public class TestLatLonPointDistanceSort extends LuceneTestCase { + + /** Add three points and sort by distance */ + public void testDistanceSort() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + + // add some docs + Document doc = new Document(); + doc.add(new LatLonPoint("location", 40.759011, -73.9844722)); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.718266, -74.007819)); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.7051157, -74.0088305)); + iw.addDocument(doc); + + IndexReader reader = iw.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + iw.close(); + + Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731)); + TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); + + FieldDoc d = (FieldDoc) td.scoreDocs[0]; + assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[1]; + assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[2]; + assertEquals(5291.798081404466D, (Double)d.fields[0], 0.0D); + + reader.close(); + dir.close(); + } + + /** Run a few iterations with just 10 docs, hopefully easy to debug */ + public void testRandom() throws Exception { + for (int iters = 0; iters < 100; iters++) { + doRandomTest(10, 100); + } + } + + /** Runs with thousands of docs */ + @Nightly + public void testRandomHuge() throws Exception { + for (int iters = 0; iters < 10; iters++) { + doRandomTest(2000, 100); + } + } + + // result class used for testing. holds an id+distance. + // we sort these with Arrays.sort and compare with lucene's results + static class Result implements Comparable { + int id; + double distance; + + Result(int id, double distance) { + this.id = id; + this.distance = distance; + } + + @Override + public int compareTo(Result o) { + int cmp = Double.compare(distance, o.distance); + if (cmp == 0) { + return Integer.compare(id, o.id); + } + return cmp; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(distance); + result = prime * result + (int) (temp ^ (temp >>> 32)); + result = prime * result + id; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + Result other = (Result) obj; + if (Double.doubleToLongBits(distance) != Double.doubleToLongBits(other.distance)) return false; + if (id != other.id) return false; + return true; + } + } + + private void doRandomTest(int numDocs, int numQueries) throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + for (int i = 0; i < numDocs; i++) { + double latRaw = -90 + 180.0 * random().nextDouble(); + double lonRaw = -180 + 360.0 * random().nextDouble(); + // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons + double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); + double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); + Document doc = new Document(); + doc.add(new StoredField("id", i)); + doc.add(new NumericDocValuesField("id", i)); + doc.add(new LatLonPoint("field", lat, lon)); + doc.add(new StoredField("lat", lat)); + doc.add(new StoredField("lon", lon)); + writer.addDocument(doc); + } + IndexReader reader = writer.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + for (int i = 0; i < numQueries; i++) { + double lat = -90 + 180.0 * random().nextDouble(); + double lon = -180 + 360.0 * random().nextDouble(); + + Result expected[] = new Result[reader.maxDoc()]; + + for (int doc = 0; doc < reader.maxDoc(); doc++) { + Document targetDoc = reader.document(doc); + double docLatitude = targetDoc.getField("lat").numericValue().doubleValue(); + double docLongitude = targetDoc.getField("lon").numericValue().doubleValue(); + double distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + int id = targetDoc.getField("id").numericValue().intValue(); + expected[doc] = new Result(id, distance); + } + + Arrays.sort(expected); + + // randomize the topN a bit + int topN = TestUtil.nextInt(random(), 1, reader.maxDoc()); + // sort by distance, then ID + Sort sort = new Sort(LatLonPoint.newDistanceSort("field", lat, lon), + new SortField("id", SortField.Type.INT)); + + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), topN, sort); + for (int resultNumber = 0; resultNumber < topN; resultNumber++) { + FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[resultNumber]; + Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]); + assertEquals(expected[resultNumber], actual); + } + } + reader.close(); + writer.close(); + dir.close(); + } +} From a4722845e3080dac06c59ca9837a602cf85f3e17 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 11 Mar 2016 21:20:07 -0500 Subject: [PATCH 0112/1113] LUCENE-7099: speed up embarrasingly slow sandbox/ distance sort --- .../LatLonPointDistanceComparator.java | 97 ++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index a5d85344d84c..e64f4b0eeace 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -27,8 +27,16 @@ import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.spatial.util.GeoDistanceUtils; +import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoUtils; -/** Compares docs by distance from an origin */ +/** + * Compares documents by distance from an origin point + *

    + * When the least competitive item on the priority queue changes (setBottom), we recompute + * a bounding box representing competitive distance to the top-N. Then in compareBottom, we can + * quickly reject hits based on bounding box alone without computing distance for every element. + */ class LatLonPointDistanceComparator extends FieldComparator implements LeafFieldComparator { final String field; final double latitude; @@ -40,6 +48,24 @@ class LatLonPointDistanceComparator extends FieldComparator implements L double topValue; SortedNumericDocValues currentDocs; + // current bounding box(es) for the bottom distance on the PQ. + // these are pre-encoded with LatLonPoint's encoding and + // used to exclude uncompetitive hits faster. + int minLon; + int maxLon; + int minLat; + int maxLat; + + // crossesDateLine is true, then we have a second box to check + boolean crossesDateLine; + int minLon2; + int maxLon2; + int minLat2; + int maxLat2; + + // the number of times setBottom has been called (adversary protection) + int setBottomCounter = 0; + public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits, double missingValue) { this.field = field; this.latitude = latitude; @@ -59,6 +85,52 @@ public int compare(int slot1, int slot2) { @Override public void setBottom(int slot) { bottom = values[slot]; + // make bounding box(es) to exclude non-competitive hits, but start + // sampling if we get called way too much: don't make gobs of bounding + // boxes if comparator hits a worst case order (e.g. backwards distance order) + if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { + GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom); + // pre-encode our box to our integer encoding, so we don't have to decode + // to double values for uncompetitive hits. This has some cost! + int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat); + int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat); + int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon); + int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon); + // be sure to not introduce quantization error in our optimization, just + // round up our encoded box safely in all directions. + if (minLatEncoded != Integer.MIN_VALUE) { + minLatEncoded--; + } + if (minLonEncoded != Integer.MIN_VALUE) { + minLonEncoded--; + } + if (maxLatEncoded != Integer.MAX_VALUE) { + maxLatEncoded++; + } + if (maxLonEncoded != Integer.MAX_VALUE) { + maxLonEncoded++; + } + crossesDateLine = box.crossesDateline(); + // crosses dateline: split + if (crossesDateLine) { + // box1 + minLon = Integer.MIN_VALUE; + maxLon = maxLonEncoded; + minLat = minLatEncoded; + maxLat = maxLatEncoded; + // box2 + minLon2 = minLonEncoded; + maxLon2 = Integer.MAX_VALUE; + minLat2 = minLatEncoded; + maxLat2 = maxLatEncoded; + } else { + minLon = minLonEncoded; + maxLon = maxLonEncoded; + minLat = minLatEncoded; + maxLat = maxLatEncoded; + } + } + setBottomCounter++; } @Override @@ -68,7 +140,28 @@ public void setTopValue(Double value) { @Override public int compareBottom(int doc) throws IOException { - return Double.compare(bottom, distance(doc)); + currentDocs.setDocument(doc); + + int numValues = currentDocs.count(); + if (numValues == 0) { + return Double.compare(bottom, missingValue); + } + + double minValue = Double.POSITIVE_INFINITY; + for (int i = 0; i < numValues; i++) { + long encoded = currentDocs.valueAt(i); + int latitudeBits = (int)(encoded >> 32); + int longitudeBits = (int)(encoded & 0xFFFFFFFF); + boolean outsideBox = ((latitudeBits < minLat || longitudeBits < minLon || latitudeBits > maxLat || longitudeBits > maxLon) && + (crossesDateLine == false || latitudeBits < minLat2 || longitudeBits < minLon2 || latitudeBits > maxLat2 || longitudeBits > maxLon2)); + // only compute actual distance if its inside "competitive bounding box" + if (outsideBox == false) { + double docLatitude = LatLonPoint.decodeLatitude(latitudeBits); + double docLongitude = LatLonPoint.decodeLongitude(longitudeBits); + minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude)); + } + } + return Double.compare(bottom, minValue); } @Override From da67b56272b4c8d6f4f5a02f87520c933cf728de Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sat, 12 Mar 2016 04:56:23 -0500 Subject: [PATCH 0113/1113] don't wrap index reader in this test: it requires leaf readers --- .../core/src/test/org/apache/lucene/search/TestTermScorer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java index 2cdcba4db2c6..1ce1bc671fe7 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java @@ -60,7 +60,7 @@ public void setUp() throws Exception { writer.forceMerge(1); indexReader = getOnlyLeafReader(writer.getReader()); writer.close(); - indexSearcher = newSearcher(indexReader); + indexSearcher = newSearcher(indexReader, false); indexSearcher.setSimilarity(new ClassicSimilarity()); } From a5b941baa1aa907886760a247227ecf2cbc3eaa1 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sat, 12 Mar 2016 05:11:51 -0500 Subject: [PATCH 0114/1113] LUCENE-7098: reduce OfflineSorter and BKDWriter IO by using 4 bytes instead of 8 bytes to encord ord in the common case Squashed commit of the following: commit 5ac2dcf2a972e46ccda3e7a8d8df5d0af58f712a Merge: 68acf7f 684b222 Author: Mike McCandless Date: Fri Mar 11 19:04:13 2016 -0500 Merge branch 'master' into intords commit 68acf7f9ee2e0249d90075bc035721c0a91619f7 Author: Mike McCandless Date: Fri Mar 11 19:04:01 2016 -0500 rename to totalPointCount and add comment; enforce that the caller doesn't exceed what they said; simplify the longOrds check to just compare to Integer.MAX_VALUE commit afc964b56015475e8c354fdc8b0e05c7fa074ec2 Merge: db79e36 fe21f7a Author: Mike McCandless Date: Fri Mar 11 10:17:09 2016 -0500 Merge branch 'master' into intords Conflicts: lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java commit db79e365e097153a05813eaa70603c601bce1853 Author: Mike McCandless Date: Fri Mar 11 10:15:05 2016 -0500 use int (4 bytes) not lon (8 bytes) if the number of points is less than ~2.1B --- .../simpletext/SimpleTextPointsWriter.java | 3 +- .../apache/lucene/codecs/PointsWriter.java | 14 +++- .../codecs/lucene60/Lucene60PointsWriter.java | 20 ++++- .../lucene/index/PointValuesWriter.java | 16 ++-- .../org/apache/lucene/util/bkd/BKDWriter.java | 83 ++++++++++++------- .../lucene/util/bkd/HeapPointReader.java | 12 ++- .../lucene/util/bkd/HeapPointWriter.java | 37 +++++++-- .../lucene/util/bkd/OfflinePointReader.java | 24 ++++-- .../lucene/util/bkd/OfflinePointWriter.java | 17 +++- .../org/apache/lucene/util/bkd/TestBKD.java | 22 +++-- 10 files changed, 178 insertions(+), 70 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java index d2b848d262cc..13494f5b3488 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java @@ -76,7 +76,8 @@ public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOExcept fieldInfo.getPointDimensionCount(), fieldInfo.getPointNumBytes(), BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, - BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP) { + BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, + values.size(fieldInfo.name)) { @Override protected void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java index 53db281777af..000d713fa9a4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java @@ -41,6 +41,17 @@ protected PointsWriter() { * from the incoming segment. The default codec overrides this for 1D fields and uses * a faster but more complex implementation. */ protected void mergeOneField(MergeState mergeState, FieldInfo fieldInfo) throws IOException { + long maxPointCount = 0; + for (int i=0;i Integer.MAX_VALUE; + + // dimensional values (numDims * bytesPerDim) + ord (int or long) + docID (int) + if (longOrds) { + bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; + } else { + bytesPerDoc = packedBytesLength + Integer.BYTES + Integer.BYTES; + } // As we recurse, we compute temporary partitions of the data, halving the // number of points at each recursion. Once there are few enough points, @@ -173,12 +187,12 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n } // We write first maxPointsSortInHeap in heap, then cutover to offline for additional points: - heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength); + heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength, longOrds); this.maxMBSortInHeap = maxMBSortInHeap; } - public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap) { + public static void verifyParams(int numDims, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) { // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it now, in case we want to use // remaining 4 bits for another purpose later if (numDims < 1 || numDims > MAX_DIMS) { @@ -193,13 +207,16 @@ public static void verifyParams(int numDims, int maxPointsInLeafNode, double max if (maxMBSortInHeap < 0.0) { throw new IllegalArgumentException("maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")"); } + if (totalPointCount < 0) { + throw new IllegalArgumentException("totalPointCount must be >=0 (got: " + totalPointCount + ")"); + } } /** If the current segment has too many points then we switchover to temp files / offline sort. */ private void switchToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: - offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength); + offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds); tempInput = offlinePointWriter.out; PointReader reader = heapPointWriter.getReader(0); for(int i=0;i totalPointCount) { + throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values"); + } docsSeen.set(docID); } @@ -437,6 +457,9 @@ public long merge(IndexOutput out, List docMaps, List 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue); valueCount++; + if (pointCount > totalPointCount) { + throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values"); + } if (leafCount == 0) { if (leafBlockFPs.size() > 0) { @@ -569,13 +592,10 @@ private void sortHeapPointWriter(final HeapPointWriter writer, int start, int le new IntroSorter() { private final byte[] pivotPackedValue = new byte[bytesPerDim]; private int pivotDocID; - private long pivotOrd; @Override protected void setPivot(int i) { pivotDocID = writer.docIDs[i]; - pivotOrd = writer.ords[i]; - int block = i / writer.valuesPerBlock; int index = i % writer.valuesPerBlock; System.arraycopy(writer.blocks.get(block), index*packedBytesLength+dim*bytesPerDim, pivotPackedValue, 0, bytesPerDim); @@ -593,12 +613,7 @@ protected int comparePivot(int j) { } // Tie-break - cmp = Integer.compare(pivotDocID, writer.docIDs[j]); - if (cmp != 0) { - return cmp; - } - - return Long.compare(pivotOrd, writer.ords[j]); + return Integer.compare(pivotDocID, writer.docIDs[j]); } @Override @@ -607,9 +622,15 @@ protected void swap(int i, int j) { writer.docIDs[i] = writer.docIDs[j]; writer.docIDs[j] = docID; - long ord = writer.ords[i]; - writer.ords[i] = writer.ords[j]; - writer.ords[j] = ord; + if (longOrds) { + long ord = writer.ordsLong[i]; + writer.ordsLong[i] = writer.ordsLong[j]; + writer.ordsLong[j] = ord; + } else { + int ord = writer.ords[i]; + writer.ords[i] = writer.ords[j]; + writer.ords[j] = ord; + } byte[] blockI = writer.blocks.get(i / writer.valuesPerBlock); int indexI = (i % writer.valuesPerBlock) * packedBytesLength; @@ -660,7 +681,7 @@ private PointWriter sort(int dim) throws IOException { sorted = heapPointWriter; } else { // Subsequent dims need a private copy - sorted = new HeapPointWriter((int) pointCount, (int) pointCount, packedBytesLength); + sorted = new HeapPointWriter((int) pointCount, (int) pointCount, packedBytesLength, longOrds); sorted.copyFrom(heapPointWriter); } @@ -691,10 +712,16 @@ public int compare(BytesRef a, BytesRef b) { } // Tie-break by docID: - reader.reset(a.bytes, a.offset + packedBytesLength + Long.BYTES, a.length); + int offset; + if (longOrds) { + offset = Long.BYTES; + } else { + offset = Integer.BYTES; + } + reader.reset(a.bytes, a.offset + packedBytesLength + offset, a.length); final int docIDA = reader.readInt(); - reader.reset(b.bytes, b.offset + packedBytesLength + Long.BYTES, b.length); + reader.reset(b.bytes, b.offset + packedBytesLength + offset, b.length); final int docIDB = reader.readInt(); // No need to tie break on ord, for the case where the same doc has the same value in a given dimension indexed more than once: it @@ -746,7 +773,7 @@ public boolean read(BytesRefBuilder ref) throws IOException { assert lastWriter[0] != null; - return new OfflinePointWriter(tempDir, lastWriter[0], packedBytesLength, pointCount); + return new OfflinePointWriter(tempDir, lastWriter[0], packedBytesLength, pointCount, longOrds); } } @@ -1005,7 +1032,7 @@ private int split(byte[] minPackedValue, byte[] maxPackedValue) { private PathSlice switchToHeap(PathSlice source) throws IOException { int count = Math.toIntExact(source.count); try ( - PointWriter writer = new HeapPointWriter(count, count, packedBytesLength); + PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds); PointReader reader = source.writer.getReader(source.start); ) { for(int i=0;i blocks; final int valuesPerBlock; final int packedBytesLength; - final long[] ords; + final long[] ordsLong; + final int[] ords; final int[] docIDs; final int end; final byte[] scratch; - HeapPointReader(List blocks, int valuesPerBlock, int packedBytesLength, long[] ords, int[] docIDs, int start, int end) { + HeapPointReader(List blocks, int valuesPerBlock, int packedBytesLength, int[] ords, long[] ordsLong, int[] docIDs, int start, int end) { this.blocks = blocks; this.valuesPerBlock = valuesPerBlock; this.ords = ords; + this.ordsLong = ordsLong; this.docIDs = docIDs; curRead = start-1; this.end = end; @@ -76,7 +78,11 @@ public int docID() { @Override public long ord() { - return ords[curRead]; + if (ordsLong != null) { + return ordsLong[curRead]; + } else { + return ords[curRead]; + } } @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java index 02361871c4c9..3b043d0ace0b 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java @@ -23,7 +23,8 @@ final class HeapPointWriter implements PointWriter { int[] docIDs; - long[] ords; + long[] ordsLong; + int[] ords; private int nextWrite; private boolean closed; final int maxSize; @@ -32,11 +33,15 @@ final class HeapPointWriter implements PointWriter { // NOTE: can't use ByteBlockPool because we need random-write access when sorting in heap final List blocks = new ArrayList<>(); - public HeapPointWriter(int initSize, int maxSize, int packedBytesLength) { + public HeapPointWriter(int initSize, int maxSize, int packedBytesLength, boolean longOrds) { docIDs = new int[initSize]; - ords = new long[initSize]; this.maxSize = maxSize; this.packedBytesLength = packedBytesLength; + if (longOrds) { + this.ordsLong = new long[initSize]; + } else { + this.ords = new int[initSize]; + } // 4K per page, unless each value is > 4K: valuesPerBlock = Math.max(1, 4096/packedBytesLength); } @@ -46,7 +51,14 @@ public void copyFrom(HeapPointWriter other) { throw new IllegalStateException("docIDs.length=" + docIDs.length + " other.nextWrite=" + other.nextWrite); } System.arraycopy(other.docIDs, 0, docIDs, 0, other.nextWrite); - System.arraycopy(other.ords, 0, ords, 0, other.nextWrite); + if (other.ords != null) { + assert this.ords != null; + System.arraycopy(other.ords, 0, ords, 0, other.nextWrite); + } else { + assert this.ordsLong != null; + System.arraycopy(other.ordsLong, 0, ordsLong, 0, other.nextWrite); + } + for(byte[] block : other.blocks) { blocks.add(block.clone()); } @@ -91,21 +103,30 @@ private long[] growExact(long[] arr, int size) { public void append(byte[] packedValue, long ord, int docID) { assert closed == false; assert packedValue.length == packedBytesLength; - if (ords.length == nextWrite) { + if (docIDs.length == nextWrite) { int nextSize = Math.min(maxSize, ArrayUtil.oversize(nextWrite+1, Integer.BYTES)); assert nextSize > nextWrite: "nextSize=" + nextSize + " vs nextWrite=" + nextWrite; - ords = growExact(ords, nextSize); docIDs = growExact(docIDs, nextSize); + if (ordsLong != null) { + ordsLong = growExact(ordsLong, nextSize); + } else { + ords = growExact(ords, nextSize); + } } writePackedValue(nextWrite, packedValue); - ords[nextWrite] = ord; + if (ordsLong != null) { + ordsLong[nextWrite] = ord; + } else { + assert ord <= Integer.MAX_VALUE; + ords[nextWrite] = (int) ord; + } docIDs[nextWrite] = docID; nextWrite++; } @Override public PointReader getReader(long start) { - return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, docIDs, (int) start, nextWrite); + return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, ordsLong, docIDs, (int) start, nextWrite); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 83d863ba3565..3c4b8b5ddd77 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -30,18 +30,22 @@ final class OfflinePointReader implements PointReader { private final byte[] packedValue; private long ord; private int docID; + // true if ords are written as long (8 bytes), else 4 bytes + private boolean longOrds; - OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length) throws IOException { - this(tempDir.openInput(tempFileName, IOContext.READONCE), packedBytesLength, start, length); - } - - private OfflinePointReader(IndexInput in, int packedBytesLength, long start, long length) throws IOException { - this.in = in; - int bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; + OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, boolean longOrds) throws IOException { + in = tempDir.openInput(tempFileName, IOContext.READONCE); + int bytesPerDoc = packedBytesLength + Integer.BYTES; + if (longOrds) { + bytesPerDoc += Long.BYTES; + } else { + bytesPerDoc += Integer.BYTES; + } long seekFP = start * bytesPerDoc; in.seek(seekFP); this.countLeft = length; packedValue = new byte[packedBytesLength]; + this.longOrds = longOrds; } @Override @@ -58,7 +62,11 @@ public boolean next() throws IOException { assert countLeft == -1; return false; } - ord = in.readLong(); + if (longOrds) { + ord = in.readLong(); + } else { + ord = in.readInt(); + } docID = in.readInt(); return true; } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 625e6fa2b13b..dcf678169fdd 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -30,27 +30,36 @@ final class OfflinePointWriter implements PointWriter { final int packedBytesLength; private long count; private boolean closed; + // true if ords are written as long (8 bytes), else 4 bytes + private boolean longOrds; - public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength) throws IOException { + public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds) throws IOException { this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd", IOContext.DEFAULT); this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; + this.longOrds = longOrds; } /** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */ - public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count) { + public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count, boolean longOrds) { this.out = out; this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.count = count; closed = true; + this.longOrds = longOrds; } @Override public void append(byte[] packedValue, long ord, int docID) throws IOException { assert packedValue.length == packedBytesLength; out.writeBytes(packedValue, 0, packedValue.length); - out.writeLong(ord); + if (longOrds) { + out.writeLong(ord); + } else { + assert ord <= Integer.MAX_VALUE; + out.writeInt((int) ord); + } out.writeInt(docID); count++; } @@ -58,7 +67,7 @@ public void append(byte[] packedValue, long ord, int docID) throws IOException { @Override public PointReader getReader(long start) throws IOException { assert closed; - return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, count-start); + return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, count-start, longOrds); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index f1402fc277fe..20177438ac33 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -40,9 +40,17 @@ public class TestBKD extends LuceneTestCase { + private long randomPointCount() { + if (random().nextBoolean()) { + return random().nextInt(Integer.MAX_VALUE); + } else { + return random().nextLong() & Long.MAX_VALUE; + } + } + public void testBasicInts1D() throws Exception { try (Directory dir = getDirectory(100)) { - BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f); + BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, randomPointCount()); byte[] scratch = new byte[4]; for(int docID=0;docID<100;docID++) { NumericUtils.intToSortableBytes(docID, scratch, 0); @@ -117,7 +125,7 @@ public void testRandomIntsNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, randomPointCount()); if (VERBOSE) { System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs); @@ -258,7 +266,7 @@ public void testBigIntNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); BigInteger[][] docs = new BigInteger[numDocs][]; byte[] scratch = new byte[numBytesPerDim*numDims]; @@ -431,7 +439,7 @@ public void testRandomBinaryBig() throws Exception { public void testTooLittleHeap() throws Exception { try (Directory dir = getDirectory(0)) { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001); + new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, randomPointCount()); }); assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode")); } @@ -554,7 +562,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi List docIDBases = null; int seg = 0; - BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB); + BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); IndexInput in = null; @@ -608,7 +616,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi seg++; maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000); maxMB = (float) 3.0 + (3*random().nextDouble()); - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); lastDocIDBase = docID; } } @@ -623,7 +631,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi out.close(); in = dir.openInput("bkd", IOContext.DEFAULT); seg++; - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); List readers = new ArrayList<>(); for(long fp : toMerge) { in.seek(fp); From 500fc15734480811613aa289789947379bc252e3 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sat, 12 Mar 2016 06:17:03 -0500 Subject: [PATCH 0115/1113] remove O(N^2) asserts --- lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index bb2402bbd147..6d3cf03c456d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -970,7 +970,6 @@ public String toString() { private byte[] markRightTree(long rightCount, int splitDim, PathSlice source, LongBitSet ordBitSet) throws IOException { // Now we mark ords that fall into the right half, so we can partition on all other dims that are not the split dim: - assert numDims == 1 || ordBitSet.cardinality() == 0: "cardinality=" + ordBitSet.cardinality(); // Read the split value, then mark all ords in the right tree (larger than the split value): try (PointReader reader = source.writer.getReader(source.start + source.count - rightCount)) { @@ -988,8 +987,6 @@ private byte[] markRightTree(long rightCount, int splitDim, PathSlice source, Lo assert result; ordBitSet.set(reader.ord()); } - - assert rightCount == ordBitSet.cardinality(): "rightCount=" + rightCount + " cardinality=" + ordBitSet.cardinality(); } } From 85d268ee8b1f15e0b35da42a44e4878c577e7506 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sat, 12 Mar 2016 10:58:32 -0500 Subject: [PATCH 0116/1113] LUCENE-7099: use two-phase iteration in LatLonPoint.newDistanceQuery --- lucene/CHANGES.txt | 3 + .../apache/lucene/document/LatLonPoint.java | 14 ++-- .../document/LatLonPointDistanceQuery.java | 66 +++++++++++++++++-- .../lucene/document/TestLatLonPoint.java | 10 +++ 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0458262b3cd4..559d3b302656 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -15,6 +15,9 @@ Optimizations * LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% speedup on merging 2D LatLonPoint values (Mike McCandless) +* LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase + iteration. (Robert Muir) + Other * LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index e8c2f177849f..ebf850c9d5a0 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -96,8 +96,10 @@ public LatLonPoint(String name, double latitude, double longitude) { } private static final int BITS = 32; - private static final double LONGITUDE_SCALE = (0x1L<> 32)); + double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); + if (GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude) <= radiusMeters) { + return true; + } + } + return false; + } + } + + @Override + public float matchCost() { + return 20; // TODO: make this fancier + } + }; + return new ConstantScoreScorer(this, score(), iterator); } }; } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 519478f99525..d180d58f5671 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -159,6 +159,16 @@ public void testEncodeDecode() throws Exception { assertEquals(-180.0, LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(-180.0)), ENCODING_TOLERANCE); } + public void testEncodeDecodeExtremeValues() throws Exception { + assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLatitude(-90.0)); + assertEquals(0, LatLonPoint.encodeLatitude(0.0)); + assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLatitude(90.0)); + + assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLongitude(-180.0)); + assertEquals(0, LatLonPoint.encodeLatitude(0.0)); + assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLongitude(180.0)); + } + public void testEncodeDecodeIsStable() throws Exception { int iters = atLeast(1000); for(int iter=0;iter Date: Sun, 13 Mar 2016 05:31:11 -0400 Subject: [PATCH 0117/1113] optimize BKD leaf block writing: use incoming sorted points to compute commonn prefix (saves one pass); remove an extra copy bytes --- .../simpletext/SimpleTextPointsWriter.java | 5 +- .../org/apache/lucene/util/bkd/BKDWriter.java | 87 ++++++++++--------- .../lucene/util/bkd/HeapPointReader.java | 1 - .../lucene/util/bkd/HeapPointWriter.java | 10 +++ 4 files changed, 57 insertions(+), 46 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java index 13494f5b3488..33af33e688e4 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java @@ -158,11 +158,10 @@ protected void writeCommonPrefixes(IndexOutput out, int[] commonPrefixLengths, b } @Override - protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes) throws IOException { + protected void writeLeafBlockPackedValue(IndexOutput out, int[] commonPrefixLengths, byte[] bytes, int bytesOffset) throws IOException { // NOTE: we don't do prefix coding, so we ignore commonPrefixLengths - assert bytes.length == packedBytesLength; write(out, BLOCK_VALUE); - write(out, new BytesRef(bytes, 0, bytes.length).toString()); + write(out, new BytesRef(bytes, bytesOffset, packedBytesLength).toString()); newline(out); } }) { diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 6d3cf03c456d..765b01c1a924 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -106,9 +106,9 @@ public class BKDWriter implements Closeable { final double maxMBSortInHeap; final byte[] scratchDiff; - final byte[] scratchPackedValue; final byte[] scratch1; final byte[] scratch2; + final BytesRef scratchBytesRef = new BytesRef(); final int[] commonPrefixLengths; protected final FixedBitSet docsSeen; @@ -152,7 +152,7 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n packedBytesLength = numDims * bytesPerDim; scratchDiff = new byte[bytesPerDim]; - scratchPackedValue = new byte[packedBytesLength]; + scratchBytesRef.length = packedBytesLength; scratch1 = new byte[packedBytesLength]; scratch2 = new byte[packedBytesLength]; commonPrefixLengths = new int[numDims]; @@ -455,7 +455,7 @@ public long merge(IndexOutput out, List docMaps, List 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue); + assert numDims > 1 || valueInOrder(valueCount, lastPackedValue, reader.state.scratchPackedValue, 0); valueCount++; if (pointCount > totalPointCount) { throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values"); @@ -502,7 +502,7 @@ public long merge(IndexOutput out, List docMaps, List 0) { + if (StringHelper.compare(bytesPerDim, packedValue.bytes, packedValue.offset + offset, maxPackedValue, offset) > 0) { return false; } } @@ -1060,16 +1060,35 @@ private void build(int nodeID, int leafNodeOffset, } if (nodeID >= leafNodeOffset) { + // Leaf node: write block + for (int dim=0;dim= maxPointsInLeafNode, so we better be in heap at this point: HeapPointWriter heapSource = (HeapPointWriter) source.writer; @@ -1083,37 +1102,21 @@ private void build(int nodeID, int leafNodeOffset, assert count > 0: "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset; writeLeafBlockDocs(out, heapSource.docIDs, Math.toIntExact(source.start), count); - // First pass: find the per-dim common prefix for all values in this block: - Arrays.fill(commonPrefixLengths, bytesPerDim); - for (int i=0;i 0 && StringHelper.compare(bytesPerDim, lastPackedValue, 0, packedValue, 0) > 0) { - throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue) + " ord=" + ord); + private boolean valueInOrder(long ord, byte[] lastPackedValue, byte[] packedValue, int packedValueOffset) { + if (ord > 0 && StringHelper.compare(bytesPerDim, lastPackedValue, 0, packedValue, packedValueOffset) > 0) { + throw new AssertionError("values out of order: last value=" + new BytesRef(lastPackedValue) + " current value=" + new BytesRef(packedValue, packedValueOffset, packedBytesLength) + " ord=" + ord); } - System.arraycopy(packedValue, 0, lastPackedValue, 0, bytesPerDim); + System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, bytesPerDim); return true; } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java index b178f085f923..63c78693b33a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.util.bkd; - import java.util.List; import org.apache.lucene.util.PagedBytes; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java index 3b043d0ace0b..45bb59181f47 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; final class HeapPointWriter implements PointWriter { int[] docIDs; @@ -72,6 +73,15 @@ void readPackedValue(int index, byte[] bytes) { System.arraycopy(blocks.get(block), blockIndex * packedBytesLength, bytes, 0, packedBytesLength); } + /** Returns a reference, in result, to the byte[] slice holding this value */ + void getPackedValueSlice(int index, BytesRef result) { + int block = index / valuesPerBlock; + int blockIndex = index % valuesPerBlock; + result.bytes = blocks.get(block); + result.offset = blockIndex * packedBytesLength; + assert result.length == packedBytesLength; + } + void writePackedValue(int index, byte[] bytes) { assert bytes.length == packedBytesLength; int block = index / valuesPerBlock; From d2aaae001df417ac44a9fe78d436ae21bf3c81b5 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 13 Mar 2016 05:35:02 -0400 Subject: [PATCH 0118/1113] don't wrap searchers in this test --- .../lucene/queries/payloads/TestPayloadSpans.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java index 179b971fbcaf..9e9228b16ae8 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java @@ -125,7 +125,7 @@ public void testSpanNot() throws Exception { IndexReader reader = getOnlyLeafReader(writer.getReader()); writer.close(); - checkSpans(snq.createWeight(newSearcher(reader), false).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2}); + checkSpans(snq.createWeight(newSearcher(reader, false), false).getSpans(reader.leaves().get(0), SpanWeight.Postings.PAYLOADS), 1, new int[]{2}); reader.close(); directory.close(); } @@ -264,7 +264,7 @@ public void testShrinkToAfterShortestMatch() throws IOException { writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); @@ -303,7 +303,7 @@ public void testShrinkToAfterShortestMatch2() throws IOException { doc.add(new TextField("content", new StringReader("a b a d k f a h i k a k"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); @@ -342,7 +342,7 @@ public void testShrinkToAfterShortestMatch3() throws IOException { doc.add(new TextField("content", new StringReader("j k a l f k k p a t a k l k t a"))); writer.addDocument(doc); IndexReader reader = writer.getReader(); - IndexSearcher is = newSearcher(getOnlyLeafReader(reader)); + IndexSearcher is = newSearcher(getOnlyLeafReader(reader), false); writer.close(); SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a")); @@ -438,7 +438,7 @@ private IndexSearcher getSearcher() throws Exception { closeIndexReader = writer.getReader(); writer.close(); - IndexSearcher searcher = newSearcher(closeIndexReader); + IndexSearcher searcher = newSearcher(closeIndexReader, false); return searcher; } From 85739d86299907e4d9f484d017c68a5b76fe075d Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 13 Mar 2016 06:28:18 -0400 Subject: [PATCH 0119/1113] fix int overflow --- lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 765b01c1a924..33d7bc446f6a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -1177,7 +1177,7 @@ private void build(int nodeID, int leafNodeOffset, PointReader reader = slices[dim].writer.getReader(slices[dim].start);) { // Partition this source according to how the splitDim split the values: - int nextRightCount = 0; + long nextRightCount = 0; for (long i=0;i Date: Sun, 13 Mar 2016 06:28:49 -0400 Subject: [PATCH 0120/1113] make BKD's temp file names a bit more descriptive --- .../src/java/org/apache/lucene/util/bkd/BKDWriter.java | 10 +++++----- .../org/apache/lucene/util/bkd/OfflinePointWriter.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 33d7bc446f6a..d4e30b722e62 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -216,7 +216,7 @@ public static void verifyParams(int numDims, int maxPointsInLeafNode, double max private void switchToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: - offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds); + offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "switch"); tempInput = offlinePointWriter.out; PointReader reader = heapPointWriter.getReader(0); for(int i=0;i Date: Sun, 13 Mar 2016 06:41:19 -0400 Subject: [PATCH 0121/1113] improve 2B points test; add new 2B test against BKD directly --- .../org/apache/lucene/index/Test2BPoints.java | 9 +- .../lucene/util/bkd/Test2BBKDPoints.java | 121 ++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java index 43207b8ceec3..75f2bbe3f47b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -70,11 +70,12 @@ public void test1D() throws Exception { } final int numDocs = (Integer.MAX_VALUE / 26) + 1; - long counter = 0; + int counter = 0; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); for (int j=0;j<26;j++) { - doc.add(new LongPoint("long", counter)); + long x = (((long) random().nextInt() << 32)) | (long) counter; + doc.add(new LongPoint("long", x)); counter++; } w.addDocument(doc); @@ -120,7 +121,9 @@ public void test2D() throws Exception { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); for (int j=0;j<26;j++) { - doc.add(new LongPoint("long", counter, 2*counter+1)); + long x = (((long) random().nextInt() << 32)) | (long) counter; + long y = (((long) random().nextInt() << 32)) | (long) random().nextInt(); + doc.add(new LongPoint("long", x, y)); counter++; } w.addDocument(doc); diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java new file mode 100644 index 000000000000..cf184094abcc --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util.bkd; + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.codecs.lucene60.Lucene60PointsReader; +import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.LuceneTestCase.Monster; +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.TimeUnits; + +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + +// e.g. run like this: ant test -Dtestcase=Test2BBKDPoints -Dtests.nightly=true -Dtests.verbose=true -Dtests.monster=true +// +// or: python -u /l/util/src/python/repeatLuceneTest.py -heap 4g -once -nolog -tmpDir /b/tmp -logDir /l/logs Test2BBKDPoints.test2D -verbose + +@TimeoutSuite(millis = 365 * 24 * TimeUnits.HOUR) // hopefully ~1 year is long enough ;) +@Monster("takes at least 4 hours and consumes many GB of temp disk space") +public class Test2BBKDPoints extends LuceneTestCase { + public void test1D() throws Exception { + Directory dir = FSDirectory.open(createTempDir("2BBKDPoints1D")); + + final int numDocs = (Integer.MAX_VALUE / 26) + 100; + + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 128, Long.BYTES, 26L * numDocs); + int counter = 0; + byte[] packedBytes = new byte[Long.BYTES]; + for (int docID = 0; docID < numDocs; docID++) { + for (int j=0;j<26;j++) { + // first a random int: + NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, 0); + // then our counter, which will overflow a bit in the end: + NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES); + w.add(packedBytes, docID); + counter++; + } + if (VERBOSE && docID % 100000 == 0) { + System.out.println(docID + " of " + numDocs + "..."); + } + } + IndexOutput out = dir.createOutput("1d.bkd", IOContext.DEFAULT); + long indexFP = w.finish(out); + out.close(); + + IndexInput in = dir.openInput("1d.bkd", IOContext.DEFAULT); + in.seek(indexFP); + BKDReader r = new BKDReader(in); + r.verify(numDocs); + in.close(); + dir.close(); + } + + public void test2D() throws Exception { + Directory dir = FSDirectory.open(createTempDir("2BBKDPoints2D")); + + final int numDocs = (Integer.MAX_VALUE / 26) + 100; + + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs); + long counter = 0; + byte[] packedBytes = new byte[2*Long.BYTES]; + for (int docID = 0; docID < numDocs; docID++) { + for (int j=0;j<26;j++) { + // first a random int: + NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, 0); + // then our counter, which will overflow a bit in the end: + NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES); + // then two random ints for the 2nd dimension: + NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES); + NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES); + w.add(packedBytes, docID); + counter++; + } + if (VERBOSE && docID % 100000 == 0) { + System.out.println(docID + " of " + numDocs + "..."); + } + } + IndexOutput out = dir.createOutput("2d.bkd", IOContext.DEFAULT); + long indexFP = w.finish(out); + out.close(); + + IndexInput in = dir.openInput("2d.bkd", IOContext.DEFAULT); + in.seek(indexFP); + BKDReader r = new BKDReader(in); + r.verify(numDocs); + in.close(); + dir.close(); + } +} From 44bd24b1d4d6abd20c24fed09f73acd0874c78e6 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 13 Mar 2016 06:53:24 -0400 Subject: [PATCH 0122/1113] let BKD use 256 MB heap in 2B tests --- .../org/apache/lucene/util/bkd/Test2BBKDPoints.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java index cf184094abcc..eb3aa4773e09 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -55,7 +55,7 @@ public void test1D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 128, Long.BYTES, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 256, Long.BYTES, 26L * numDocs); int counter = 0; byte[] packedBytes = new byte[Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { @@ -88,8 +88,8 @@ public void test2D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs); - long counter = 0; + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 1024, 256, Long.BYTES, 26L * numDocs); + int counter = 0; byte[] packedBytes = new byte[2*Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { for (int j=0;j<26;j++) { @@ -98,8 +98,8 @@ public void test2D() throws Exception { // then our counter, which will overflow a bit in the end: NumericUtils.intToSortableBytes(counter, packedBytes, Integer.BYTES); // then two random ints for the 2nd dimension: - NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES); - NumericUtils.intoSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES); + NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, Long.BYTES); + NumericUtils.intToSortableBytes(random().nextInt(), packedBytes, Long.BYTES + Integer.BYTES); w.add(packedBytes, docID); counter++; } From 42d1eb536461a5871f44a6d9f0b38ec839ce7283 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 13 Mar 2016 07:02:28 -0400 Subject: [PATCH 0123/1113] fix compiler warning --- lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java index 75f2bbe3f47b..5e6997971cb6 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -117,7 +117,7 @@ public void test2D() throws Exception { } final int numDocs = (Integer.MAX_VALUE / 26) + 1; - long counter = 0; + int counter = 0; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); for (int j=0;j<26;j++) { From 8a9b033349cdfb515cd1772acd15ee8e9c08fd7c Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 13 Mar 2016 08:55:31 -0400 Subject: [PATCH 0124/1113] optimize offline -> offline partition --- .../org/apache/lucene/util/bkd/BKDWriter.java | 19 +---- .../lucene/util/bkd/HeapPointReader.java | 4 +- .../lucene/util/bkd/OfflinePointReader.java | 77 ++++++++++++++++++- .../lucene/util/bkd/OfflinePointWriter.java | 2 +- .../apache/lucene/util/bkd/PointReader.java | 37 +++++++-- 5 files changed, 111 insertions(+), 28 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index d4e30b722e62..c5cdc303195c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -1176,24 +1176,7 @@ private void build(int nodeID, int leafNodeOffset, PointWriter rightPointWriter = getPointWriter(source.count - leftCount, "right" + dim); PointReader reader = slices[dim].writer.getReader(slices[dim].start);) { - // Partition this source according to how the splitDim split the values: - long nextRightCount = 0; - for (long i=0;i blocks; final int valuesPerBlock; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 3c4b8b5ddd77..c8ab47e1e296 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -22,9 +22,11 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.LongBitSet; /** Reads points from disk in a fixed-with format, previously written with {@link OfflinePointWriter}. */ -final class OfflinePointReader implements PointReader { +final class OfflinePointReader extends PointReader { long countLeft; private final IndexInput in; private final byte[] packedValue; @@ -90,5 +92,78 @@ public int docID() { public void close() throws IOException { in.close(); } + + @Override + public long split(long count, LongBitSet rightTree, PointWriter left, PointWriter right, boolean doClearBits) throws IOException { + + if (left instanceof OfflinePointWriter == false || + right instanceof OfflinePointWriter == false) { + return super.split(count, rightTree, left, right, doClearBits); + } + + // We specialize the offline -> offline split since the default impl + // is somewhat wasteful otherwise (e.g. decoding docID when we don't + // need to) + + int packedBytesLength = packedValue.length; + + int bytesPerDoc = packedBytesLength + Integer.BYTES; + if (longOrds) { + bytesPerDoc += Long.BYTES; + } else { + bytesPerDoc += Integer.BYTES; + } + + long rightCount = 0; + + IndexOutput rightOut = ((OfflinePointWriter) right).out; + IndexOutput leftOut = ((OfflinePointWriter) left).out; + + ((OfflinePointWriter) right).count = count; + ((OfflinePointWriter) left).count = count; + + assert count <= countLeft: "count=" + count + " countLeft=" + countLeft; + + countLeft -= count; + + byte[] buffer = new byte[bytesPerDoc]; + while (count > 0) { + in.readBytes(buffer, 0, buffer.length); + long ord; + if (longOrds) { + ord = readLong(buffer, packedBytesLength); + } else { + ord = readInt(buffer, packedBytesLength); + } + if (rightTree.get(ord)) { + rightOut.writeBytes(buffer, 0, bytesPerDoc); + if (doClearBits) { + rightTree.clear(ord); + } + rightCount++; + } else { + leftOut.writeBytes(buffer, 0, bytesPerDoc); + } + + count--; + } + + return rightCount; + } + + // Poached from ByteArrayDataInput: + private static long readLong(byte[] bytes, int pos) { + final int i1 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) | + ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff); + final int i2 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) | + ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff); + return (((long)i1) << 32) | (i2 & 0xFFFFFFFFL); + } + + // Poached from ByteArrayDataInput: + private static int readInt(byte[] bytes, int pos) { + return ((bytes[pos++] & 0xFF) << 24) | ((bytes[pos++] & 0xFF) << 16) + | ((bytes[pos++] & 0xFF) << 8) | (bytes[pos++] & 0xFF); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 5aa11de8977b..f9580503b01f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -28,7 +28,7 @@ final class OfflinePointWriter implements PointWriter { final Directory tempDir; final IndexOutput out; final int packedBytesLength; - private long count; + long count; private boolean closed; // true if ords are written as long (8 bytes), else 4 bytes private boolean longOrds; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java index fe7a9612426b..1919f584f47d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/PointReader.java @@ -20,21 +20,48 @@ import java.io.Closeable; import java.io.IOException; +import org.apache.lucene.util.LongBitSet; + /** One pass iterator through all points previously written with a * {@link PointWriter}, abstracting away whether points a read * from (offline) disk or simple arrays in heap. */ -interface PointReader extends Closeable { +abstract class PointReader implements Closeable { /** Returns false once iteration is done, else true. */ - boolean next() throws IOException; + abstract boolean next() throws IOException; /** Returns the packed byte[] value */ - byte[] packedValue(); + abstract byte[] packedValue(); /** Point ordinal */ - long ord(); + abstract long ord(); /** DocID for this point */ - int docID(); + abstract int docID(); + + /** Splits this reader into left and right partitions */ + public long split(long count, LongBitSet rightTree, PointWriter left, PointWriter right, boolean doClearBits) throws IOException { + + // Partition this source according to how the splitDim split the values: + long rightCount = 0; + for (long i=0;i Date: Sun, 13 Mar 2016 08:56:51 -0400 Subject: [PATCH 0125/1113] fix wrong param order in 2B tests --- .../core/src/test/org/apache/lucene/index/Test2BPoints.java | 4 ++-- .../src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java index 5e6997971cb6..5766b911230c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -86,7 +86,7 @@ public void test1D() throws Exception { w.forceMerge(1); DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(1250, s.count(LongPoint.newRangeQuery("long", 33640828, 33673327))); + assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", Long.MIN_VALUE, Long.MAX_VALUE))); assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE); r.close(); w.close(); @@ -134,7 +134,7 @@ public void test2D() throws Exception { w.forceMerge(1); DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(1250, s.count(LongPoint.newRangeQuery("long", new long[] {33640828, 33673327}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE}))); + assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MAX_VALUE}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE}))); assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE); r.close(); w.close(); diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java index eb3aa4773e09..4df2ebe19293 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -55,7 +55,7 @@ public void test1D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, 1024, 256, Long.BYTES, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 1024, 256, 26L * numDocs); int counter = 0; byte[] packedBytes = new byte[Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { @@ -88,7 +88,7 @@ public void test2D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, 1024, 256, Long.BYTES, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 1024, 256, 26L * numDocs); int counter = 0; byte[] packedBytes = new byte[2*Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { From d1ba89959137c120be5cf116e72f43b26339cb6e Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 14 Mar 2016 06:03:17 -0400 Subject: [PATCH 0126/1113] LUCENE-7097: let IntroSorter go 2X deeper in quicksort before switching to heapsort --- lucene/CHANGES.txt | 3 +++ .../core/src/java/org/apache/lucene/util/IntroSorter.java | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 559d3b302656..3618141cd5fb 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -18,6 +18,9 @@ Optimizations * LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase iteration. (Robert Muir) +* LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort + stack depth before switching to heapsort (Adrien Grand, Mike McCandless) + Other * LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' diff --git a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java index d9cdd62bf61b..498c06aa6ff1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java @@ -28,17 +28,13 @@ */ public abstract class IntroSorter extends Sorter { - static int ceilLog2(int n) { - return Integer.SIZE - Integer.numberOfLeadingZeros(n - 1); - } - /** Create a new {@link IntroSorter}. */ public IntroSorter() {} @Override public final void sort(int from, int to) { checkRange(from, to); - quicksort(from, to, ceilLog2(to - from)); + quicksort(from, to, 2 * MathUtil.log(to - from, 2)); } void quicksort(int from, int to, int maxDepth) { From 826ff66716cf3b9027295190edc9931f14d809c6 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 14 Mar 2016 10:21:57 -0400 Subject: [PATCH 0127/1113] fix test bug: these tests expect only one segment --- .../test/org/apache/lucene/search/spans/TestSpanCollection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java index dfc0439e32ca..7b90b2b82cd1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java @@ -61,7 +61,7 @@ public void setUp() throws Exception { super.setUp(); directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); + newIndexWriterConfig(new MockAnalyzer(random()))); for (int i = 0; i < docFields.length; i++) { Document doc = new Document(); doc.add(newField(FIELD, docFields[i], OFFSETS)); From 7f6fb8dae9a470162662ad797048bd110eb7678d Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 14 Mar 2016 10:42:03 -0400 Subject: [PATCH 0128/1113] fix test bug: this test expects single segment index --- .../org/apache/lucene/queries/payloads/TestPayloadSpans.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java index 9e9228b16ae8..2c0204d5f9f0 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadSpans.java @@ -435,6 +435,7 @@ private IndexSearcher getSearcher() throws Exception { writer.addDocument(doc); } + writer.forceMerge(1); closeIndexReader = writer.getReader(); writer.close(); From 5d7592bc14c4a20cb365e4cc8515bee92eec1f6f Mon Sep 17 00:00:00 2001 From: yonik Date: Mon, 14 Mar 2016 11:33:16 -0400 Subject: [PATCH 0129/1113] SOLR-8835: fix faceting exception (uif) on multi-valued numeric docValues --- solr/CHANGES.txt | 3 ++ .../apache/solr/search/facet/FacetField.java | 13 ++--- .../solr/search/facet/TestJsonFacets.java | 50 +++++++++++++++---- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 044cfae48239..f4e1c44a2872 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -293,6 +293,9 @@ Bug Fixes * SOLR-8804: Fix a race condition in the ClusterStatus API call whereby the call would fail when a concurrent delete collection api command was executed (Alexey Serba, Varun Thacker) +* SOLR-8835: JSON Facet API: fix faceting exception on multi-valued numeric fields that + have docValues. (yonik) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java index a1f113196c3b..ce7f9198f83d 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java @@ -125,13 +125,8 @@ public FacetProcessor createFacetProcessor(FacetContext fcontext) { org.apache.lucene.document.FieldType.LegacyNumericType ntype = ft.getNumericType(); - if (sf.hasDocValues() && ntype==null) { - // single and multi-valued string docValues - return new FacetFieldProcessorDV(fcontext, this, sf); - } - if (!multiToken) { - if (sf.getType().getNumericType() != null) { + if (ntype != null) { // single valued numeric (docvalues or fieldcache) return new FacetFieldProcessorNumeric(fcontext, this, sf); } else { @@ -140,8 +135,10 @@ public FacetProcessor createFacetProcessor(FacetContext fcontext) { } } - // multivalued but field doesn't have docValues - if (method == FacetMethod.DV) { + // multi-valued after this point + + if (sf.hasDocValues() || method == FacetMethod.DV) { + // single and multi-valued string docValues return new FacetFieldProcessorDV(fcontext, this, sf); } diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 83220edc468f..8aaede16862a 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -343,25 +343,27 @@ public void doStats(Client client, ModifiableSolrParams p) throws Exception { doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_s", "cat_s","cat_s", "where_s","where_s", "num_d","num_d", "num_i","num_i", "super_s","super_s", "val_b","val_b", "date","date_dt", "sparse_s","sparse_s" ,"multi_ss","multi_ss") ); // multi-valued strings, long/float substitute for int/double - doStatsTemplated(client, params(p, "facet","true", "rows","0", "noexist","noexist_ss", "cat_s","cat_ss", "where_s","where_ss", "num_d","num_f", "num_i","num_l", "super_s","super_ss", "val_b","val_b", "date","date_dt", "sparse_s","sparse_ss", "multi_ss","multi_ss") ); + doStatsTemplated(client, params(p, "facet","true", "rows","0", "noexist","noexist_ss", "cat_s","cat_ss", "where_s","where_ss", "num_d","num_f", "num_i","num_l", "num_is","num_ls", "num_fs", "num_ds", "super_s","super_ss", "val_b","val_b", "date","date_dt", "sparse_s","sparse_ss", "multi_ss","multi_ss") ); // multi-valued strings, method=dv for terms facets doStatsTemplated(client, params(p, "terms", "method:dv,", "rows", "0", "noexist", "noexist_ss", "cat_s", "cat_ss", "where_s", "where_ss", "num_d", "num_f", "num_i", "num_l", "super_s", "super_ss", "val_b", "val_b", "date", "date_dt", "sparse_s", "sparse_ss", "multi_ss", "multi_ss")); // single valued docvalues for strings, and single valued numeric doc values for numeric fields - doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") ); + doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "num_is","num_lds", "num_fs","num_dds", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") ); // multi-valued docvalues FacetFieldProcessorDV.unwrap_singleValued_multiDv = false; // better multi-valued coverage - doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); + doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); // multi-valued docvalues FacetFieldProcessorDV.unwrap_singleValued_multiDv = true; - doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); + doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sds", "cat_s","cat_sds", "where_s","where_sds", "num_d","num_d", "num_i","num_i", "num_is","num_ids", "num_fs","num_fds", "super_s","super_sds", "val_b","val_b", "date","date_dtds", "sparse_s","sparse_sds" ,"multi_ss","multi_sds") ); } public static void doStatsTemplated(Client client, ModifiableSolrParams p) throws Exception { p.set("Z_num_i", "Z_" + p.get("num_i") ); + if (p.get("num_is") == null) p.add("num_is","num_is"); + if (p.get("num_fs") == null) p.add("num_fs","num_fs"); MacroExpander m = new MacroExpander( p.getMap() ); @@ -369,6 +371,8 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw String where_s = m.expand("${where_s}"); String num_d = m.expand("${num_d}"); String num_i = m.expand("${num_i}"); + String num_is = m.expand("${num_is}"); + String num_fs = m.expand("${num_fs}"); String Z_num_i = m.expand("${Z_num_i}"); String val_b = m.expand("${val_b}"); String date = m.expand("${date}"); @@ -379,17 +383,17 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw client.deleteByQuery("*:*", null); SolrInputDocument doc = - sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", super_s, "zodiac", date, "2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one"); + sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", num_is,"2",num_is,"-5", num_fs,"2",num_fs,"-5", super_s, "zodiac", date, "2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one"); client.add(doc, null); client.add(doc, null); client.add(doc, null); // a couple of deleted docs - client.add(sdoc("id", "2", cat_s, "B", where_s, "NJ", num_d, "-9", num_i, "-5", super_s,"superman", date,"2002-02-02T02:02:02Z", val_b, "false" , multi_ss,"a", multi_ss,"b" , Z_num_i, "0"), null); + client.add(sdoc("id", "2", cat_s, "B", where_s, "NJ", num_d, "-9", num_i, "-5", num_is,"3",num_is,"-1", num_fs,"3",num_fs,"-1.5", super_s,"superman", date,"2002-02-02T02:02:02Z", val_b, "false" , multi_ss,"a", multi_ss,"b" , Z_num_i, "0"), null); client.add(sdoc("id", "3"), null); client.commit(); - client.add(sdoc("id", "4", cat_s, "A", where_s, "NJ", num_d, "2", num_i, "3", super_s,"spiderman", date,"2003-03-03T03:03:03Z" , multi_ss, "b", Z_num_i, ""+Integer.MIN_VALUE), null); - client.add(sdoc("id", "5", cat_s, "B", where_s, "NJ", num_d, "11", num_i, "7", super_s,"batman" , date,"2001-02-03T01:02:03Z" ,sparse_s,"two", multi_ss, "a"), null); + client.add(sdoc("id", "4", cat_s, "A", where_s, "NJ", num_d, "2", num_i, "3", num_is,"0",num_is,"3", num_fs,"0", num_fs,"3", super_s,"spiderman", date,"2003-03-03T03:03:03Z" , multi_ss, "b", Z_num_i, ""+Integer.MIN_VALUE), null); + client.add(sdoc("id", "5", cat_s, "B", where_s, "NJ", num_d, "11", num_i, "7", num_is,"0", num_fs,"0", super_s,"batman" , date,"2001-02-03T01:02:03Z" ,sparse_s,"two", multi_ss, "a"), null); client.commit(); - client.add(sdoc("id", "6", cat_s, "B", where_s, "NY", num_d, "-5", num_i, "-5", super_s,"hulk" , date,"2002-03-01T03:02:01Z" , multi_ss, "b", multi_ss, "a", Z_num_i, ""+Integer.MAX_VALUE), null); + client.add(sdoc("id", "6", cat_s, "B", where_s, "NY", num_d, "-5", num_i, "-5", num_is,"-1", num_fs,"-1.5", super_s,"hulk" , date,"2002-03-01T03:02:01Z" , multi_ss, "b", multi_ss, "a", Z_num_i, ""+Integer.MAX_VALUE), null); client.commit(); // test for presence of debugging info @@ -1097,6 +1101,34 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw ); + + // multi-valued integer + client.testJQ(params(p, "q", "*:*" + , "json.facet", "{ " + + " c1:'unique(${num_is})', c2:'hll(${num_is})'" + + ",f1:{${terms} type:terms, field:${num_is} } " + + "}" + ) + , "facets=={ count:6 " + + ", c1:5, c2:5" + + ", f1:{ buckets:[ {val:-1,count:2},{val:0,count:2},{val:3,count:2},{val:-5,count:1},{val:2,count:1} ] } " + + "} " + ); + + // multi-valued float + client.testJQ(params(p, "q", "*:*" + , "json.facet", "{ " + + " c1:'unique(${num_fs})', c2:'hll(${num_fs})'" + + ",f1:{${terms} type:terms, field:${num_fs} } " + + "}" + ) + , "facets=={ count:6 " + + ", c1:5, c2:5" + + ", f1:{ buckets:[ {val:-1.5,count:2},{val:0.0,count:2},{val:3.0,count:2},{val:-5.0,count:1},{val:2.0,count:1} ] } " + + "} " + ); + + } From ae866b01499a1474a2178749025c2c571b6a4777 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 14 Mar 2016 12:19:19 -0400 Subject: [PATCH 0130/1113] LUCENE-7099: improve test to exercise searchAfter --- .../document/TestLatLonPointDistanceSort.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index a0ee83f3de0a..ea36ea641b18 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -27,7 +27,6 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -59,7 +58,7 @@ public void testDistanceSort() throws Exception { iw.close(); Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731)); - TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); + TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); @@ -130,6 +129,11 @@ public boolean equals(Object obj) { if (id != other.id) return false; return true; } + + @Override + public String toString() { + return "Result [id=" + id + ", distance=" + distance + "]"; + } } private void doRandomTest(int numDocs, int numQueries) throws IOException { @@ -182,6 +186,17 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]); assertEquals(expected[resultNumber], actual); } + + // get page2 with searchAfter() + if (topN < reader.maxDoc()) { + int page2 = TestUtil.nextInt(random(), 1, reader.maxDoc() - topN); + TopDocs topDocs2 = searcher.searchAfter(topDocs.scoreDocs[topN - 1], new MatchAllDocsQuery(), page2, sort); + for (int resultNumber = 0; resultNumber < page2; resultNumber++) { + FieldDoc fieldDoc = (FieldDoc) topDocs2.scoreDocs[resultNumber]; + Result actual = new Result((Integer) fieldDoc.fields[1], (Double) fieldDoc.fields[0]); + assertEquals(expected[topN + resultNumber], actual); + } + } } reader.close(); writer.close(); From a6fe1c0b155ff464d0596289bb209eef99c269d3 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 14 Mar 2016 14:08:25 -0400 Subject: [PATCH 0131/1113] LUCENE-7102: LatLonPoint.newDistanceSort fails with "sort missing first" --- .../apache/lucene/document/LatLonPoint.java | 4 +- .../LatLonPointDistanceComparator.java | 84 +++++++------ .../lucene/document/LatLonPointSortField.java | 6 +- .../document/TestLatLonPointDistanceSort.java | 111 ++++++++++++++++-- 4 files changed, 152 insertions(+), 53 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index ebf850c9d5a0..9677baa3a9e7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -333,8 +333,8 @@ public static Query newPolygonQuery(String field, double[] polyLats, double[] po * the hits contains a Double instance with the distance in meters. *

    * If a document is missing the field, then by default it is treated as having {@link Double#POSITIVE_INFINITY} distance - * (missing last). You can change this by calling {@link SortField#setMissingValue(Object)} on the returned SortField - * to a different Double value. + * (missing values sort last). You can change this to sort missing values first by calling + * {@link SortField#setMissingValue(Object) setMissingValue(Double.NEGATIVE_INFINITY)} on the returned SortField. *

    * If a document contains multiple values for the field, the closest distance to the location is used. *

    diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index e64f4b0eeace..86c9134fd7a9 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -89,45 +89,53 @@ public void setBottom(int slot) { // sampling if we get called way too much: don't make gobs of bounding // boxes if comparator hits a worst case order (e.g. backwards distance order) if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { - GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom); - // pre-encode our box to our integer encoding, so we don't have to decode - // to double values for uncompetitive hits. This has some cost! - int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat); - int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat); - int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon); - int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon); - // be sure to not introduce quantization error in our optimization, just - // round up our encoded box safely in all directions. - if (minLatEncoded != Integer.MIN_VALUE) { - minLatEncoded--; - } - if (minLonEncoded != Integer.MIN_VALUE) { - minLonEncoded--; - } - if (maxLatEncoded != Integer.MAX_VALUE) { - maxLatEncoded++; - } - if (maxLonEncoded != Integer.MAX_VALUE) { - maxLonEncoded++; - } - crossesDateLine = box.crossesDateline(); - // crosses dateline: split - if (crossesDateLine) { - // box1 - minLon = Integer.MIN_VALUE; - maxLon = maxLonEncoded; - minLat = minLatEncoded; - maxLat = maxLatEncoded; - // box2 - minLon2 = minLonEncoded; - maxLon2 = Integer.MAX_VALUE; - minLat2 = minLatEncoded; - maxLat2 = maxLatEncoded; + // don't pass infinite values to circleToBBox: just make a complete box. + if (bottom == missingValue) { + minLat = minLon = Integer.MIN_VALUE; + maxLat = maxLon = Integer.MAX_VALUE; + crossesDateLine = false; } else { - minLon = minLonEncoded; - maxLon = maxLonEncoded; - minLat = minLatEncoded; - maxLat = maxLatEncoded; + assert Double.isFinite(bottom); + GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom); + // pre-encode our box to our integer encoding, so we don't have to decode + // to double values for uncompetitive hits. This has some cost! + int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat); + int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat); + int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon); + int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon); + // be sure to not introduce quantization error in our optimization, just + // round up our encoded box safely in all directions. + if (minLatEncoded != Integer.MIN_VALUE) { + minLatEncoded--; + } + if (minLonEncoded != Integer.MIN_VALUE) { + minLonEncoded--; + } + if (maxLatEncoded != Integer.MAX_VALUE) { + maxLatEncoded++; + } + if (maxLonEncoded != Integer.MAX_VALUE) { + maxLonEncoded++; + } + crossesDateLine = box.crossesDateline(); + // crosses dateline: split + if (crossesDateLine) { + // box1 + minLon = Integer.MIN_VALUE; + maxLon = maxLonEncoded; + minLat = minLatEncoded; + maxLat = maxLatEncoded; + // box2 + minLon2 = minLonEncoded; + maxLon2 = Integer.MAX_VALUE; + minLat2 = minLatEncoded; + maxLat2 = maxLatEncoded; + } else { + minLon = minLonEncoded; + maxLon = maxLonEncoded; + minLat = minLatEncoded; + maxLat = maxLatEncoded; + } } } setBottomCounter++; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java index 9d3928ea8c37..da90b8674657 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java @@ -62,7 +62,11 @@ public void setMissingValue(Object missingValue) { } if (missingValue.getClass() != Double.class) throw new IllegalArgumentException("Missing value can only be of type java.lang.Double, but got " + missingValue.getClass()); - this.missingValue = missingValue; + Double value = (Double) missingValue; + if (!Double.isInfinite(value)) { + throw new IllegalArgumentException("Missing value can only be Double.NEGATIVE_INFINITY (missing values first) or Double.POSITIVE_INFINITY (missing values last), but got " + value); + } + this.missingValue = value; } @Override diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index ea36ea641b18..a776b3fc1552 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -73,6 +73,82 @@ public void testDistanceSort() throws Exception { dir.close(); } + /** Add two points (one doc missing) and sort by distance */ + public void testMissingLast() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + + // missing + Document doc = new Document(); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.718266, -74.007819)); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.7051157, -74.0088305)); + iw.addDocument(doc); + + IndexReader reader = iw.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + iw.close(); + + Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731)); + TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); + + FieldDoc d = (FieldDoc) td.scoreDocs[0]; + assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[1]; + assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[2]; + assertEquals(Double.POSITIVE_INFINITY, (Double)d.fields[0], 0.0D); + + reader.close(); + dir.close(); + } + + /** Add two points (one doc missing) and sort by distance */ + public void testMissingFirst() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + + // missing + Document doc = new Document(); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.718266, -74.007819)); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new LatLonPoint("location", 40.7051157, -74.0088305)); + iw.addDocument(doc); + + IndexReader reader = iw.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + iw.close(); + + SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731); + sortField.setMissingValue(Double.NEGATIVE_INFINITY); + Sort sort = new Sort(sortField); + TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); + + FieldDoc d = (FieldDoc) td.scoreDocs[0]; + assertEquals(Double.NEGATIVE_INFINITY, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[1]; + assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); + + d = (FieldDoc) td.scoreDocs[2]; + assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); + + reader.close(); + dir.close(); + } + /** Run a few iterations with just 10 docs, hopefully easy to debug */ public void testRandom() throws Exception { for (int iters = 0; iters < 100; iters++) { @@ -141,17 +217,20 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { RandomIndexWriter writer = new RandomIndexWriter(random(), dir); for (int i = 0; i < numDocs; i++) { - double latRaw = -90 + 180.0 * random().nextDouble(); - double lonRaw = -180 + 360.0 * random().nextDouble(); - // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons - double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); - double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); Document doc = new Document(); doc.add(new StoredField("id", i)); doc.add(new NumericDocValuesField("id", i)); - doc.add(new LatLonPoint("field", lat, lon)); - doc.add(new StoredField("lat", lat)); - doc.add(new StoredField("lon", lon)); + if (random().nextInt(10) > 7) { + double latRaw = -90 + 180.0 * random().nextDouble(); + double lonRaw = -180 + 360.0 * random().nextDouble(); + // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons + double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); + double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); + + doc.add(new LatLonPoint("field", lat, lon)); + doc.add(new StoredField("lat", lat)); + doc.add(new StoredField("lon", lon)); + } // otherwise "missing" writer.addDocument(doc); } IndexReader reader = writer.getReader(); @@ -160,14 +239,20 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { for (int i = 0; i < numQueries; i++) { double lat = -90 + 180.0 * random().nextDouble(); double lon = -180 + 360.0 * random().nextDouble(); + double missingValue = random().nextBoolean() ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; Result expected[] = new Result[reader.maxDoc()]; for (int doc = 0; doc < reader.maxDoc(); doc++) { Document targetDoc = reader.document(doc); - double docLatitude = targetDoc.getField("lat").numericValue().doubleValue(); - double docLongitude = targetDoc.getField("lon").numericValue().doubleValue(); - double distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + final double distance; + if (targetDoc.getField("lat") == null) { + distance = missingValue; // missing + } else { + double docLatitude = targetDoc.getField("lat").numericValue().doubleValue(); + double docLongitude = targetDoc.getField("lon").numericValue().doubleValue(); + distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + } int id = targetDoc.getField("id").numericValue().intValue(); expected[doc] = new Result(id, distance); } @@ -177,7 +262,9 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { // randomize the topN a bit int topN = TestUtil.nextInt(random(), 1, reader.maxDoc()); // sort by distance, then ID - Sort sort = new Sort(LatLonPoint.newDistanceSort("field", lat, lon), + SortField distanceSort = LatLonPoint.newDistanceSort("field", lat, lon); + distanceSort.setMissingValue(missingValue); + Sort sort = new Sort(distanceSort, new SortField("id", SortField.Type.INT)); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), topN, sort); From 3163b98959ae834b27a0ee8479344b61460d8851 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 14 Mar 2016 16:25:31 -0400 Subject: [PATCH 0132/1113] LUCENE-7103: further optimize LatLonPoint.newDistanceSort --- .../LatLonPointDistanceComparator.java | 41 +++++++++++++++---- .../lucene/document/TestBigIntegerPoint.java | 4 +- .../lucene/document/TestInetAddressPoint.java | 4 +- .../lucene/document/TestLatLonPoint.java | 2 +- .../TestLatLonPointDistanceQuery.java | 4 +- .../document/TestLatLonPointDistanceSort.java | 28 +++++++++++-- 6 files changed, 64 insertions(+), 19 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index 86c9134fd7a9..ef4c3f39d280 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -26,9 +26,10 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; -import org.apache.lucene.spatial.util.GeoDistanceUtils; +import org.apache.lucene.spatial.util.GeoProjectionUtils; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.util.SloppyMath; /** * Compares documents by distance from an origin point @@ -96,7 +97,7 @@ public void setBottom(int slot) { crossesDateLine = false; } else { assert Double.isFinite(bottom); - GeoRect box = GeoUtils.circleToBBox(longitude, latitude, bottom); + GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom)); // pre-encode our box to our integer encoding, so we don't have to decode // to double values for uncompetitive hits. This has some cost! int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat); @@ -166,7 +167,7 @@ public int compareBottom(int doc) throws IOException { if (outsideBox == false) { double docLatitude = LatLonPoint.decodeLatitude(latitudeBits); double docLongitude = LatLonPoint.decodeLongitude(longitudeBits); - minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude)); + minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude)); } } return Double.compare(bottom, minValue); @@ -174,7 +175,7 @@ public int compareBottom(int doc) throws IOException { @Override public void copy(int slot, int doc) throws IOException { - values[slot] = distance(doc); + values[slot] = sortKey(doc); } @Override @@ -190,17 +191,17 @@ public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws I @Override public Double value(int slot) { - return Double.valueOf(values[slot]); + return Double.valueOf(haversin2(values[slot])); } @Override public int compareTop(int doc) throws IOException { - return Double.compare(topValue, distance(doc)); + return Double.compare(topValue, haversin2(sortKey(doc))); } // TODO: optimize for single-valued case? // TODO: do all kinds of other optimizations! - double distance(int doc) { + double sortKey(int doc) { currentDocs.setDocument(doc); int numValues = currentDocs.count(); @@ -213,8 +214,32 @@ public int compareTop(int doc) throws IOException { long encoded = currentDocs.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - minValue = Math.min(minValue, GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude)); + minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude)); } return minValue; } + + // sort by first part of the haversin computation. note that this value is meaningless to the user. + // invoke haversin2() to "complete" the calculation and get a distance in meters. + static double haversin1(double lat1, double lon1, double lat2, double lon2) { + double dLat = SloppyMath.TO_RADIANS * (lat2 - lat1); + double dLon = SloppyMath.TO_RADIANS * (lon2 - lon1); + lat1 = SloppyMath.TO_RADIANS * (lat1); + lat2 = SloppyMath.TO_RADIANS * (lat2); + + final double sinDLatO2 = SloppyMath.sin(dLat / 2); + final double sinDLonO2 = SloppyMath.sin(dLon / 2); + + return sinDLatO2*sinDLatO2 + sinDLonO2 * sinDLonO2 * SloppyMath.cos(lat1) * SloppyMath.cos(lat2); + } + + // second half of the haversin calculation, used to convert results from haversin1 (used internally + // for sorting) for display purposes. + static double haversin2(double partial) { + if (Double.isInfinite(partial)) { + return partial; + } + double c = 2 * SloppyMath.asin(Math.sqrt(partial)); + return (GeoProjectionUtils.SEMIMAJOR_AXIS * c); + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index 8f38bcd1aa46..a7a129551dbf 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -41,7 +41,7 @@ public void testBasics() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", large))); assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", large.subtract(BigInteger.ONE), large.add(BigInteger.ONE)))); assertEquals(1, searcher.count(BigIntegerPoint.newSetQuery("field", large))); @@ -66,7 +66,7 @@ public void testNegative() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(BigIntegerPoint.newExactQuery("field", negative))); assertEquals(1, searcher.count(BigIntegerPoint.newRangeQuery("field", negative.subtract(BigInteger.ONE), negative.add(BigInteger.ONE)))); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index c91b52b252fb..b0e7107fff1d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -41,7 +41,7 @@ public void testBasics() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 24))); assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")))); @@ -68,7 +68,7 @@ public void testBasicsV6() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(InetAddressPoint.newExactQuery("field", address))); assertEquals(1, searcher.count(InetAddressPoint.newPrefixQuery("field", address, 64))); assertEquals(1, searcher.count(InetAddressPoint.newRangeQuery("field", InetAddress.getByName("fec0::f66c"), InetAddress.getByName("fec0::f66e")))); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index d180d58f5671..ff4af122234f 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -39,7 +39,7 @@ public void testBoxQuery() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(LatLonPoint.newBoxQuery("field", 18, 19, -66, -65))); reader.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java index fa957103d9f0..c69791ca55d5 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java @@ -55,7 +55,7 @@ public void testBasics() throws Exception { // search within 50km and verify we found our doc IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader, false); + IndexSearcher searcher = newSearcher(reader); assertEquals(1, searcher.count(LatLonPoint.newDistanceQuery("field", 18, -65, 50_000))); reader.close(); @@ -148,7 +148,7 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException writer.addDocument(doc); } IndexReader reader = writer.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); + IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { double lat = -90 + 180.0 * random().nextDouble(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index a776b3fc1552..7df956f94bfa 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -54,7 +54,7 @@ public void testDistanceSort() throws Exception { iw.addDocument(doc); IndexReader reader = iw.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); + IndexSearcher searcher = newSearcher(reader); iw.close(); Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731)); @@ -91,7 +91,7 @@ public void testMissingLast() throws Exception { iw.addDocument(doc); IndexReader reader = iw.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); + IndexSearcher searcher = newSearcher(reader); iw.close(); Sort sort = new Sort(LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731)); @@ -128,7 +128,7 @@ public void testMissingFirst() throws Exception { iw.addDocument(doc); IndexReader reader = iw.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); + IndexSearcher searcher = newSearcher(reader); iw.close(); SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731); @@ -234,7 +234,7 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { writer.addDocument(doc); } IndexReader reader = writer.getReader(); - IndexSearcher searcher = new IndexSearcher(reader); + IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { double lat = -90 + 180.0 * random().nextDouble(); @@ -289,4 +289,24 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { writer.close(); dir.close(); } + + /** Test this method sorts the same way as real haversin */ + public void testPartialHaversin() { + for (int i = 0; i < 100000; i++) { + double centerLat = -90 + 180.0 * random().nextDouble(); + double centerLon = -180 + 360.0 * random().nextDouble(); + + double lat1 = -90 + 180.0 * random().nextDouble(); + double lon1 = -180 + 360.0 * random().nextDouble(); + + double lat2 = -90 + 180.0 * random().nextDouble(); + double lon2 = -180 + 360.0 * random().nextDouble(); + + int expected = Integer.signum(Double.compare(GeoDistanceUtils.haversin(centerLat, centerLon, lat1, lon1), + GeoDistanceUtils.haversin(centerLat, centerLon, lat2, lon2))); + int actual = Integer.signum(Double.compare(LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat1, lon1), + LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat2, lon2))); + assertEquals(expected, actual); + } + } } From bd87ca0c4d0922de446a9de1d6b86c0ab65ec994 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 14 Mar 2016 19:07:30 -0400 Subject: [PATCH 0133/1113] LUCENE-7104: remove "sort missing first" from LatLonPoint.newDistanceSort and simplify/speedup code --- .../apache/lucene/document/LatLonPoint.java | 3 +- .../LatLonPointDistanceComparator.java | 116 +++++++----------- .../document/LatLonPointDistanceQuery.java | 8 +- .../lucene/document/LatLonPointSortField.java | 14 +-- .../document/TestLatLonPointDistanceSort.java | 57 +++------ 5 files changed, 69 insertions(+), 129 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 9677baa3a9e7..f5541bd9dcf4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -333,8 +333,7 @@ public static Query newPolygonQuery(String field, double[] polyLats, double[] po * the hits contains a Double instance with the distance in meters. *

    * If a document is missing the field, then by default it is treated as having {@link Double#POSITIVE_INFINITY} distance - * (missing values sort last). You can change this to sort missing values first by calling - * {@link SortField#setMissingValue(Object) setMissingValue(Double.NEGATIVE_INFINITY)} on the returned SortField. + * (missing values sort last). *

    * If a document contains multiple values for the field, the closest distance to the location is used. *

    diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index ef4c3f39d280..2102d81d4966 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -42,7 +42,6 @@ class LatLonPointDistanceComparator extends FieldComparator implements L final String field; final double latitude; final double longitude; - final double missingValue; final double[] values; double bottom; @@ -52,27 +51,22 @@ class LatLonPointDistanceComparator extends FieldComparator implements L // current bounding box(es) for the bottom distance on the PQ. // these are pre-encoded with LatLonPoint's encoding and // used to exclude uncompetitive hits faster. - int minLon; - int maxLon; - int minLat; - int maxLat; - - // crossesDateLine is true, then we have a second box to check - boolean crossesDateLine; - int minLon2; - int maxLon2; - int minLat2; - int maxLat2; + int minLon = Integer.MIN_VALUE; + int maxLon = Integer.MAX_VALUE; + int minLat = Integer.MIN_VALUE; + int maxLat = Integer.MAX_VALUE; + + // second set of longitude ranges to check (for cross-dateline case) + int minLon2 = Integer.MAX_VALUE; // the number of times setBottom has been called (adversary protection) int setBottomCounter = 0; - public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits, double missingValue) { + public LatLonPointDistanceComparator(String field, double latitude, double longitude, int numHits) { this.field = field; this.latitude = latitude; this.longitude = longitude; this.values = new double[numHits]; - this.missingValue = missingValue; } @Override @@ -90,53 +84,22 @@ public void setBottom(int slot) { // sampling if we get called way too much: don't make gobs of bounding // boxes if comparator hits a worst case order (e.g. backwards distance order) if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { - // don't pass infinite values to circleToBBox: just make a complete box. - if (bottom == missingValue) { - minLat = minLon = Integer.MIN_VALUE; - maxLat = maxLon = Integer.MAX_VALUE; - crossesDateLine = false; + GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom)); + // pre-encode our box to our integer encoding, so we don't have to decode + // to double values for uncompetitive hits. This has some cost! + minLat = LatLonPoint.encodeLatitude(box.minLat); + maxLat = LatLonPoint.encodeLatitude(box.maxLat); + if (box.crossesDateline()) { + // box1 + minLon = Integer.MIN_VALUE; + maxLon = LatLonPoint.encodeLongitude(box.maxLon); + // box2 + minLon2 = LatLonPoint.encodeLongitude(box.minLon); } else { - assert Double.isFinite(bottom); - GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom)); - // pre-encode our box to our integer encoding, so we don't have to decode - // to double values for uncompetitive hits. This has some cost! - int minLatEncoded = LatLonPoint.encodeLatitude(box.minLat); - int maxLatEncoded = LatLonPoint.encodeLatitude(box.maxLat); - int minLonEncoded = LatLonPoint.encodeLongitude(box.minLon); - int maxLonEncoded = LatLonPoint.encodeLongitude(box.maxLon); - // be sure to not introduce quantization error in our optimization, just - // round up our encoded box safely in all directions. - if (minLatEncoded != Integer.MIN_VALUE) { - minLatEncoded--; - } - if (minLonEncoded != Integer.MIN_VALUE) { - minLonEncoded--; - } - if (maxLatEncoded != Integer.MAX_VALUE) { - maxLatEncoded++; - } - if (maxLonEncoded != Integer.MAX_VALUE) { - maxLonEncoded++; - } - crossesDateLine = box.crossesDateline(); - // crosses dateline: split - if (crossesDateLine) { - // box1 - minLon = Integer.MIN_VALUE; - maxLon = maxLonEncoded; - minLat = minLatEncoded; - maxLat = maxLatEncoded; - // box2 - minLon2 = minLonEncoded; - maxLon2 = Integer.MAX_VALUE; - minLat2 = minLatEncoded; - maxLat2 = maxLatEncoded; - } else { - minLon = minLonEncoded; - maxLon = maxLonEncoded; - minLat = minLatEncoded; - maxLat = maxLatEncoded; - } + minLon = LatLonPoint.encodeLongitude(box.minLon); + maxLon = LatLonPoint.encodeLongitude(box.maxLon); + // disable box2 + minLon2 = Integer.MAX_VALUE; } } setBottomCounter++; @@ -153,24 +116,33 @@ public int compareBottom(int doc) throws IOException { int numValues = currentDocs.count(); if (numValues == 0) { - return Double.compare(bottom, missingValue); + return Double.compare(bottom, Double.POSITIVE_INFINITY); } - double minValue = Double.POSITIVE_INFINITY; + int cmp = -1; for (int i = 0; i < numValues; i++) { long encoded = currentDocs.valueAt(i); + + // test bounding box int latitudeBits = (int)(encoded >> 32); + if (latitudeBits < minLat || latitudeBits > maxLat) { + continue; + } int longitudeBits = (int)(encoded & 0xFFFFFFFF); - boolean outsideBox = ((latitudeBits < minLat || longitudeBits < minLon || latitudeBits > maxLat || longitudeBits > maxLon) && - (crossesDateLine == false || latitudeBits < minLat2 || longitudeBits < minLon2 || latitudeBits > maxLat2 || longitudeBits > maxLon2)); + if ((longitudeBits < minLon || longitudeBits > maxLon) && (longitudeBits < minLon2)) { + continue; + } + // only compute actual distance if its inside "competitive bounding box" - if (outsideBox == false) { - double docLatitude = LatLonPoint.decodeLatitude(latitudeBits); - double docLongitude = LatLonPoint.decodeLongitude(longitudeBits); - minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude)); + double docLatitude = LatLonPoint.decodeLatitude(latitudeBits); + double docLongitude = LatLonPoint.decodeLongitude(longitudeBits); + cmp = Math.max(cmp, Double.compare(bottom, haversin1(latitude, longitude, docLatitude, docLongitude))); + // once we compete in the PQ, no need to continue. + if (cmp > 0) { + return cmp; } } - return Double.compare(bottom, minValue); + return cmp; } @Override @@ -204,12 +176,8 @@ public int compareTop(int doc) throws IOException { double sortKey(int doc) { currentDocs.setDocument(doc); - int numValues = currentDocs.count(); - if (numValues == 0) { - return missingValue; - } - double minValue = Double.POSITIVE_INFINITY; + int numValues = currentDocs.count(); for (int i = 0; i < numValues; i++) { long encoded = currentDocs.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 9d2398653ef4..3f86f1e5a5d5 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -142,9 +142,11 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - if ((latMax < box1.minLat || lonMax < box1.minLon || latMin > box1.maxLat || lonMin > box1.maxLon) && - (box2 == null || latMax < box2.minLat || lonMax < box2.minLon || latMin > box2.maxLat || lonMin > box2.maxLon)) { - // we are fully outside of bounding box(es), don't proceed any further. + if (latMax < box1.minLat || latMin > box1.maxLat) { + // latitude out of bounding box range + return Relation.CELL_OUTSIDE_QUERY; + } else if ((lonMax < box1.minLon || lonMin > box1.maxLon) && (box2 == null || lonMax < box2.minLon)) { + // longitude out of bounding box range return Relation.CELL_OUTSIDE_QUERY; } else if (lonMax - longitude < 90 && longitude - lonMin < 90 && GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMin) <= radiusMeters && diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java index da90b8674657..f88304367e5f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java @@ -47,7 +47,7 @@ final class LatLonPointSortField extends SortField { @Override public FieldComparator getComparator(int numHits, int sortPos) throws IOException { - return new LatLonPointDistanceComparator(getField(), latitude, longitude, numHits, getMissingValue()); + return new LatLonPointDistanceComparator(getField(), latitude, longitude, numHits); } @Override @@ -57,16 +57,10 @@ public Double getMissingValue() { @Override public void setMissingValue(Object missingValue) { - if (missingValue == null) { - throw new IllegalArgumentException("Missing value cannot be null"); + if (Double.valueOf(Double.POSITIVE_INFINITY).equals(missingValue) == false) { + throw new IllegalArgumentException("Missing value can only be Double.POSITIVE_INFINITY (missing values last), but got " + missingValue); } - if (missingValue.getClass() != Double.class) - throw new IllegalArgumentException("Missing value can only be of type java.lang.Double, but got " + missingValue.getClass()); - Double value = (Double) missingValue; - if (!Double.isInfinite(value)) { - throw new IllegalArgumentException("Missing value can only be Double.NEGATIVE_INFINITY (missing values first) or Double.POSITIVE_INFINITY (missing values last), but got " + value); - } - this.missingValue = value; + this.missingValue = missingValue; } @Override diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 7df956f94bfa..437631310ad3 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -28,6 +28,8 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.spatial.util.GeoDistanceUtils; +import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -110,45 +112,6 @@ public void testMissingLast() throws Exception { dir.close(); } - /** Add two points (one doc missing) and sort by distance */ - public void testMissingFirst() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - - // missing - Document doc = new Document(); - iw.addDocument(doc); - - doc = new Document(); - doc.add(new LatLonPoint("location", 40.718266, -74.007819)); - iw.addDocument(doc); - - doc = new Document(); - doc.add(new LatLonPoint("location", 40.7051157, -74.0088305)); - iw.addDocument(doc); - - IndexReader reader = iw.getReader(); - IndexSearcher searcher = newSearcher(reader); - iw.close(); - - SortField sortField = LatLonPoint.newDistanceSort("location", 40.7143528, -74.0059731); - sortField.setMissingValue(Double.NEGATIVE_INFINITY); - Sort sort = new Sort(sortField); - TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); - - FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(Double.NEGATIVE_INFINITY, (Double)d.fields[0], 0.0D); - - d = (FieldDoc) td.scoreDocs[1]; - assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); - - d = (FieldDoc) td.scoreDocs[2]; - assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); - - reader.close(); - dir.close(); - } - /** Run a few iterations with just 10 docs, hopefully easy to debug */ public void testRandom() throws Exception { for (int iters = 0; iters < 100; iters++) { @@ -239,7 +202,7 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { for (int i = 0; i < numQueries; i++) { double lat = -90 + 180.0 * random().nextDouble(); double lon = -180 + 360.0 * random().nextDouble(); - double missingValue = random().nextBoolean() ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + double missingValue = Double.POSITIVE_INFINITY; Result expected[] = new Result[reader.maxDoc()]; @@ -309,4 +272,18 @@ public void testPartialHaversin() { assertEquals(expected, actual); } } + + /** Test infinite radius covers whole earth */ + public void testInfiniteRect() { + for (int i = 0; i < 100000; i++) { + double centerLat = -90 + 180.0 * random().nextDouble(); + double centerLon = -180 + 360.0 * random().nextDouble(); + GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY); + assertEquals(-180.0, rect.minLon, 0.0D); + assertEquals(180.0, rect.maxLon, 0.0D); + assertEquals(-90.0, rect.minLat, 0.0D); + assertEquals(90.0, rect.maxLat, 0.0D); + assertFalse(rect.crossesDateline()); + } + } } From 13ce0815a72abca783b1bed89f04b14071306d9e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 15 Mar 2016 09:49:17 +0100 Subject: [PATCH 0134/1113] Change entry to reflect that LUCENE-7087 has been backported to 6.0 --- lucene/CHANGES.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3618141cd5fb..96321b033e52 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -21,11 +21,6 @@ Optimizations * LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort stack depth before switching to heapsort (Adrien Grand, Mike McCandless) -Other - -* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' - as document instead of 'Document'. (Martijn van Groningen) - ======================= Lucene 6.0.0 ======================= System Requirements @@ -198,6 +193,9 @@ Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) +* LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' + as document instead of 'Document'. (Martijn van Groningen) + ======================= Lucene 5.5.0 ======================= New Features From 559432fcdcd828864a800072cf144e1f5c96647e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 9 Mar 2016 17:38:23 +0100 Subject: [PATCH 0135/1113] LUCENE-7091: Added doc values support to memory index --- lucene/CHANGES.txt | 3 + .../lucene/index/memory/MemoryIndex.java | 465 ++++++++++++++---- .../lucene/index/memory/TestMemoryIndex.java | 133 +++++ .../memory/TestMemoryIndexAgainstRAMDir.java | 134 +++++ 4 files changed, 636 insertions(+), 99 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 96321b033e52..0897c1a2f18a 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -196,6 +196,9 @@ Other * LUCENE-7087: Let MemoryIndex#fromDocument(...) accept 'Iterable' as document instead of 'Document'. (Martijn van Groningen) +* LUCENE-7091: Add doc values support to MemoryIndex + (Martijn van Groningen, David Smiley) + ======================= Lucene 5.5.0 ======================= New Features diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 9e01182ec902..40159aa1f972 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -17,12 +17,15 @@ package org.apache.lucene.index.memory; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.Locale; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import java.util.stream.Collectors; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -255,7 +258,7 @@ public void addField(String fieldName, String text, Analyzer analyzer) { throw new IllegalArgumentException("analyzer must not be null"); TokenStream stream = analyzer.tokenStream(fieldName, text); - addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName)); + addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName), DocValuesType.NONE, null); } /** @@ -351,7 +354,9 @@ public void addField(String fieldName, TokenStream stream) { } /** - * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer + * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer. + * Also stores doc values based on {@link IndexableFieldType#docValuesType()} if set. + * * @param field the field to add * @param analyzer the analyzer to use for term analysis * @throws IllegalArgumentException if the field is a DocValues or Point field, as these @@ -362,7 +367,9 @@ public void addField(IndexableField field, Analyzer analyzer) { } /** - * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer + * Adds a lucene {@link IndexableField} to the MemoryIndex using the provided analyzer. + * Also stores doc values based on {@link IndexableFieldType#docValuesType()} if set. + * * @param field the field to add * @param analyzer the analyzer to use for term analysis * @param boost a field boost @@ -370,17 +377,42 @@ public void addField(IndexableField field, Analyzer analyzer) { * structures are not supported by MemoryIndex */ public void addField(IndexableField field, Analyzer analyzer, float boost) { - if (field.fieldType().docValuesType() != DocValuesType.NONE) - throw new IllegalArgumentException("MemoryIndex does not support DocValues fields"); - if (field.fieldType().pointDimensionCount() != 0) + if (field.fieldType().pointDimensionCount() != 0) { throw new IllegalArgumentException("MemoryIndex does not support Points"); - if (analyzer == null) { - addField(field.name(), field.tokenStream(null, null), boost); } - else { - addField(field.name(), field.tokenStream(analyzer, null), boost, - analyzer.getPositionIncrementGap(field.name()), analyzer.getOffsetGap(field.name())); + + int offsetGap; + TokenStream tokenStream; + int positionIncrementGap; + if (analyzer != null) { + offsetGap = analyzer.getOffsetGap(field.name()); + tokenStream = field.tokenStream(analyzer, null); + positionIncrementGap = analyzer.getPositionIncrementGap(field.name()); + } else { + offsetGap = 1; + tokenStream = field.tokenStream(null, null); + positionIncrementGap = 0; + } + + DocValuesType docValuesType = field.fieldType().docValuesType(); + Object docValuesValue; + switch (docValuesType) { + case NONE: + docValuesValue = null; + break; + case BINARY: + case SORTED: + case SORTED_SET: + docValuesValue = field.binaryValue(); + break; + case NUMERIC: + case SORTED_NUMERIC: + docValuesValue = field.numericValue(); + break; + default: + throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]"); } + addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue); } /** @@ -448,66 +480,126 @@ public void addField(String fieldName, TokenStream stream, float boost, int posi * the offset gap if fields with the same name are added more than once * @see org.apache.lucene.document.Field#setBoost(float) */ - public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, - int offsetGap) { - try (TokenStream stream = tokenStream) { - if (frozen) - throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen"); - if (fieldName == null) - throw new IllegalArgumentException("fieldName must not be null"); - if (stream == null) - throw new IllegalArgumentException("token stream must not be null"); - if (boost <= 0.0f) - throw new IllegalArgumentException("boost factor must be greater than 0.0"); - int numTokens = 0; - int numOverlapTokens = 0; - int pos = -1; - final BytesRefHash terms; - final SliceByteStartArray sliceArray; - Info info; - long sumTotalTermFreq = 0; - int offset = 0; - FieldInfo fieldInfo; - if ((info = fields.get(fieldName)) != null) { - fieldInfo = info.fieldInfo; - numTokens = info.numTokens; - numOverlapTokens = info.numOverlapTokens; - pos = info.lastPosition + positionIncrementGap; - offset = info.lastOffset + offsetGap; - terms = info.terms; - boost *= info.boost; - sliceArray = info.sliceArray; - sumTotalTermFreq = info.sumTotalTermFreq; - } else { - fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, this.storePayloads, - this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, - DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0); - sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY); - terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray); - } + public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) { + addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null); + } + + private void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap, + DocValuesType docValuesType, Object docValuesValue) { + + if (frozen) { + throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen"); + } + if (fieldName == null) { + throw new IllegalArgumentException("fieldName must not be null"); + } + if (boost <= 0.0f) { + throw new IllegalArgumentException("boost factor must be greater than 0.0"); + } + + Info info = fields.get(fieldName); + if (info == null) { + IndexOptions indexOptions = storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + FieldInfo fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, storePayloads, indexOptions, docValuesType, -1, Collections.emptyMap(), 0, 0); + fields.put(fieldName, info = new Info(fieldInfo, byteBlockPool)); + } + + if (docValuesType != DocValuesType.NONE) { + storeDocValues(info, docValuesType, docValuesValue); + } + if (tokenStream != null) { + storeTerms(info, tokenStream, boost, positionIncrementGap, offsetGap); + } + } + private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) { + String fieldName = info.fieldInfo.name; + DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType(); + if (existingDocValuesType == DocValuesType.NONE) { + // first time we add doc values for this field: + info.fieldInfo = new FieldInfo( + info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(), + info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(), + info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointNumBytes() + ); + } else if (existingDocValuesType != docValuesType) { + throw new IllegalArgumentException("Can't add [" + docValuesType + "] doc values field [" + fieldName + "], because [" + existingDocValuesType + "] doc values field already exists"); + } + switch (docValuesType) { + case NUMERIC: + if (info.numericProducer.dvLongValues != null) { + throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]"); + } + info.numericProducer.dvLongValues = new long[]{(long) docValuesValue}; + info.numericProducer.count++; + break; + case SORTED_NUMERIC: + if (info.numericProducer.dvLongValues == null) { + info.numericProducer.dvLongValues = new long[4]; + } + info.numericProducer.dvLongValues = ArrayUtil.grow(info.numericProducer.dvLongValues, info.numericProducer.count + 1); + info.numericProducer.dvLongValues[info.numericProducer.count++] = (long) docValuesValue; + break; + case BINARY: + if (info.binaryProducer.dvBytesValuesSet != null) { + throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]"); + } + info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool); + info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue); + break; + case SORTED: + if (info.binaryProducer.dvBytesValuesSet != null) { + throw new IllegalArgumentException("Only one value per field allowed for [" + docValuesType + "] doc values field [" + fieldName + "]"); + } + info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool); + info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue); + break; + case SORTED_SET: + if (info.binaryProducer.dvBytesValuesSet == null) { + info.binaryProducer.dvBytesValuesSet = new BytesRefHash(byteBlockPool); + } + info.binaryProducer.dvBytesValuesSet.add((BytesRef) docValuesValue); + break; + default: + throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]"); + } + } + + private void storeTerms(Info info, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) { + int pos = -1; + int offset = 0; + if (info.numTokens == 0) { + info.boost = boost; + } else if (info.numTokens > 0) { + pos = info.lastPosition + positionIncrementGap; + offset = info.lastOffset + offsetGap; + info.boost *= boost; + } + + try (TokenStream stream = tokenStream) { TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); PayloadAttribute payloadAtt = storePayloads ? stream.addAttribute(PayloadAttribute.class) : null; stream.reset(); - + while (stream.incrementToken()) { // if (DEBUG) System.err.println("token='" + term + "'"); - numTokens++; + info.numTokens++; final int posIncr = posIncrAttribute.getPositionIncrement(); - if (posIncr == 0) - numOverlapTokens++; + if (posIncr == 0) { + info.numOverlapTokens++; + } pos += posIncr; - int ord = terms.add(termAtt.getBytesRef()); + int ord = info.terms.add(termAtt.getBytesRef()); if (ord < 0) { ord = (-ord) - 1; - postingsWriter.reset(sliceArray.end[ord]); + postingsWriter.reset(info.sliceArray.end[ord]); } else { - sliceArray.start[ord] = postingsWriter.startNewSlice(); + info.sliceArray.start[ord] = postingsWriter.startNewSlice(); } - sliceArray.freq[ord]++; - sumTotalTermFreq++; + info.sliceArray.freq[ord]++; + info.sumTotalTermFreq++; postingsWriter.writeInt(pos); if (storeOffsets) { postingsWriter.writeInt(offsetAtt.startOffset() + offset); @@ -523,13 +615,12 @@ public void addField(String fieldName, TokenStream tokenStream, float boost, int } postingsWriter.writeInt(pIndex); } - sliceArray.end[ord] = postingsWriter.getCurrentOffset(); + info.sliceArray.end[ord] = postingsWriter.getCurrentOffset(); } stream.end(); - - // ensure infos.numTokens > 0 invariant; needed for correct operation of terms() - if (numTokens > 0) { - fields.put(fieldName, new Info(fieldInfo, terms, sliceArray, numTokens, numOverlapTokens, boost, pos, offsetAtt.endOffset() + offset, sumTotalTermFreq)); + if (info.numTokens > 0) { + info.lastPosition = pos; + info.lastOffset = offsetAtt.endOffset() + offset; } } catch (IOException e) { throw new RuntimeException(e); @@ -573,8 +664,7 @@ public IndexSearcher createSearcher() { public void freeze() { this.frozen = true; for (Info info : fields.values()) { - info.sortTerms(); - info.getNormDocValues();//lazily computed + info.freeze(); } } @@ -702,7 +792,7 @@ public String toString() { */ private final class Info { - private final FieldInfo fieldInfo; + private FieldInfo fieldInfo; /** The norms for this field; computed on demand. */ private transient NumericDocValues norms; @@ -711,40 +801,48 @@ private final class Info { * Term strings and their positions for this field: Map <String * termText, ArrayIntList positions> */ - private final BytesRefHash terms; // note unfortunate variable name class with Terms type + private BytesRefHash terms; // note unfortunate variable name class with Terms type - private final SliceByteStartArray sliceArray; + private SliceByteStartArray sliceArray; /** Terms sorted ascending by term text; computed on demand */ private transient int[] sortedTerms; /** Number of added tokens for this field */ - private final int numTokens; + private int numTokens; /** Number of overlapping tokens for this field */ - private final int numOverlapTokens; + private int numOverlapTokens; /** Boost factor for hits for this field */ - private final float boost; + private float boost; - private final long sumTotalTermFreq; + private long sumTotalTermFreq; /** the last position encountered in this field for multi field support*/ - private final int lastPosition; + private int lastPosition; /** the last offset encountered in this field for multi field support*/ - private final int lastOffset; + private int lastOffset; - public Info(FieldInfo fieldInfo, BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, int lastOffset, long sumTotalTermFreq) { + private BinaryDocValuesProducer binaryProducer; + + private NumericDocValuesProducer numericProducer; + + private boolean preparedDocValues; + + private Info(FieldInfo fieldInfo, ByteBlockPool byteBlockPool) { this.fieldInfo = fieldInfo; - this.terms = terms; - this.sliceArray = sliceArray; - this.numTokens = numTokens; - this.numOverlapTokens = numOverlapTokens; - this.boost = boost; - this.sumTotalTermFreq = sumTotalTermFreq; - this.lastPosition = lastPosition; - this.lastOffset = lastOffset; + this.sliceArray = new SliceByteStartArray(BytesRefHash.DEFAULT_CAPACITY); + this.terms = new BytesRefHash(byteBlockPool, BytesRefHash.DEFAULT_CAPACITY, sliceArray);; + this.binaryProducer = new BinaryDocValuesProducer(); + this.numericProducer = new NumericDocValuesProducer(); + } + + void freeze() { + sortTerms(); + prepareDocValues(); + getNormDocValues(); } /** @@ -755,13 +853,26 @@ public Info(FieldInfo fieldInfo, BytesRefHash terms, SliceByteStartArray sliceAr * (which would be an alternative and somewhat more elegant approach, * apart from more sophisticated Tries / prefix trees). */ - public void sortTerms() { + void sortTerms() { if (sortedTerms == null) { sortedTerms = terms.sort(); } } - public NumericDocValues getNormDocValues() { + void prepareDocValues() { + if (preparedDocValues == false) { + DocValuesType dvType = fieldInfo.getDocValuesType(); + if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) { + numericProducer.prepareForUsage(); + } + if (dvType == DocValuesType.BINARY || dvType == DocValuesType.SORTED || dvType == DocValuesType.SORTED_SET) { + binaryProducer.prepareForUsage(); + } + preparedDocValues = true; + } + } + + NumericDocValues getNormDocValues() { if (norms == null) { FieldInvertState invertState = new FieldInvertState(fieldInfo.name, fieldInfo.number, numTokens, numOverlapTokens, 0, boost); @@ -786,7 +897,81 @@ public long get(int docID) { /////////////////////////////////////////////////////////////////////////////// // Nested classes: /////////////////////////////////////////////////////////////////////////////// - + + private static final class BinaryDocValuesProducer { + + BytesRefHash dvBytesValuesSet; + final SortedDocValues sortedDocValues; + final BytesRef spare = new BytesRef(); + + int[] bytesIds; + + private BinaryDocValuesProducer() { + sortedDocValues = new SortedDocValues() { + @Override + public int getOrd(int docID) { + return 0; + } + + @Override + public BytesRef lookupOrd(int ord) { + return getValue(ord); + } + + @Override + public int getValueCount() { + return 1; + } + }; + } + + private void prepareForUsage() { + bytesIds = dvBytesValuesSet.sort(); + } + + private BytesRef getValue(int index) { + return dvBytesValuesSet.get(bytesIds[index], spare); + } + + } + + private static final class NumericDocValuesProducer { + + long[] dvLongValues; + int count; + + final NumericDocValues numericDocValues; + final SortedNumericDocValues sortedNumericDocValues; + + private NumericDocValuesProducer() { + this.numericDocValues = new NumericDocValues() { + @Override + public long get(int docID) { + return dvLongValues[0]; + } + }; + this.sortedNumericDocValues = new SortedNumericDocValues() { + @Override + public void setDocument(int doc) { + } + + @Override + public long valueAt(int index) { + return dvLongValues[index]; + } + + @Override + public int count() { + return count; + } + }; + } + + private void prepareForUsage() { + Arrays.sort(dvLongValues, 0, count); + } + } + /** * Search support for Lucene framework integration; implements all methods * required by the Lucene IndexReader contracts. @@ -795,6 +980,9 @@ private final class MemoryIndexReader extends LeafReader { private MemoryIndexReader() { super(); // avoid as much superclass baggage as possible + for (Info info : fields.values()) { + info.prepareDocValues(); + } } @Override @@ -807,8 +995,18 @@ public void removeCoreClosedListener(CoreClosedListener listener) { removeCoreClosedListenerAsReaderClosedListener(this, listener); } - private Info getInfo(String fieldName) { - return fields.get(fieldName); + private Info getInfoForExpectedDocValuesType(String fieldName, DocValuesType expectedType) { + if (expectedType == DocValuesType.NONE) { + return null; + } + Info info = fields.get(fieldName); + if (info == null) { + return null; + } + if (info.fieldInfo.getDocValuesType() != expectedType) { + return null; + } + return info; } @Override @@ -828,32 +1026,87 @@ public FieldInfos getFieldInfos() { @Override public NumericDocValues getNumericDocValues(String field) { - return null; + Info info = getInfoForExpectedDocValuesType(field, DocValuesType.NUMERIC); + if (info != null) { + return info.numericProducer.numericDocValues; + } else { + return null; + } } @Override public BinaryDocValues getBinaryDocValues(String field) { - return null; + return getSortedDocValues(field, DocValuesType.BINARY); } @Override public SortedDocValues getSortedDocValues(String field) { - return null; + return getSortedDocValues(field, DocValuesType.SORTED); + } + + private SortedDocValues getSortedDocValues(String field, DocValuesType docValuesType) { + Info info = getInfoForExpectedDocValuesType(field, docValuesType); + if (info != null) { + return info.binaryProducer.sortedDocValues; + } else { + return null; + } } @Override public SortedNumericDocValues getSortedNumericDocValues(String field) { - return null; + Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_NUMERIC); + if (info != null) { + return info.numericProducer.sortedNumericDocValues; + } else { + return null; + } } @Override public SortedSetDocValues getSortedSetDocValues(String field) { - return null; + Info info = getInfoForExpectedDocValuesType(field, DocValuesType.SORTED_SET); + if (info != null) { + return new SortedSetDocValues() { + + int index = 0; + + @Override + public long nextOrd() { + if (index >= info.binaryProducer.dvBytesValuesSet.size()) { + return NO_MORE_ORDS; + } + return index++; + } + + @Override + public void setDocument(int docID) { + index = 0; + } + + @Override + public BytesRef lookupOrd(long ord) { + return info.binaryProducer.getValue((int) ord); + } + + @Override + public long getValueCount() { + return info.binaryProducer.dvBytesValuesSet.size(); + } + }; + } else { + return null; + } } @Override public Bits getDocsWithField(String field) throws IOException { - return null; + Info info = fields.get(field); + if (info != null && info.fieldInfo.getDocValuesType() != DocValuesType.NONE) { + return new Bits.MatchAllBits(1); + } else { + return null; + } } @Override @@ -866,7 +1119,25 @@ public void checkIntegrity() throws IOException { // no-op } + @Override + public Fields fields() { + Map filteredFields = fields.entrySet().stream() + .filter(entry -> entry.getValue().numTokens > 0) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, + (u,v) -> { throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", u));}, + TreeMap::new + )); + return new MemoryFields(filteredFields ); + } + private class MemoryFields extends Fields { + + private final Map fields; + + public MemoryFields(Map fields) { + this.fields = fields; + } + @Override public Iterator iterator() { return fields.keySet().iterator(); @@ -875,8 +1146,9 @@ public Iterator iterator() { @Override public Terms terms(final String field) { final Info info = fields.get(field); - if (info == null) + if (info == null) { return null; + } return new Terms() { @Override @@ -932,11 +1204,6 @@ public int size() { return fields.size(); } } - - @Override - public Fields fields() { - return new MemoryFields(); - } private class MemoryTermsEnum extends TermsEnum { private final Info info; diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index b150ea3e085d..7282e0eb50f6 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -21,13 +21,25 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockPayloadAnalyzer; +import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -35,6 +47,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.ClassicSimilarity; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.junit.Before; @@ -176,5 +189,125 @@ public void testBuildFromDocument() { } + public void testDocValues() throws Exception { + Document doc = new Document(); + doc.add(new NumericDocValuesField("numeric", 29L)); + doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L)); + doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L)); + doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L)); + doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L)); + doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L)); + doc.add(new BinaryDocValuesField("binary", new BytesRef("a"))); + doc.add(new SortedDocValuesField("sorted", new BytesRef("b"))); + doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f"))); + doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d"))); + doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d"))); + doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c"))); + + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); + LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); + NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric"); + assertEquals(29L, numericDocValues.get(0)); + SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric"); + sortedNumericDocValues.setDocument(0); + assertEquals(5, sortedNumericDocValues.count()); + assertEquals(30L, sortedNumericDocValues.valueAt(0)); + assertEquals(31L, sortedNumericDocValues.valueAt(1)); + assertEquals(32L, sortedNumericDocValues.valueAt(2)); + assertEquals(32L, sortedNumericDocValues.valueAt(3)); + assertEquals(33L, sortedNumericDocValues.valueAt(4)); + BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary"); + assertEquals("a", binaryDocValues.get(0).utf8ToString()); + SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted"); + assertEquals("b", sortedDocValues.get(0).utf8ToString()); + assertEquals(0, sortedDocValues.getOrd(0)); + assertEquals("b", sortedDocValues.lookupOrd(0).utf8ToString()); + SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set"); + assertEquals(3, sortedSetDocValues.getValueCount()); + sortedSetDocValues.setDocument(0); + assertEquals(0L, sortedSetDocValues.nextOrd()); + assertEquals(1L, sortedSetDocValues.nextOrd()); + assertEquals(2L, sortedSetDocValues.nextOrd()); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd()); + assertEquals("c", sortedSetDocValues.lookupOrd(0L).utf8ToString()); + assertEquals("d", sortedSetDocValues.lookupOrd(1L).utf8ToString()); + assertEquals("f", sortedSetDocValues.lookupOrd(2L).utf8ToString()); + } + + public void testInvalidDocValuesUsage() throws Exception { + Document doc = new Document(); + doc.add(new NumericDocValuesField("field", 29L)); + doc.add(new BinaryDocValuesField("field", new BytesRef("30"))); + try { + MemoryIndex.fromDocument(doc, analyzer); + } catch (IllegalArgumentException e) { + assertEquals("Can't add [BINARY] doc values field [field], because [NUMERIC] doc values field already exists", e.getMessage()); + } + + doc = new Document(); + doc.add(new NumericDocValuesField("field", 29L)); + doc.add(new NumericDocValuesField("field", 30L)); + try { + MemoryIndex.fromDocument(doc, analyzer); + } catch (IllegalArgumentException e) { + assertEquals("Only one value per field allowed for [NUMERIC] doc values field [field]", e.getMessage()); + } + + doc = new Document(); + doc.add(new TextField("field", "a b", Field.Store.NO)); + doc.add(new BinaryDocValuesField("field", new BytesRef("a"))); + doc.add(new BinaryDocValuesField("field", new BytesRef("b"))); + try { + MemoryIndex.fromDocument(doc, analyzer); + } catch (IllegalArgumentException e) { + assertEquals("Only one value per field allowed for [BINARY] doc values field [field]", e.getMessage()); + } + + doc = new Document(); + doc.add(new SortedDocValuesField("field", new BytesRef("a"))); + doc.add(new SortedDocValuesField("field", new BytesRef("b"))); + doc.add(new TextField("field", "a b", Field.Store.NO)); + try { + MemoryIndex.fromDocument(doc, analyzer); + } catch (IllegalArgumentException e) { + assertEquals("Only one value per field allowed for [SORTED] doc values field [field]", e.getMessage()); + } + } + + public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception { + Document doc = new Document(); + doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox"))); + doc.add(new TextField("text", "quick brown fox", Field.Store.NO)); + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true); + LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); + TermsEnum tenum = leafReader.terms("text").iterator(); + + assertEquals("brown", tenum.next().utf8ToString()); + PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(1, penum.nextPosition()); + assertEquals(6, penum.startOffset()); + assertEquals(11, penum.endOffset()); + + assertEquals("fox", tenum.next().utf8ToString()); + penum = tenum.postings(penum, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(2, penum.nextPosition()); + assertEquals(12, penum.startOffset()); + assertEquals(15, penum.endOffset()); + + assertEquals("quick", tenum.next().utf8ToString()); + penum = tenum.postings(penum, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(0, penum.nextPosition()); + assertEquals(0, penum.startOffset()); + assertEquals(5, penum.endOffset()); + + BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text"); + assertEquals("quick brown fox", binaryDocValues.get(0).utf8ToString()); + } } diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index 57e25fe0b0b6..3e6778aa6b0e 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -35,10 +35,18 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LegacyLongField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Fields; @@ -52,6 +60,9 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -434,6 +445,129 @@ public void testNonExistentField() throws IOException { assertNull(reader.terms("not-in-index")); } + public void testDocValuesMemoryIndexVsNormalIndex() throws Exception { + Document doc = new Document(); + long randomLong = random().nextLong(); + doc.add(new NumericDocValuesField("numeric", randomLong)); + if (random().nextBoolean()) { + doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); + } + int numValues = atLeast(5); + for (int i = 0; i < numValues; i++) { + randomLong = random().nextLong(); + doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); + if (random().nextBoolean()) { + // randomly duplicate field/value + doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong)); + } + if (random().nextBoolean()) { + doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO)); + } + } + BytesRef randomTerm = new BytesRef(randomTerm()); + doc.add(new BinaryDocValuesField("binary", randomTerm)); + if (random().nextBoolean()) { + doc.add(new StringField("binary", randomTerm, Field.Store.NO)); + } + randomTerm = new BytesRef(randomTerm()); + doc.add(new SortedDocValuesField("sorted", randomTerm)); + if (random().nextBoolean()) { + doc.add(new StringField("sorted", randomTerm, Field.Store.NO)); + } + numValues = atLeast(5); + for (int i = 0; i < numValues; i++) { + randomTerm = new BytesRef(randomTerm()); + doc.add(new SortedSetDocValuesField("sorted_set", randomTerm)); + if (random().nextBoolean()) { + // randomly duplicate field/value + doc.add(new SortedSetDocValuesField("sorted_set", randomTerm)); + } + if (random().nextBoolean()) { + // randomily just add a normal string field + doc.add(new StringField("sorted_set", randomTerm, Field.Store.NO)); + } + } + + MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer); + IndexReader indexReader = memoryIndex.createSearcher().getIndexReader(); + LeafReader leafReader = indexReader.leaves().get(0).reader(); + + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + writer.addDocument(doc); + writer.close(); + IndexReader controlIndexReader = DirectoryReader.open(dir); + LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader(); + + NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric"); + NumericDocValues controlNumericDocValues = controlLeafReader.getNumericDocValues("numeric"); + assertEquals(controlNumericDocValues.get(0), numericDocValues.get(0)); + + SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric"); + sortedNumericDocValues.setDocument(0); + SortedNumericDocValues controlSortedNumericDocValues = controlLeafReader.getSortedNumericDocValues("sorted_numeric"); + controlSortedNumericDocValues.setDocument(0); + assertEquals(controlSortedNumericDocValues.count(), sortedNumericDocValues.count()); + for (int i = 0; i < controlSortedNumericDocValues.count(); i++) { + assertEquals(controlSortedNumericDocValues.valueAt(i), sortedNumericDocValues.valueAt(i)); + } + + BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary"); + BinaryDocValues controlBinaryDocValues = controlLeafReader.getBinaryDocValues("binary"); + assertEquals(controlBinaryDocValues.get(0), binaryDocValues.get(0)); + + SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted"); + SortedDocValues controlSortedDocValues = controlLeafReader.getSortedDocValues("sorted"); + assertEquals(controlSortedDocValues.getValueCount(), sortedDocValues.getValueCount()); + assertEquals(controlSortedDocValues.get(0), sortedDocValues.get(0)); + assertEquals(controlSortedDocValues.getOrd(0), sortedDocValues.getOrd(0)); + assertEquals(controlSortedDocValues.lookupOrd(0), sortedDocValues.lookupOrd(0)); + + SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set"); + sortedSetDocValues.setDocument(0); + SortedSetDocValues controlSortedSetDocValues = controlLeafReader.getSortedSetDocValues("sorted_set"); + controlSortedSetDocValues.setDocument(0); + assertEquals(controlSortedSetDocValues.getValueCount(), sortedSetDocValues.getValueCount()); + for (long controlOrd = controlSortedSetDocValues.nextOrd(); controlOrd != SortedSetDocValues.NO_MORE_ORDS; + controlOrd = controlSortedSetDocValues.nextOrd()) { + assertEquals(controlOrd, sortedSetDocValues.nextOrd()); + assertEquals(controlSortedSetDocValues.lookupOrd(controlOrd), sortedSetDocValues.lookupOrd(controlOrd)); + } + assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd()); + + indexReader.close(); + controlIndexReader.close(); + dir.close(); + } + + public void testNormsWithDocValues() throws Exception { + MemoryIndex mi = new MemoryIndex(true, true); + MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + + mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer, 5f); + mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer, 5f); + LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); + + Document doc = new Document(); + doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox"))); + Field field = new TextField("text", "quick brown fox", Field.Store.NO); + field.setBoost(5f); + doc.add(field); + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + writer.addDocument(doc); + writer.close(); + + IndexReader controlIndexReader = DirectoryReader.open(dir); + LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader(); + + assertEquals(controlLeafReader.getNormValues("text").get(0), leafReader.getNormValues("text").get(0)); + + controlIndexReader.close(); + dir.close(); + } + public void testDuellMemIndex() throws IOException { LineFileDocs lineFileDocs = new LineFileDocs(random()); int numDocs = atLeast(10); From 4e3f376ca4fe84d99dad22c3e413b29c6e3a5d34 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 15 Mar 2016 05:18:53 -0400 Subject: [PATCH 0136/1113] re-enable accidentally turned off test evilness --- .../core/src/test/org/apache/lucene/index/TestPointValues.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 094623491152..49cbc2a6d7bb 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -501,7 +501,7 @@ public void testTieBreakByDocID() throws Exception { doc.add(new IntPoint("int", 17)); for(int i=0;i<300000;i++) { w.addDocument(doc); - if (false && random().nextInt(1000) == 17) { + if (random().nextInt(1000) == 17) { w.commit(); } } From 287d6c8be6a75f43ff298483ef79e8226f60d138 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 15 Mar 2016 07:36:02 -0400 Subject: [PATCH 0137/1113] LUCENE-7101: OfflineSorter had O(N^2) merge cost, and used too many temporary file descriptors, for large sorts --- lucene/CHANGES.txt | 3 ++ .../org/apache/lucene/util/OfflineSorter.java | 44 +++++++++++++------ .../org/apache/lucene/util/bkd/BKDWriter.java | 14 +++--- .../org/apache/lucene/index/Test2BPoints.java | 21 +-------- .../apache/lucene/util/TestOfflineSorter.java | 2 +- .../lucene/util/bkd/Test2BBKDPoints.java | 4 +- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0897c1a2f18a..27b238cbafcb 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -189,6 +189,9 @@ Bug Fixes On top of that with score mode average, the explain would fail with a NPE. (Martijn van Groningen) +* LUCENE-7101: OfflineSorter had O(N^2) merge cost, and used too many + temporary file descriptors, for large sorts (Mike McCandless) + Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java index 18e421b7bedf..3a22e33cb0c1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java @@ -64,7 +64,7 @@ public class OfflineSorter { /** * Maximum number of temporary files before doing an intermediate merge. */ - public final static int MAX_TEMPFILES = 128; + public final static int MAX_TEMPFILES = 10; private final Directory dir; @@ -232,6 +232,7 @@ public String sort(String inputFileName) throws IOException { sortInfo.totalTime = System.currentTimeMillis(); List segments = new ArrayList<>(); + int[] levelCounts = new int[1]; // So we can remove any partially written temp files on exception: TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir); @@ -244,15 +245,26 @@ public String sort(String inputFileName) throws IOException { segments.add(sortPartition(trackingDir)); sortInfo.tempMergeFiles++; sortInfo.lineCount += lineCount; + levelCounts[0]++; - // Handle intermediate merges. - if (segments.size() == maxTempFiles) { + // Handle intermediate merges; we need a while loop to "cascade" the merge when necessary: + int mergeLevel = 0; + while (levelCounts[mergeLevel] == maxTempFiles) { mergePartitions(trackingDir, segments); + if (mergeLevel+2 > levelCounts.length) { + levelCounts = ArrayUtil.grow(levelCounts, mergeLevel+2); + } + levelCounts[mergeLevel+1]++; + levelCounts[mergeLevel] = 0; + mergeLevel++; } } + + // TODO: we shouldn't have to do this? Can't we return a merged reader to + // the caller, who often consumes the result just once, instead? - // Merge the partitions to the output file with a priority queue. - if (segments.size() > 1) { + // Merge all partitions down to 1 (basically a forceMerge(1)): + while (segments.size() > 1) { mergePartitions(trackingDir, segments); } @@ -304,19 +316,25 @@ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOEx } } - /** Merge a list of sorted temporary files (partitions) into an output file. Note that this closes the - * incoming {@link IndexOutput}. */ + /** Merge the most recent {@code maxTempFile} partitions into a new partition. */ void mergePartitions(Directory trackingDir, List segments) throws IOException { long start = System.currentTimeMillis(); - PriorityQueue queue = new PriorityQueue(segments.size()) { + List segmentsToMerge; + if (segments.size() > maxTempFiles) { + segmentsToMerge = segments.subList(segments.size() - maxTempFiles, segments.size()); + } else { + segmentsToMerge = segments; + } + + PriorityQueue queue = new PriorityQueue(segmentsToMerge.size()) { @Override protected boolean lessThan(FileAndTop a, FileAndTop b) { return comparator.compare(a.current.get(), b.current.get()) < 0; } }; - ByteSequencesReader[] streams = new ByteSequencesReader[segments.size()]; + ByteSequencesReader[] streams = new ByteSequencesReader[segmentsToMerge.size()]; String newSegmentName = null; @@ -326,8 +344,8 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { newSegmentName = out.getName(); // Open streams and read the top for each file - for (int i = 0; i < segments.size(); i++) { - streams[i] = getReader(dir.openInput(segments.get(i), IOContext.READONCE)); + for (int i = 0; i < segmentsToMerge.size(); i++) { + streams[i] = getReader(dir.openInput(segmentsToMerge.get(i), IOContext.READONCE)); BytesRefBuilder bytes = new BytesRefBuilder(); boolean result = streams[i].read(bytes); assert result; @@ -354,9 +372,9 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { IOUtils.close(streams); } - IOUtils.deleteFiles(trackingDir, segments); + IOUtils.deleteFiles(trackingDir, segmentsToMerge); - segments.clear(); + segmentsToMerge.clear(); segments.add(newSegmentName); sortInfo.tempMergeFiles++; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index c5cdc303195c..10d97e3db461 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -212,11 +212,11 @@ public static void verifyParams(int numDims, int maxPointsInLeafNode, double max } } - /** If the current segment has too many points then we switchover to temp files / offline sort. */ - private void switchToOffline() throws IOException { + /** If the current segment has too many points then we spill over to temp files / offline sort. */ + private void spillToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: - offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "switch"); + offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill"); tempInput = offlinePointWriter.out; PointReader reader = heapPointWriter.getReader(0); for(int i=0;i= maxPointsSortInHeap) { if (offlinePointWriter == null) { - switchToOffline(); + spillToOffline(); } offlinePointWriter.append(packedValue, pointCount, docID); } else { @@ -733,7 +733,7 @@ public int compare(BytesRef a, BytesRef b) { // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: IndexOutput[] lastWriter = new IndexOutput[1]; - OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { + OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @Override @@ -742,9 +742,7 @@ protected ByteSequencesWriter getWriter(IndexOutput out) { return new ByteSequencesWriter(out) { @Override public void write(byte[] bytes, int off, int len) throws IOException { - if (len != bytesPerDoc) { - throw new IllegalArgumentException("len=" + len + " bytesPerDoc=" + bytesPerDoc); - } + assert len == bytesPerDoc: "len=" + len + " bytesPerDoc=" + bytesPerDoc; out.writeBytes(bytes, off, len); } }; diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java index 5766b911230c..f15ba19ed38f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -49,7 +49,6 @@ public class Test2BPoints extends LuceneTestCase { public void test1D() throws Exception { Directory dir = FSDirectory.open(createTempDir("2BPoints1D")); - System.out.println("DIR: " + ((FSDirectory) dir).getDirectory()); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())) .setCodec(getCodec()) @@ -144,24 +143,6 @@ public void test2D() throws Exception { } private static Codec getCodec() { - - return new FilterCodec("Lucene60", Codec.forName("Lucene60")) { - @Override - public PointsFormat pointsFormat() { - return new PointsFormat() { - @Override - public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - int maxPointsInLeafNode = 1024; - double maxMBSortInHeap = 256.0; - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); - } - - @Override - public PointsReader fieldsReader(SegmentReadState readState) throws IOException { - return new Lucene60PointsReader(readState); - } - }; - } - }; + return Codec.forName("Lucene60"); } } diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java index 53225313ade9..b45a3bbb53fb 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java @@ -82,7 +82,7 @@ public void testSmallRandom() throws Exception { try (Directory dir = newDirectory()) { SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES), generateRandom((int)OfflineSorter.MB * 20)); - assertEquals(1, sortInfo.mergeRounds); + assertEquals(3, sortInfo.mergeRounds); } } diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java index 4df2ebe19293..cfb98b09c78a 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -55,7 +55,7 @@ public void test1D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 1024, 256, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 26L * numDocs); int counter = 0; byte[] packedBytes = new byte[Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { @@ -88,7 +88,7 @@ public void test2D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 1024, 256, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs); int counter = 0; byte[] packedBytes = new byte[2*Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { From d4fe0c3849d9e1140e5d6cd5d7b4d0e0495d9ea5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 15 Mar 2016 12:32:46 +0100 Subject: [PATCH 0138/1113] LUCENE-7093: Added point values support to the memory index --- lucene/CHANGES.txt | 3 + .../lucene/index/memory/MemoryIndex.java | 185 ++++++++++++++++-- .../lucene/index/memory/TestMemoryIndex.java | 160 +++++++++++++++ .../memory/TestMemoryIndexAgainstRAMDir.java | 58 ++++++ 4 files changed, 389 insertions(+), 17 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 27b238cbafcb..8004b7f94743 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -202,6 +202,9 @@ Other * LUCENE-7091: Add doc values support to MemoryIndex (Martijn van Groningen, David Smiley) +* LUCENE-7093: Add point values support to MemoryIndex + (Martijn van Groningen, Mike McCandless) + ======================= Lucene 5.5.0 ======================= New Features diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 40159aa1f972..58a10172329e 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -258,7 +258,8 @@ public void addField(String fieldName, String text, Analyzer analyzer) { throw new IllegalArgumentException("analyzer must not be null"); TokenStream stream = analyzer.tokenStream(fieldName, text); - addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName), DocValuesType.NONE, null); + addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName), + DocValuesType.NONE, null, 0, 0, null); } /** @@ -377,10 +378,6 @@ public void addField(IndexableField field, Analyzer analyzer) { * structures are not supported by MemoryIndex */ public void addField(IndexableField field, Analyzer analyzer, float boost) { - if (field.fieldType().pointDimensionCount() != 0) { - throw new IllegalArgumentException("MemoryIndex does not support Points"); - } - int offsetGap; TokenStream tokenStream; int positionIncrementGap; @@ -412,7 +409,12 @@ public void addField(IndexableField field, Analyzer analyzer, float boost) { default: throw new UnsupportedOperationException("unknown doc values type [" + docValuesType + "]"); } - addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue); + BytesRef pointValue = null; + if (field.fieldType().pointDimensionCount() > 0) { + pointValue = field.binaryValue(); + } + addField(field.name(), tokenStream, boost, positionIncrementGap, offsetGap, docValuesType, docValuesValue, + field.fieldType().pointDimensionCount(), field.fieldType().pointNumBytes(), pointValue); } /** @@ -481,11 +483,12 @@ public void addField(String fieldName, TokenStream stream, float boost, int posi * @see org.apache.lucene.document.Field#setBoost(float) */ public void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap) { - addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null); + addField(fieldName, tokenStream, boost, positionIncrementGap, offsetGap, DocValuesType.NONE, null, 0, 0, null); } private void addField(String fieldName, TokenStream tokenStream, float boost, int positionIncrementGap, int offsetGap, - DocValuesType docValuesType, Object docValuesValue) { + DocValuesType docValuesType, Object docValuesValue, int pointDimensionCount, int pointNumBytes, + BytesRef pointValue) { if (frozen) { throw new IllegalArgumentException("Cannot call addField() when MemoryIndex is frozen"); @@ -503,7 +506,9 @@ private void addField(String fieldName, TokenStream tokenStream, float boost, in FieldInfo fieldInfo = new FieldInfo(fieldName, fields.size(), true, false, storePayloads, indexOptions, docValuesType, -1, Collections.emptyMap(), 0, 0); fields.put(fieldName, info = new Info(fieldInfo, byteBlockPool)); } - + if (pointDimensionCount > 0) { + storePointValues(info, pointDimensionCount, pointNumBytes, pointValue); + } if (docValuesType != DocValuesType.NONE) { storeDocValues(info, docValuesType, docValuesValue); } @@ -512,6 +517,15 @@ private void addField(String fieldName, TokenStream tokenStream, float boost, in } } + private void storePointValues(Info info, int pointDimensionCount, int pointNumBytes, BytesRef pointValue) { + info.fieldInfo.setPointDimensions(pointDimensionCount, pointNumBytes); + if (info.pointValues == null) { + info.pointValues = new BytesRef[4]; + } + info.pointValues = ArrayUtil.grow(info.pointValues, info.pointValuesCount + 1); + info.pointValues[info.pointValuesCount++] = BytesRef.deepCopyOf(pointValue); + } + private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) { String fieldName = info.fieldInfo.name; DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType(); @@ -829,7 +843,15 @@ private final class Info { private NumericDocValuesProducer numericProducer; - private boolean preparedDocValues; + private boolean preparedDocValuesAndPointValues; + + private BytesRef[] pointValues; + + private byte[] minPackedValue; + + private byte[] maxPackedValue; + + private int pointValuesCount; private Info(FieldInfo fieldInfo, ByteBlockPool byteBlockPool) { this.fieldInfo = fieldInfo; @@ -841,7 +863,7 @@ private Info(FieldInfo fieldInfo, ByteBlockPool byteBlockPool) { void freeze() { sortTerms(); - prepareDocValues(); + prepareDocValuesAndPointValues(); getNormDocValues(); } @@ -859,8 +881,8 @@ void sortTerms() { } } - void prepareDocValues() { - if (preparedDocValues == false) { + void prepareDocValuesAndPointValues() { + if (preparedDocValuesAndPointValues == false) { DocValuesType dvType = fieldInfo.getDocValuesType(); if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) { numericProducer.prepareForUsage(); @@ -868,7 +890,30 @@ void prepareDocValues() { if (dvType == DocValuesType.BINARY || dvType == DocValuesType.SORTED || dvType == DocValuesType.SORTED_SET) { binaryProducer.prepareForUsage(); } - preparedDocValues = true; + if (pointValues != null) { + assert pointValues[0].bytes.length == pointValues[0].length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this"; + + final int numDimensions = fieldInfo.getPointDimensionCount(); + final int numBytesPerDimension = fieldInfo.getPointNumBytes(); + minPackedValue = pointValues[0].bytes.clone(); + maxPackedValue = pointValues[0].bytes.clone(); + + for (int i = 0; i < pointValuesCount; i++) { + BytesRef pointValue = pointValues[i]; + assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this"; + + for (int dim = 0; dim < numDimensions; ++dim) { + int offset = dim * numBytesPerDimension; + if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) { + System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension); + } + if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) { + System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension); + } + } + } + } + preparedDocValuesAndPointValues = true; } } @@ -977,11 +1022,22 @@ private void prepareForUsage() { * required by the Lucene IndexReader contracts. */ private final class MemoryIndexReader extends LeafReader { - + + private final PointValues pointValues; + private MemoryIndexReader() { super(); // avoid as much superclass baggage as possible + boolean hasPointValues = false; for (Info info : fields.values()) { - info.prepareDocValues(); + info.prepareDocValuesAndPointValues(); + if (info.pointValues != null) { + hasPointValues = true; + } + } + if (hasPointValues) { + pointValues = new MemoryIndexPointValues(); + } else { + pointValues = null; } } @@ -1111,7 +1167,7 @@ public Bits getDocsWithField(String field) throws IOException { @Override public PointValues getPointValues() { - return null; + return pointValues; } @Override @@ -1412,6 +1468,101 @@ public long cost() { return 1; } } + + private class MemoryIndexPointValues extends PointValues { + + @Override + public void intersect(String fieldName, IntersectVisitor visitor) throws IOException { + Info info = fields.get(fieldName); + if (info == null) { + return; + } + BytesRef[] values = info.pointValues; + if (values == null) { + return; + } + + visitor.grow(info.pointValuesCount); + for (int i = 0; i < info.pointValuesCount; i++) { + visitor.visit(0, values[i].bytes); + } + } + + @Override + public byte[] getMinPackedValue(String fieldName) throws IOException { + Info info = fields.get(fieldName); + if (info == null) { + return null; + } + BytesRef[] values = info.pointValues; + if (values != null) { + return info.minPackedValue; + } else { + return null; + } + } + + @Override + public byte[] getMaxPackedValue(String fieldName) throws IOException { + Info info = fields.get(fieldName); + if (info == null) { + return null; + } + BytesRef[] values = info.pointValues; + if (values != null) { + return info.maxPackedValue; + } else { + return null; + } + } + + @Override + public int getNumDimensions(String fieldName) throws IOException { + Info info = fields.get(fieldName); + if (info == null){ + return 0; + } + return info.fieldInfo.getPointDimensionCount(); + } + + @Override + public int getBytesPerDimension(String fieldName) throws IOException { + Info info = fields.get(fieldName); + if (info == null){ + return 0; + } + return info.fieldInfo.getPointNumBytes(); + } + + @Override + public long size(String fieldName) { + Info info = fields.get(fieldName); + if (info == null) { + return 0; + } + BytesRef[] values = info.pointValues; + if (values != null) { + return info.pointValuesCount; + } else { + return 0; + } + } + + @Override + public int getDocCount(String fieldName) { + Info info = fields.get(fieldName); + if (info == null) { + return 0; + } + BytesRef[] values = info.pointValues; + if (values != null) { + return 1; + } else { + return 0; + } + } + + } @Override public Fields getTermVectors(int docID) { diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index 7282e0eb50f6..1010c137461d 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -17,13 +17,28 @@ package org.apache.lucene.index.memory; import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import java.util.stream.LongStream; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockPayloadAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -31,9 +46,12 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; @@ -44,7 +62,9 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.util.BytesRef; @@ -310,4 +330,144 @@ public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception { assertEquals("quick brown fox", binaryDocValues.get(0).utf8ToString()); } + public void testPointValues() throws Exception { + List> fieldFunctions = Arrays.asList( + (t) -> new IntPoint("number", t.intValue()), + (t) -> new LongPoint("number", t), + (t) -> new FloatPoint("number", t.floatValue()), + (t) -> new DoublePoint("number", t.doubleValue()) + ); + List> exactQueryFunctions = Arrays.asList( + (t) -> IntPoint.newExactQuery("number", t.intValue()), + (t) -> LongPoint.newExactQuery("number", t), + (t) -> FloatPoint.newExactQuery("number", t.floatValue()), + (t) -> DoublePoint.newExactQuery("number", t.doubleValue()) + ); + List> setQueryFunctions = Arrays.asList( + (t) -> IntPoint.newSetQuery("number", LongStream.of(t).mapToInt(value -> (int) value).toArray()), + (t) -> LongPoint.newSetQuery("number", t), + (t) -> FloatPoint.newSetQuery("number", Arrays.asList(LongStream.of(t).mapToObj(value -> (float) value).toArray(Float[]::new))), + (t) -> DoublePoint.newSetQuery("number", LongStream.of(t).mapToDouble(value -> (double) value).toArray()) + ); + List> rangeQueryFunctions = Arrays.asList( + (t, u) -> IntPoint.newRangeQuery("number", t.intValue(), u.intValue()), + (t, u) -> LongPoint.newRangeQuery("number", t, u), + (t, u) -> FloatPoint.newRangeQuery("number", t.floatValue(), u.floatValue()), + (t, u) -> DoublePoint.newRangeQuery("number", t.doubleValue(), u.doubleValue()) + ); + + for (int i = 0; i < fieldFunctions.size(); i++) { + Function fieldFunction = fieldFunctions.get(i); + Function exactQueryFunction = exactQueryFunctions.get(i); + Function setQueryFunction = setQueryFunctions.get(i); + BiFunction rangeQueryFunction = rangeQueryFunctions.get(i); + + Document doc = new Document(); + for (int number = 1; number < 32; number += 2) { + doc.add(fieldFunction.apply((long) number)); + } + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); + IndexSearcher indexSearcher = mi.createSearcher(); + Query query = exactQueryFunction.apply(5L); + assertEquals(1, indexSearcher.count(query)); + query = exactQueryFunction.apply(4L); + assertEquals(0, indexSearcher.count(query)); + + + query = setQueryFunction.apply(new long[]{3L, 9L, 19L}); + assertEquals(1, indexSearcher.count(query)); + query = setQueryFunction.apply(new long[]{2L, 8L, 13L}); + assertEquals(1, indexSearcher.count(query)); + query = setQueryFunction.apply(new long[]{2L, 8L, 16L}); + assertEquals(0, indexSearcher.count(query)); + + query = rangeQueryFunction.apply(2L, 16L); + assertEquals(1, indexSearcher.count(query)); + query = rangeQueryFunction.apply(24L, 48L); + assertEquals(1, indexSearcher.count(query)); + query = rangeQueryFunction.apply(48L, 68L); + assertEquals(0, indexSearcher.count(query)); + } + } + + public void testPointValuesDoNotAffectBoostPositionsOrOffset() throws Exception { + MemoryIndex mi = new MemoryIndex(true, true); + mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), analyzer, 5f); + mi.addField(new BinaryPoint("text", "quick".getBytes(StandardCharsets.UTF_8)), analyzer, 5f); + mi.addField(new BinaryPoint("text", "brown".getBytes(StandardCharsets.UTF_8)), analyzer, 5f); + LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); + TermsEnum tenum = leafReader.terms("text").iterator(); + + assertEquals("brown", tenum.next().utf8ToString()); + PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(1, penum.nextPosition()); + assertEquals(6, penum.startOffset()); + assertEquals(11, penum.endOffset()); + + assertEquals("fox", tenum.next().utf8ToString()); + penum = tenum.postings(penum, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(2, penum.nextPosition()); + assertEquals(12, penum.startOffset()); + assertEquals(15, penum.endOffset()); + + assertEquals("quick", tenum.next().utf8ToString()); + penum = tenum.postings(penum, PostingsEnum.OFFSETS); + assertEquals(0, penum.nextDoc()); + assertEquals(1, penum.freq()); + assertEquals(0, penum.nextPosition()); + assertEquals(0, penum.startOffset()); + assertEquals(5, penum.endOffset()); + + IndexSearcher indexSearcher = mi.createSearcher(); + assertEquals(1, indexSearcher.count(BinaryPoint.newExactQuery("text", "quick".getBytes(StandardCharsets.UTF_8)))); + assertEquals(1, indexSearcher.count(BinaryPoint.newExactQuery("text", "brown".getBytes(StandardCharsets.UTF_8)))); + assertEquals(0, indexSearcher.count(BinaryPoint.newExactQuery("text", "jumps".getBytes(StandardCharsets.UTF_8)))); + } + + public void test2DPoints() throws Exception { + Document doc = new Document(); + doc.add(new IntPoint("ints", 0, -100)); + doc.add(new IntPoint("ints", 20, 20)); + doc.add(new IntPoint("ints", 100, -100)); + doc.add(new LongPoint("longs", 0L, -100L)); + doc.add(new LongPoint("longs", 20L, 20L)); + doc.add(new LongPoint("longs", 100L, -100L)); + doc.add(new FloatPoint("floats", 0F, -100F)); + doc.add(new FloatPoint("floats", 20F, 20F)); + doc.add(new FloatPoint("floats", 100F, -100F)); + doc.add(new DoublePoint("doubles", 0D, -100D)); + doc.add(new DoublePoint("doubles", 20D, 20D)); + doc.add(new DoublePoint("doubles", 100D, -100D)); + + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); + IndexSearcher s = mi.createSearcher(); + + assertEquals(1, s.count(IntPoint.newRangeQuery("ints", new int[] {10, 10}, new int[] {30, 30}))); + assertEquals(1, s.count(LongPoint.newRangeQuery("longs", new long[] {10L, 10L}, new long[] {30L, 30L}))); + assertEquals(1, s.count(FloatPoint.newRangeQuery("floats", new float[] {10F, 10F}, new float[] {30F, 30F}))); + assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D}))); + } + + public void testIndexingPointsAndDocValues() throws Exception { + FieldType type = new FieldType(); + type.setDimensions(1, 4); + type.setDocValuesType(DocValuesType.BINARY); + type.freeze(); + Document doc = new Document(); + byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8); + doc.add(new BinaryPoint("field", packedPoint, type)); + MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); + LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader(); + + assertEquals(1, leafReader.getPointValues().size("field")); + assertArrayEquals(packedPoint, leafReader.getPointValues().getMinPackedValue("field")); + assertArrayEquals(packedPoint, leafReader.getPointValues().getMaxPackedValue("field")); + + assertEquals("term", leafReader.getBinaryDocValues("field").get(0).utf8ToString()); + } + } diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java index 3e6778aa6b0e..ba1263ecea93 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java @@ -21,8 +21,12 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.function.IntSupplier; +import java.util.function.Supplier; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -37,9 +41,13 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LegacyLongField; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -70,6 +78,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -568,6 +577,55 @@ public void testNormsWithDocValues() throws Exception { dir.close(); } + public void testPointValuesMemoryIndexVsNormalIndex() throws Exception { + int size = atLeast(12); + + List randomValues = new ArrayList<>(); + + Document doc = new Document(); + for (Integer randomInteger : random().ints(size).toArray()) { + doc.add(new IntPoint("int", randomInteger)); + randomValues.add(randomInteger); + doc.add(new LongPoint("long", randomInteger)); + doc.add(new FloatPoint("float", randomInteger)); + doc.add(new DoublePoint("double", randomInteger)); + } + + MockAnalyzer mockAnalyzer = new MockAnalyzer(random()); + MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer); + IndexSearcher memoryIndexSearcher = memoryIndex.createSearcher(); + + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer)); + writer.addDocument(doc); + writer.close(); + IndexReader controlIndexReader = DirectoryReader.open(dir); + IndexSearcher controlIndexSearcher = new IndexSearcher(controlIndexReader); + + Supplier valueSupplier = () -> randomValues.get(random().nextInt(randomValues.size())); + Query[] queries = new Query[] { + IntPoint.newExactQuery("int", valueSupplier.get()), + LongPoint.newExactQuery("long", valueSupplier.get()), + FloatPoint.newExactQuery("float", valueSupplier.get()), + DoublePoint.newExactQuery("double", valueSupplier.get()), + IntPoint.newSetQuery("int", valueSupplier.get(), valueSupplier.get()), + LongPoint.newSetQuery("long", valueSupplier.get(), valueSupplier.get()), + FloatPoint.newSetQuery("float", valueSupplier.get(), valueSupplier.get()), + DoublePoint.newSetQuery("double", valueSupplier.get(), valueSupplier.get()), + IntPoint.newRangeQuery("int", valueSupplier.get(), valueSupplier.get()), + LongPoint.newRangeQuery("long", valueSupplier.get(), valueSupplier.get()), + FloatPoint.newRangeQuery("float", valueSupplier.get(), valueSupplier.get()), + DoublePoint.newRangeQuery("double", valueSupplier.get(), valueSupplier.get()) + }; + for (Query query : queries) { + assertEquals(controlIndexSearcher.count(query), controlIndexSearcher.count(query)); + } + + memoryIndexSearcher.getIndexReader().close(); + controlIndexReader.close(); + dir.close(); + } + public void testDuellMemIndex() throws IOException { LineFileDocs lineFileDocs = new LineFileDocs(random()); int numDocs = atLeast(10); From 15fdfd3caa54c1e6d8cf3435ded285c3b65bea8c Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 15 Mar 2016 12:29:07 +0100 Subject: [PATCH 0139/1113] SOLR-7739 - applied patch from Alessandro Benedetti for integrating Lucene classification into Solr (cherry picked from commit 5801caa) --- .../idea/solr/core/src/java/solr-core.iml | 1 + .../idea/solr/core/src/solr-core-tests.iml | 1 + solr/common-build.xml | 4 +- .../ClassificationUpdateProcessor.java | 102 ++++++++ .../ClassificationUpdateProcessorFactory.java | 223 +++++++++++++++++ .../conf/schema-classification.xml | 43 ++++ .../conf/solrconfig-classification.xml | 53 ++++ ...ssificationUpdateProcessorFactoryTest.java | 234 ++++++++++++++++++ 8 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java create mode 100644 solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-classification.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml create mode 100644 solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java diff --git a/dev-tools/idea/solr/core/src/java/solr-core.iml b/dev-tools/idea/solr/core/src/java/solr-core.iml index f03268c350bd..822b24f6cabb 100644 --- a/dev-tools/idea/solr/core/src/java/solr-core.iml +++ b/dev-tools/idea/solr/core/src/java/solr-core.iml @@ -27,6 +27,7 @@ + diff --git a/dev-tools/idea/solr/core/src/solr-core-tests.iml b/dev-tools/idea/solr/core/src/solr-core-tests.iml index c9f722a1212b..56f768b49f39 100644 --- a/dev-tools/idea/solr/core/src/solr-core-tests.iml +++ b/dev-tools/idea/solr/core/src/solr-core-tests.iml @@ -21,6 +21,7 @@ + diff --git a/solr/common-build.xml b/solr/common-build.xml index 6a0692860772..78e10aabac17 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -108,6 +108,7 @@ + @@ -169,7 +170,7 @@ + jar-misc, jar-spatial-extras, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification"> @@ -322,6 +323,7 @@ + diff --git a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java new file mode 100644 index 000000000000..b752565cd6d0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessor.java @@ -0,0 +1,102 @@ +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.classification.ClassificationResult; +import org.apache.lucene.classification.document.DocumentClassifier; +import org.apache.lucene.classification.document.KNearestNeighborDocumentClassifier; +import org.apache.lucene.classification.document.SimpleNaiveBayesDocumentClassifier; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.update.AddUpdateCommand; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This Class is a Request Update Processor to classify the document in input and add a field + * containing the class to the Document. + * It uses the Lucene Document Classification module, see {@link DocumentClassifier}. + */ +class ClassificationUpdateProcessor + extends UpdateRequestProcessor { + + private String classFieldName; // the field to index the assigned class + + private DocumentClassifier classifier; + + /** + * Sole constructor + * + * @param inputFieldNames fields to be used as classifier's inputs + * @param classFieldName field to be used as classifier's output + * @param minDf setting for {@link org.apache.lucene.queries.mlt.MoreLikeThis#minDocFreq}, in case algorithm is {@code "knn"} + * @param minTf setting for {@link org.apache.lucene.queries.mlt.MoreLikeThis#minTermFreq}, in case algorithm is {@code "knn"} + * @param k setting for k nearest neighbors to analyze, in case algorithm is {@code "knn"} + * @param algorithm the name of the classifier to use + * @param next next update processor in the chain + * @param indexReader index reader + * @param schema schema + */ + public ClassificationUpdateProcessor(String[] inputFieldNames, String classFieldName, int minDf, int minTf, int k, String algorithm, + UpdateRequestProcessor next, LeafReader indexReader, IndexSchema schema) { + super(next); + this.classFieldName = classFieldName; + Map field2analyzer = new HashMap(); + for (String fieldName : inputFieldNames) { + SchemaField fieldFromSolrSchema = schema.getField(fieldName); + Analyzer indexAnalyzer = fieldFromSolrSchema.getType().getQueryAnalyzer(); + field2analyzer.put(fieldName, indexAnalyzer); + } + switch (algorithm) { + case "knn": + classifier = new KNearestNeighborDocumentClassifier(indexReader, null, null, k, minDf, minTf, classFieldName, field2analyzer, inputFieldNames); + break; + case "bayes": + classifier = new SimpleNaiveBayesDocumentClassifier(indexReader, null, classFieldName, field2analyzer, inputFieldNames); + break; + } + } + + /** + * @param cmd the update command in input conaining the Document to classify + * @throws IOException If there is a low-level I/O error + */ + @Override + public void processAdd(AddUpdateCommand cmd) + throws IOException { + SolrInputDocument doc = cmd.getSolrInputDocument(); + Document luceneDocument = cmd.getLuceneDocument(); + String assignedClass; + Object documentClass = doc.getFieldValue(classFieldName); + if (documentClass == null) { + ClassificationResult classificationResult = classifier.assignClass(luceneDocument); + if (classificationResult != null) { + assignedClass = classificationResult.getAssignedClass().utf8ToString(); + doc.addField(classFieldName, assignedClass); + } + } + super.processAdd(cmd); + } +} diff --git a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java new file mode 100644 index 000000000000..79b3240f0053 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java @@ -0,0 +1,223 @@ +package org.apache.solr.update.processor; + +import org.apache.lucene.index.LeafReader; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This class implements an UpdateProcessorFactory for the Classification Update Processor. + * It takes in input a series of parameter that will be necessary to instantiate and use the Classifier + */ +public class ClassificationUpdateProcessorFactory extends UpdateRequestProcessorFactory { + + // Update Processor Config params + private static final String INPUT_FIELDS_PARAM = "inputFields"; + private static final String CLASS_FIELD_PARAM = "classField"; + private static final String ALGORITHM_PARAM = "algorithm"; + private static final String KNN_MIN_TF_PARAM = "knn.minTf"; + private static final String KNN_MIN_DF_PARAM = "knn.minDf"; + private static final String KNN_K_PARAM = "knn.k"; + + //Update Processor Defaults + private static final int DEFAULT_MIN_TF = 1; + private static final int DEFAULT_MIN_DF = 1; + private static final int DEFAULT_K = 10; + private static final String DEFAULT_ALGORITHM = "knn"; + + private String[] inputFieldNames; // the array of fields to be sent to the Classifier + + private String classFieldName; // the field containing the class for the Document + + private String algorithm; // the Classification Algorithm to use - currently 'knn' or 'bayes' + + private int minTf; // knn specific - the minimum Term Frequency for considering a term + + private int minDf; // knn specific - the minimum Document Frequency for considering a term + + private int k; // knn specific - thw window of top results to evaluate, when assgning the class + + @Override + public void init(final NamedList args) { + if (args != null) { + SolrParams params = SolrParams.toSolrParams(args); + + String fieldNames = params.get(INPUT_FIELDS_PARAM);// must be a comma separated list of fields + checkNotNull(INPUT_FIELDS_PARAM, fieldNames); + inputFieldNames = fieldNames.split("\\,"); + + classFieldName = params.get(CLASS_FIELD_PARAM); + checkNotNull(CLASS_FIELD_PARAM, classFieldName); + + algorithm = params.get(ALGORITHM_PARAM); + if (algorithm == null) + algorithm = DEFAULT_ALGORITHM; + + minTf = getIntParam(params, KNN_MIN_TF_PARAM, DEFAULT_MIN_TF); + minDf = getIntParam(params, KNN_MIN_DF_PARAM, DEFAULT_MIN_DF); + k = getIntParam(params, KNN_K_PARAM, DEFAULT_K); + } + } + + /* + * Returns an Int parsed param or a default if the param is null + * + * @param params Solr params in input + * @param name the param name + * @param defaultValue the param default + * @return the Int value for the param + */ + private int getIntParam(SolrParams params, String name, int defaultValue) { + String paramString = params.get(name); + int paramInt; + if (paramString != null && !paramString.isEmpty()) { + paramInt = Integer.parseInt(paramString); + } else { + paramInt = defaultValue; + } + return paramInt; + } + + private void checkNotNull(String paramName, Object param) { + if (param == null) { + throw new SolrException + (SolrException.ErrorCode.SERVER_ERROR, + "Classification UpdateProcessor '" + paramName + "' can not be null"); + } + } + + @Override + public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { + IndexSchema schema = req.getSchema(); + LeafReader leafReader = req.getSearcher().getLeafReader(); + return new ClassificationUpdateProcessor(inputFieldNames, classFieldName, minDf, minTf, k, algorithm, next, leafReader, schema); + } + + /** + * get field names used as classifier's inputs + * + * @return the input field names + */ + public String[] getInputFieldNames() { + return inputFieldNames; + } + + /** + * set field names used as classifier's inputs + * + * @param inputFieldNames the input field names + */ + public void setInputFieldNames(String[] inputFieldNames) { + this.inputFieldNames = inputFieldNames; + } + + /** + * get field names used as classifier's output + * + * @return the output field name + */ + public String getClassFieldName() { + return classFieldName; + } + + /** + * set field names used as classifier's output + * + * @param classFieldName the output field name + */ + public void setClassFieldName(String classFieldName) { + this.classFieldName = classFieldName; + } + + /** + * get the name of the classifier algorithm used + * + * @return the classifier algorithm used + */ + public String getAlgorithm() { + return algorithm; + } + + /** + * set the name of the classifier algorithm used + * + * @param algorithm the classifier algorithm used + */ + public void setAlgorithm(String algorithm) { + this.algorithm = algorithm; + } + + /** + * get the min term frequency value to be used in case algorithm is {@code "knn"} + * + * @return the min term frequency + */ + public int getMinTf() { + return minTf; + } + + /** + * set the min term frequency value to be used in case algorithm is {@code "knn"} + * + * @param minTf the min term frequency + */ + public void setMinTf(int minTf) { + this.minTf = minTf; + } + + /** + * get the min document frequency value to be used in case algorithm is {@code "knn"} + * + * @return the min document frequency + */ + public int getMinDf() { + return minDf; + } + + /** + * set the min document frequency value to be used in case algorithm is {@code "knn"} + * + * @param minDf the min document frequency + */ + public void setMinDf(int minDf) { + this.minDf = minDf; + } + + /** + * get the the no. of nearest neighbor to analyze, to be used in case algorithm is {@code "knn"} + * + * @return the no. of neighbors to analyze + */ + public int getK() { + return k; + } + + /** + * set the the no. of nearest neighbor to analyze, to be used in case algorithm is {@code "knn"} + * + * @param k the no. of neighbors to analyze + */ + public void setK(int k) { + this.k = k; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml b/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml new file mode 100644 index 000000000000..89c27a6766aa --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-classification.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml new file mode 100644 index 000000000000..3656335d1840 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml @@ -0,0 +1,53 @@ + + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + + ${solr.ulog.dir:} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + title,content,author + cat + + knn + 1 + 1 + 5 + + + + diff --git a/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java new file mode 100644 index 000000000000..27d8dca71eef --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/ClassificationUpdateProcessorFactoryTest.java @@ -0,0 +1,234 @@ +package org.apache.solr.update.processor; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.MultiMapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests for {@link ClassificationUpdateProcessor} and {@link ClassificationUpdateProcessorFactory} + */ +public class ClassificationUpdateProcessorFactoryTest extends SolrTestCaseJ4 { + // field names are used in accordance with the solrconfig and schema supplied + private static final String ID = "id"; + private static final String TITLE = "title"; + private static final String CONTENT = "content"; + private static final String AUTHOR = "author"; + private static final String CLASS = "cat"; + + private static final String CHAIN = "classification"; + + + private ClassificationUpdateProcessorFactory cFactoryToTest = new ClassificationUpdateProcessorFactory(); + private NamedList args = new NamedList(); + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("enable.update.log", "false"); + initCore("solrconfig-classification.xml", "schema-classification.xml"); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + assertU(commit()); + } + + @Before + public void initArgs() { + args.add("inputFields", "inputField1,inputField2"); + args.add("classField", "classField1"); + args.add("algorithm", "bayes"); + args.add("knn.k", "9"); + args.add("knn.minDf", "8"); + args.add("knn.minTf", "10"); + } + + @Test + public void testFullInit() { + cFactoryToTest.init(args); + + String[] inputFieldNames = cFactoryToTest.getInputFieldNames(); + assertEquals("inputField1", inputFieldNames[0]); + assertEquals("inputField2", inputFieldNames[1]); + assertEquals("classField1", cFactoryToTest.getClassFieldName()); + assertEquals("bayes", cFactoryToTest.getAlgorithm()); + assertEquals(8, cFactoryToTest.getMinDf()); + assertEquals(10, cFactoryToTest.getMinTf()); + assertEquals(9, cFactoryToTest.getK()); + + } + + @Test + public void testInitEmptyInputField() { + args.removeAll("inputFields"); + try { + cFactoryToTest.init(args); + } catch (SolrException e) { + assertEquals("Classification UpdateProcessor 'inputFields' can not be null", e.getMessage()); + } + } + + @Test + public void testInitEmptyClassField() { + args.removeAll("classField"); + try { + cFactoryToTest.init(args); + } catch (SolrException e) { + assertEquals("Classification UpdateProcessor 'classField' can not be null", e.getMessage()); + } + } + + @Test + public void testDefaults() { + args.removeAll("algorithm"); + args.removeAll("knn.k"); + args.removeAll("knn.minDf"); + args.removeAll("knn.minTf"); + cFactoryToTest.init(args); + assertEquals("knn", cFactoryToTest.getAlgorithm()); + assertEquals(1, cFactoryToTest.getMinDf()); + assertEquals(1, cFactoryToTest.getMinTf()); + assertEquals(10, cFactoryToTest.getK()); + } + + @Test + public void testBasicClassification() throws Exception { + prepareTrainedIndex(); + // To be classified,we index documents without a class and verify the expected one is returned + addDoc(adoc(ID, "10", + TITLE, "word4 word4 word4", + CONTENT, "word5 word5 ", + AUTHOR, "Name1 Surname1")); + addDoc(adoc(ID, "11", + TITLE, "word1 word1", + CONTENT, "word2 word2", + AUTHOR, "Name Surname")); + addDoc(commit()); + + Document doc10 = getDoc("10"); + assertEquals("class2", doc10.get(CLASS)); + Document doc11 = getDoc("11"); + assertEquals("class1", doc11.get(CLASS)); + } + + /** + * Index some example documents with a class manually assigned. + * This will be our trained model. + * + * @throws Exception If there is a low-level I/O error + */ + private void prepareTrainedIndex() throws Exception { + //class1 + addDoc(adoc(ID, "1", + TITLE, "word1 word1 word1", + CONTENT, "word2 word2 word2", + AUTHOR, "Name Surname", + CLASS, "class1")); + addDoc(adoc(ID, "2", + TITLE, "word1 word1", + CONTENT, "word2 word2", + AUTHOR, "Name Surname", + CLASS, "class1")); + addDoc(adoc(ID, "3", + TITLE, "word1 word1 word1", + CONTENT, "word2", + AUTHOR, "Name Surname", + CLASS, "class1")); + addDoc(adoc(ID, "4", + TITLE, "word1 word1 word1", + CONTENT, "word2 word2 word2", + AUTHOR, "Name Surname", + CLASS, "class1")); + //class2 + addDoc(adoc(ID, "5", + TITLE, "word4 word4 word4", + CONTENT, "word5 word5", + AUTHOR, "Name1 Surname1", + CLASS, "class2")); + addDoc(adoc(ID, "6", + TITLE, "word4 word4", + CONTENT, "word5", + AUTHOR, "Name1 Surname1", + CLASS, "class2")); + addDoc(adoc(ID, "7", + TITLE, "word4 word4 word4", + CONTENT, "word5 word5 word5", + AUTHOR, "Name1 Surname1", + CLASS, "class2")); + addDoc(adoc(ID, "8", + TITLE, "word4", + CONTENT, "word5 word5 word5 word5", + AUTHOR, "Name1 Surname1", + CLASS, "class2")); + addDoc(commit()); + } + + private Document getDoc(String id) throws IOException { + try (SolrQueryRequest req = req()) { + SolrIndexSearcher searcher = req.getSearcher(); + TermQuery query = new TermQuery(new Term(ID, id)); + TopDocs doc1 = searcher.search(query, 1); + ScoreDoc scoreDoc = doc1.scoreDocs[0]; + return searcher.doc(scoreDoc.doc); + } + } + + static void addDoc(String doc) throws Exception { + Map params = new HashMap<>(); + MultiMapSolrParams mmparams = new MultiMapSolrParams(params); + params.put(UpdateParams.UPDATE_CHAIN, new String[]{CHAIN}); + SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), + (SolrParams) mmparams) { + }; + + UpdateRequestHandler handler = new UpdateRequestHandler(); + handler.init(null); + ArrayList streams = new ArrayList<>(2); + streams.add(new ContentStreamBase.StringStream(doc)); + req.setContentStreams(streams); + handler.handleRequestBody(req, new SolrQueryResponse()); + req.close(); + } +} From 4a4db23362bb0f02723bd3046b23833b89fecbcc Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 15 Mar 2016 09:50:12 -0400 Subject: [PATCH 0140/1113] LUCENE-7108: this test was running the wrong range query --- lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java index f15ba19ed38f..2f3a3a698906 100644 --- a/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java +++ b/lucene/core/src/test/org/apache/lucene/index/Test2BPoints.java @@ -133,7 +133,7 @@ public void test2D() throws Exception { w.forceMerge(1); DirectoryReader r = DirectoryReader.open(w); IndexSearcher s = new IndexSearcher(r); - assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MAX_VALUE}, new long[] {Long.MIN_VALUE, Long.MAX_VALUE}))); + assertEquals(numDocs, s.count(LongPoint.newRangeQuery("long", new long[] {Long.MIN_VALUE, Long.MIN_VALUE}, new long[] {Long.MAX_VALUE, Long.MAX_VALUE}))); assertTrue(r.leaves().get(0).reader().getPointValues().size("long") > Integer.MAX_VALUE); r.close(); w.close(); From cc412ebcaeb4925e3bfe19739582167a3835005f Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 15 Mar 2016 11:17:52 -0400 Subject: [PATCH 0141/1113] LUCENE-7105: Optimize LatLonPoint.newDistanceQuery --- lucene/CHANGES.txt | 2 + .../document/LatLonPointDistanceQuery.java | 88 +++++++++++++++---- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 8004b7f94743..2a01ddb293ae 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -18,6 +18,8 @@ Optimizations * LUCENE-7099: LatLonPoint's newDistanceQuery supports two-phase iteration. (Robert Muir) +* LUCENE-7105: Optimize LatLonPoint's newDistanceQuery. (Robert Muir) + * LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort stack depth before switching to heapsort (Adrien Grand, Mike McCandless) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 3f86f1e5a5d5..e8c7c085c1e7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -41,7 +41,9 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.lucene.util.StringHelper; /** * Distance query for {@link LatLonPoint}. @@ -74,16 +76,41 @@ public LatLonPointDistanceQuery(String field, double latitude, double longitude, @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { GeoRect box = GeoUtils.circleToBBox(longitude, latitude, radiusMeters); - final GeoRect box1; - final GeoRect box2; + // create bounding box(es) for the distance range + // these are pre-encoded with LatLonPoint's encoding + final byte minLat[] = new byte[Integer.BYTES]; + final byte maxLat[] = new byte[Integer.BYTES]; + final byte minLon[] = new byte[Integer.BYTES]; + final byte maxLon[] = new byte[Integer.BYTES]; + // second set of longitude ranges to check (for cross-dateline case) + final byte minLon2[] = new byte[Integer.BYTES]; + + NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.minLat), minLat, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.maxLat), maxLat, 0); // crosses dateline: split if (box.crossesDateline()) { - box1 = new GeoRect(-180.0, box.maxLon, box.minLat, box.maxLat); - box2 = new GeoRect(box.minLon, 180.0, box.minLat, box.maxLat); + // box1 + NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minLon, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + // box2 + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon2, 0); + } else { + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + // disable box2 + NumericUtils.intToSortableBytes(Integer.MAX_VALUE, minLon2, 0); + } + + // compute a maximum partial haversin: unless our box is crazy, we can use this bound + // to reject edge cases faster in matches() + final double minPartialDistance; + if (box.maxLon - longitude < 90 && longitude - box.minLon < 90) { + minPartialDistance = Math.max(LatLonPointDistanceComparator.haversin1(latitude, longitude, latitude, box.maxLon), + LatLonPointDistanceComparator.haversin1(latitude, longitude, box.maxLat, longitude)); + assert LatLonPointDistanceComparator.haversin2(minPartialDistance) >= radiusMeters; } else { - box1 = box; - box2 = null; + minPartialDistance = Double.POSITIVE_INFINITY; } return new ConstantScoreWeight(this) { @@ -128,6 +155,22 @@ public void visit(int docID) { @Override public void visit(int docID, byte[] packedValue) { + // we bounds check individual values, as subtrees may cross, but we are being sent the values anyway: + // this reduces the amount of docvalues fetches (improves approximation) + + if (StringHelper.compare(Integer.BYTES, packedValue, 0, maxLat, 0) > 0 || + StringHelper.compare(Integer.BYTES, packedValue, 0, minLat, 0) < 0) { + // latitude out of bounding box range + return; + } + + if ((StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, maxLon, 0) > 0 || + StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon, 0) < 0) + && StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon2, 0) < 0) { + // longitude out of bounding box range + return; + } + result.add(docID); } @@ -137,18 +180,25 @@ public void visit(int docID, byte[] packedValue) { // 3. recurse naively. @Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - double latMin = LatLonPoint.decodeLatitude(minPackedValue, 0); - double lonMin = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES); - double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); - double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - - if (latMax < box1.minLat || latMin > box1.maxLat) { + if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 || + StringHelper.compare(Integer.BYTES, maxPackedValue, 0, minLat, 0) < 0) { // latitude out of bounding box range return Relation.CELL_OUTSIDE_QUERY; - } else if ((lonMax < box1.minLon || lonMin > box1.maxLon) && (box2 == null || lonMax < box2.minLon)) { + } + + if ((StringHelper.compare(Integer.BYTES, minPackedValue, Integer.BYTES, maxLon, 0) > 0 || + StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon, 0) < 0) + && StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon2, 0) < 0) { // longitude out of bounding box range return Relation.CELL_OUTSIDE_QUERY; - } else if (lonMax - longitude < 90 && longitude - lonMin < 90 && + } + + double latMin = LatLonPoint.decodeLatitude(minPackedValue, 0); + double lonMin = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES); + double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); + double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); + + if (lonMax - longitude < 90 && longitude - lonMin < 90 && GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMin) <= radiusMeters && GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMax) <= radiusMeters && GeoDistanceUtils.haversin(latitude, longitude, latMax, lonMin) <= radiusMeters && @@ -183,7 +233,15 @@ public boolean matches() throws IOException { long encoded = docValues.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - if (GeoDistanceUtils.haversin(latitude, longitude, docLatitude, docLongitude) <= radiusMeters) { + + // first check the partial distance, if its more than that, it can't be <= radiusMeters + double h1 = LatLonPointDistanceComparator.haversin1(latitude, longitude, docLatitude, docLongitude); + if (h1 > minPartialDistance) { + continue; + } + + // fully confirm with part 2: + if (LatLonPointDistanceComparator.haversin2(h1) <= radiusMeters) { return true; } } From 0739f9155b80fdacb90309159523258452b27484 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Tue, 15 Mar 2016 11:04:33 -0700 Subject: [PATCH 0142/1113] SOLR-8849: improve reproducibility in random order of chaosmonkey actions --- .../org/apache/solr/cloud/ChaosMonkey.java | 100 ++++++++++-------- 1 file changed, 56 insertions(+), 44 deletions(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java index 511fdf34b157..93a670ed434f 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java @@ -85,6 +85,13 @@ public class ChaosMonkey { private List deadPool = new ArrayList<>(); private Thread monkeyThread; + + /** + * Our own Random, seeded from LuceneTestCase on init, so that we can produce a consistent sequence + * of random chaos regardless of if/how othe threads access the test randomness in other threads + * @see LuceneTestCase#random() + */ + private final Random chaosRandom; public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader, String collection, Map> shardToJetty, @@ -94,22 +101,22 @@ public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader, this.zkServer = zkServer; this.zkStateReader = zkStateReader; this.collection = collection; + this.chaosRandom = new Random(LuceneTestCase.random().nextLong()); if (!MONKEY_ENABLED) { monkeyLog("The Monkey is Disabled and will not run"); return; } - Random random = LuceneTestCase.random(); if (EXP != null) { expireSessions = EXP; } else { - expireSessions = random.nextBoolean(); + expireSessions = chaosRandom.nextBoolean(); } if (CONN_LOSS != null) { causeConnectionLoss = CONN_LOSS; } else { - causeConnectionLoss = random.nextBoolean(); + causeConnectionLoss = chaosRandom.nextBoolean(); } @@ -326,7 +333,7 @@ private String getRandomSlice() { List sliceKeyList = new ArrayList<>(slices.size()); sliceKeyList.addAll(slices.keySet()); - String sliceName = sliceKeyList.get(LuceneTestCase.random().nextInt(sliceKeyList.size())); + String sliceName = sliceKeyList.get(chaosRandom.nextInt(sliceKeyList.size())); return sliceName; } @@ -365,8 +372,7 @@ public CloudJettyRunner getRandomJetty(String slice, boolean aggressivelyKillLea return null; } - Random random = LuceneTestCase.random(); - int chance = random.nextInt(10); + int chance = chaosRandom.nextInt(10); CloudJettyRunner cjetty; if (chance <= 5 && aggressivelyKillLeaders) { // if killLeader, really aggressively go after leaders @@ -374,7 +380,7 @@ public CloudJettyRunner getRandomJetty(String slice, boolean aggressivelyKillLea } else { // get random shard List jetties = shardToJetty.get(slice); - int index = random.nextInt(jetties.size()); + int index = chaosRandom.nextInt(jetties.size()); cjetty = jetties.get(index); ZkNodeProps leader = null; @@ -457,7 +463,7 @@ public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) monkeyLog("starting"); - if (LuceneTestCase.random().nextBoolean()) { + if (chaosRandom.nextBoolean()) { monkeyLog("Jetty will not commit on close"); DirectUpdateHandler2.commitOnClose = false; } @@ -474,43 +480,9 @@ public void run() { while (!stop) { try { - Random random = LuceneTestCase.random(); - Thread.sleep(random.nextInt(roundPauseUpperLimit)); - if (random.nextBoolean()) { - if (!deadPool.isEmpty()) { - int index = random.nextInt(deadPool.size()); - JettySolrRunner jetty = deadPool.get(index).jetty; - if (jetty.isStopped() && !ChaosMonkey.start(jetty)) { - continue; - } - //System.out.println("started on port:" + jetty.getLocalPort()); - deadPool.remove(index); - starts.incrementAndGet(); - continue; - } - } - - int rnd = random.nextInt(10); + Thread.sleep(chaosRandom.nextInt(roundPauseUpperLimit)); - if (expireSessions && rnd < EXPIRE_PERCENT) { - expireRandomSession(); - } - - if (causeConnectionLoss && rnd < CONLOSS_PERCENT) { - randomConnectionLoss(); - } - - CloudJettyRunner cjetty; - if (random.nextBoolean()) { - cjetty = stopRandomShard(); - } else { - cjetty = killRandomShard(); - } - if (cjetty == null) { - // we cannot kill - } else { - deadPool.add(cjetty); - } + causeSomeChaos(); } catch (InterruptedException e) { // @@ -549,6 +521,46 @@ public void stopTheMonkey() { } } + /** + * causes some randomly selected chaos + */ + public void causeSomeChaos() throws Exception { + if (chaosRandom.nextBoolean()) { + if (!deadPool.isEmpty()) { + int index = chaosRandom.nextInt(deadPool.size()); + JettySolrRunner jetty = deadPool.get(index).jetty; + if (jetty.isStopped() && !ChaosMonkey.start(jetty)) { + return; + } + deadPool.remove(index); + starts.incrementAndGet(); + return; + } + } + + int rnd = chaosRandom.nextInt(10); + + if (expireSessions && rnd < EXPIRE_PERCENT) { + expireRandomSession(); + } + + if (causeConnectionLoss && rnd < CONLOSS_PERCENT) { + randomConnectionLoss(); + } + + CloudJettyRunner cjetty; + if (chaosRandom.nextBoolean()) { + cjetty = stopRandomShard(); + } else { + cjetty = killRandomShard(); + } + if (cjetty == null) { + // we cannot kill + } else { + deadPool.add(cjetty); + } + } + public int getStarts() { return starts.get(); } From 4deb4cd1bac4d18914164fccc3e377eb8f11a117 Mon Sep 17 00:00:00 2001 From: anshum Date: Tue, 15 Mar 2016 10:55:03 -0700 Subject: [PATCH 0143/1113] SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler instead of 500. --- solr/CHANGES.txt | 3 +++ .../solr/handler/loader/JsonLoader.java | 6 +++++- .../apache/solr/handler/JsonLoaderTest.java | 20 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f4e1c44a2872..979b95f2a08e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -437,6 +437,9 @@ Other Changes * SOLR-8799: Improve error message when tuple can't be read by SolrJ JDBC (Kevin Risden, Joel Bernstein) +* SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler + instead of 500. (Jason Gerlowski via Anshum Gupta) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index e99b24359589..ba800ff3fafd 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -50,6 +50,7 @@ import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.util.RecordingJSONParser; import org.noggit.JSONParser; +import org.noggit.JSONParser.ParseException; import org.noggit.ObjectBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +112,10 @@ public void load(SolrQueryRequest req, } this.processUpdate(reader); - } finally { + } catch (ParseException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot parse provided JSON: " + e.getMessage()); + } + finally { IOUtils.closeQuietly(reader); } } diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java index f2316da9f1d0..e27ca1a0639f 100644 --- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -181,6 +181,26 @@ public void testSimpleFormat() throws Exception req.close(); } + @Test + public void testInvalidJsonProducesBadRequestSolrException() throws Exception + { + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + String invalidJsonString = "}{"; + + try(SolrQueryRequest req = req()) { + try { + loader.load(req, rsp, new ContentStreamBase.StringStream(invalidJsonString), p); + fail("Expected invalid JSON to produce a SolrException."); + } catch (SolrException expectedException) { + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, expectedException.code()); + assertTrue(expectedException.getMessage().contains("Cannot parse")); + assertTrue(expectedException.getMessage().contains("JSON")); + } + } + } + public void testSimpleFormatInAdd() throws Exception { String str = "{'add':[{'id':'1'},{'id':'2'}]}".replace('\'', '"'); From e68dc4a330bed0d3cc90167b74b83261ed29fd0a Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 16 Mar 2016 09:25:06 -0400 Subject: [PATCH 0144/1113] LUCENE-7109: LatLonPoint.newPolygonQuery should use two-phase iterator --- lucene/CHANGES.txt | 3 + .../document/LatLonPointInPolygonQuery.java | 65 ++++++++++++++++--- .../TestLatLonPointInPolygonQuery.java | 49 ++++++++++++++ 3 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 2a01ddb293ae..4941dee02de8 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -20,6 +20,9 @@ Optimizations * LUCENE-7105: Optimize LatLonPoint's newDistanceQuery. (Robert Muir) +* LUCENE-7109: LatLonPoint's newPolygonQuery supports two-phase + iteration. (Robert Muir) + * LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort stack depth before switching to heapsort (Adrien Grand, Mike McCandless) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index fb9189f2bb6c..cb988951b94f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -23,15 +23,23 @@ import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SparseFixedBitSet; import org.apache.lucene.spatial.util.GeoRelationUtils; import org.apache.lucene.spatial.util.GeoUtils; @@ -110,9 +118,6 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // I don't use RandomAccessWeight here: it's no good to approximate with "match all docs"; this is an inverted structure and should be // used in the first pass: - // TODO: except that the polygon verify is costly! The approximation should be all docs in all overlapping cells, and matches() should - // then check the polygon - return new ConstantScoreWeight(this) { @Override @@ -130,22 +135,28 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } LatLonPoint.checkCompatible(fieldInfo); + // approximation (postfiltering has not yet been applied) DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc()); + // subset of documents that need no postfiltering, this is purely an optimization + final BitSet preApproved; + // dumb heuristic: if the field is really sparse, use a sparse impl + if (values.getDocCount(field) * 100L < reader.maxDoc()) { + preApproved = new SparseFixedBitSet(reader.maxDoc()); + } else { + preApproved = new FixedBitSet(reader.maxDoc()); + } values.intersect(field, new IntersectVisitor() { @Override public void visit(int docID) { result.add(docID); + preApproved.set(docID); } @Override public void visit(int docID, byte[] packedValue) { - assert packedValue.length == 8; - double lat = LatLonPoint.decodeLatitude(packedValue, 0); - double lon = LatLonPoint.decodeLongitude(packedValue, Integer.BYTES); - if (GeoRelationUtils.pointInPolygon(polyLons, polyLats, lat, lon)) { - result.add(docID); - } + // TODO: range checks + result.add(docID); } @Override @@ -172,7 +183,41 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { } }); - return new ConstantScoreScorer(this, score(), result.build().iterator()); + DocIdSet set = result.build(); + final DocIdSetIterator disi = set.iterator(); + if (disi == null) { + return null; + } + + // return two-phase iterator using docvalues to postfilter candidates + SortedNumericDocValues docValues = DocValues.getSortedNumeric(reader, field); + TwoPhaseIterator iterator = new TwoPhaseIterator(disi) { + @Override + public boolean matches() throws IOException { + int docId = disi.docID(); + if (preApproved.get(docId)) { + return true; + } else { + docValues.setDocument(docId); + int count = docValues.count(); + for (int i = 0; i < count; i++) { + long encoded = docValues.valueAt(i); + double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); + double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); + if (GeoRelationUtils.pointInPolygon(polyLons, polyLats, docLatitude, docLongitude)) { + return true; + } + } + return false; + } + } + + @Override + public float matchCost() { + return 20 * polyLons.length; // TODO: make this fancier, but currently linear with number of vertices + } + }; + return new ConstantScoreScorer(this, score(), iterator); } }; } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java new file mode 100644 index 000000000000..de870277a33c --- /dev/null +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +/** Simple tests for {@link LatLonPoint#newPolygonQuery} */ +public class TestLatLonPointInPolygonQuery extends LuceneTestCase { + + /** test we can search for a polygon */ + public void testBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + document.add(new LatLonPoint("field", 18.313694, -65.227444)); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + assertEquals(1, searcher.count(LatLonPoint.newPolygonQuery("field", + new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }))); + + reader.close(); + writer.close(); + dir.close(); + } +} From 6f124e10017f65c6b0a1753cf9228f065a03db8e Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 16 Mar 2016 09:30:45 -0400 Subject: [PATCH 0145/1113] remove obselete warning: this sort makes bounding boxes for you --- .../src/java/org/apache/lucene/document/LatLonPoint.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index f5541bd9dcf4..7c056e252271 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -336,10 +336,6 @@ public static Query newPolygonQuery(String field, double[] polyLats, double[] po * (missing values sort last). *

    * If a document contains multiple values for the field, the closest distance to the location is used. - *

    - * NOTE: distance sorting might be expensive for many documents. Consider restricting the document - * set with a {@link #newBoxQuery box}, {@link #newDistanceQuery radius} radius, or {@link #newPolygonQuery polygon} - * query for better performance * * @param field field name. cannot be null. * @param latitude latitude at the center: must be within standard +/-90 coordinate bounds. From cb85f2611e164b089c72ffa0c29bb72e6512cd2d Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 16 Mar 2016 15:34:40 +0100 Subject: [PATCH 0146/1113] LUCENE-7106: Add helpers to compute aggregated stats on points. --- .../org/apache/lucene/index/PointValues.java | 90 ++++++++++++++++++- .../apache/lucene/index/TestPointValues.java | 62 +++++++++++-- 2 files changed, 145 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java index 1fb265443028..a4fd3239851e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java @@ -23,10 +23,10 @@ import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.bkd.BKDWriter; /** @@ -86,6 +86,94 @@ public abstract class PointValues { /** Maximum number of dimensions */ public static final int MAX_DIMENSIONS = BKDWriter.MAX_DIMS; + /** Return the cumulated number of points across all leaves of the given + * {@link IndexReader}. + * @see PointValues#size(String) */ + public static long size(IndexReader reader, String field) throws IOException { + long size = 0; + for (LeafReaderContext ctx : reader.leaves()) { + PointValues values = ctx.reader().getPointValues(); + if (values != null) { + size += values.size(field); + } + } + return size; + } + + /** Return the cumulated number of docs that have points across all leaves + * of the given {@link IndexReader}. + * @see PointValues#getDocCount(String) */ + public static int getDocCount(IndexReader reader, String field) throws IOException { + int count = 0; + for (LeafReaderContext ctx : reader.leaves()) { + PointValues values = ctx.reader().getPointValues(); + if (values != null) { + count += values.getDocCount(field); + } + } + return count; + } + + /** Return the minimum packed values across all leaves of the given + * {@link IndexReader}. + * @see PointValues#getMinPackedValue(String) */ + public static byte[] getMinPackedValue(IndexReader reader, String field) throws IOException { + byte[] minValue = null; + for (LeafReaderContext ctx : reader.leaves()) { + PointValues values = ctx.reader().getPointValues(); + if (values == null) { + continue; + } + byte[] leafMinValue = values.getMinPackedValue(field); + if (leafMinValue == null) { + continue; + } + if (minValue == null) { + minValue = leafMinValue.clone(); + } else { + final int numDimensions = values.getNumDimensions(field); + final int numBytesPerDimension = values.getBytesPerDimension(field); + for (int i = 0; i < numDimensions; ++i) { + int offset = i * numBytesPerDimension; + if (StringHelper.compare(numBytesPerDimension, leafMinValue, offset, minValue, offset) < 0) { + System.arraycopy(leafMinValue, offset, minValue, offset, numBytesPerDimension); + } + } + } + } + return minValue; + } + + /** Return the maximum packed values across all leaves of the given + * {@link IndexReader}. + * @see PointValues#getMaxPackedValue(String) */ + public static byte[] getMaxPackedValue(IndexReader reader, String field) throws IOException { + byte[] maxValue = null; + for (LeafReaderContext ctx : reader.leaves()) { + PointValues values = ctx.reader().getPointValues(); + if (values == null) { + continue; + } + byte[] leafMaxValue = values.getMaxPackedValue(field); + if (leafMaxValue == null) { + continue; + } + if (maxValue == null) { + maxValue = leafMaxValue.clone(); + } else { + final int numDimensions = values.getNumDimensions(field); + final int numBytesPerDimension = values.getBytesPerDimension(field); + for (int i = 0; i < numDimensions; ++i) { + int offset = i * numBytesPerDimension; + if (StringHelper.compare(numBytesPerDimension, leafMaxValue, offset, maxValue, offset) > 0) { + System.arraycopy(leafMaxValue, offset, maxValue, offset, numBytesPerDimension); + } + } + } + } + return maxValue; + } + /** Default constructor */ protected PointValues() { } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 49cbc2a6d7bb..c7ca2dcb651f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -22,12 +22,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PointsReader; -import org.apache.lucene.codecs.PointsWriter; -import org.apache.lucene.codecs.lucene60.Lucene60PointsReader; -import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter; import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; @@ -657,4 +651,60 @@ public void testCheckIndexIncludesPoints() throws Exception { assertTrue(output.toString(IOUtils.UTF_8).contains("test: points...")); dir.close(); } + + public void testMergedStats() throws IOException { + final int iters = atLeast(3); + for (int iter = 0; iter < iters; ++iter) { + doTestMergedStats(); + } + } + + private static byte[][] randomBinaryValue(int numDims, int numBytesPerDim) { + byte[][] bytes = new byte[numDims][]; + for (int i = 0; i < numDims; ++i) { + bytes[i] = new byte[numBytesPerDim]; + random().nextBytes(bytes[i]); + } + return bytes; + } + + private void doTestMergedStats() throws IOException { + final int numDims = TestUtil.nextInt(random(), 1, 8); + final int numBytesPerDim = TestUtil.nextInt(random(), 1, 16); + Directory dir = new RAMDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + final int numDocs = TestUtil.nextInt(random(), 10, 20); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final int numPoints = random().nextInt(3); + for (int j = 0; j < numPoints; ++j) { + doc.add(new BinaryPoint("field", randomBinaryValue(numDims, numBytesPerDim))); + } + w.addDocument(doc); + if (random().nextBoolean()) { + DirectoryReader.open(w).close(); + } + } + + final IndexReader reader1 = DirectoryReader.open(w); + w.forceMerge(1); + final IndexReader reader2 = DirectoryReader.open(w); + final PointValues expected = getOnlyLeafReader(reader2).getPointValues(); + if (expected == null) { + assertNull(PointValues.getMinPackedValue(reader1, "field")); + assertNull(PointValues.getMaxPackedValue(reader1, "field")); + assertEquals(0, PointValues.getDocCount(reader1, "field")); + assertEquals(0, PointValues.size(reader1, "field")); + } else { + assertArrayEquals( + expected.getMinPackedValue("field"), + PointValues.getMinPackedValue(reader1, "field")); + assertArrayEquals( + expected.getMaxPackedValue("field"), + PointValues.getMaxPackedValue(reader1, "field")); + assertEquals(expected.getDocCount("field"), PointValues.getDocCount(reader1, "field")); + assertEquals(expected.size("field"), PointValues.size(reader1, "field")); + } + IOUtils.close(w, reader1, reader2, dir); + } } From 5731331be1f5fcef829950fcfa9edcb3632babae Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 16 Mar 2016 09:49:46 -0700 Subject: [PATCH 0147/1113] SOLR-8814: Support GeoJSON response format --- solr/CHANGES.txt | 12 + .../java/org/apache/solr/core/SolrCore.java | 3 +- .../solr/response/GeoJSONResponseWriter.java | 345 ++++++++++++++++++ .../solr/response/JSONResponseWriter.java | 2 +- .../transform/GeoTransformerFactory.java | 224 ++++++++++++ .../transform/TransformerFactory.java | 1 + .../response/transform/WriteableGeoJSON.java | 55 +++ .../solr/schema/AbstractSpatialFieldType.java | 7 + .../solr/collection1/conf/schema-spatial.xml | 1 + .../response/TestGeoJSONResponseWriter.java | 279 ++++++++++++++ 10 files changed, 927 insertions(+), 2 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java create mode 100644 solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java create mode 100644 solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java create mode 100644 solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 979b95f2a08e..6adee44b5613 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -31,6 +31,18 @@ Detailed Change List New Features ---------------------- +* SOLR-8814: Support GeoJSON response writer and general spatial formatting. Adding + &wt=geojson&geojson.field= + Will return a FeatureCollection for each SolrDocumentList and a Feature with the + requested geometry for each SolrDocument. The requested geometry field needs + to either extend AbstractSpatialFieldType or store a GeoJSON string. This also adds + a [geo] DocumentTransformer that can return the Shape in a variety of formats: + &fl=[geo f= w=(GeoJSON|WKT|POLY)] + The default format is GeoJSON. For information on the supported formats, see: + https://github.com/locationtech/spatial4j/blob/master/FORMATS.md + To return the FeatureCollection as the root element, add '&omitHeader=true" (ryan) + + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 7a65a72decbb..c5e54d21a0da 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -2123,10 +2123,11 @@ public PluginBag getResponseWriters() { private final PluginBag responseWriters = new PluginBag<>(QueryResponseWriter.class, this); public static final Map DEFAULT_RESPONSE_WRITERS ; static{ - HashMap m= new HashMap<>(14, 1); + HashMap m= new HashMap<>(15, 1); m.put("xml", new XMLResponseWriter()); m.put("standard", m.get("xml")); m.put(CommonParams.JSON, new JSONResponseWriter()); + m.put("geojson", new GeoJSONResponseWriter()); m.put("python", new PythonResponseWriter()); m.put("php", new PHPResponseWriter()); m.put("phps", new PHPSerializedResponseWriter()); diff --git a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java new file mode 100644 index 000000000000..896be92f8f65 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response; + +import java.io.IOException; +import java.io.Writer; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.index.IndexableField; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.transform.GeoTransformerFactory; +import org.apache.solr.response.transform.WriteableGeoJSON; +import org.apache.solr.schema.AbstractSpatialFieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.ReturnFields; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.io.ShapeWriter; +import org.locationtech.spatial4j.io.SupportedFormats; +import org.locationtech.spatial4j.shape.Shape; + +/** + * Extend the standard JSONResponseWriter to support GeoJSON. This writes + * a {@link SolrDocumentList} with a 'FeatureCollection', following the + * specification in geojson.org + */ +public class GeoJSONResponseWriter extends JSONResponseWriter { + + public static final String FIELD = "geojson.field"; + + @Override + public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + + String geofield = req.getParams().get(FIELD, null); + if(geofield==null || geofield.length()==0) { + throw new SolrException(ErrorCode.BAD_REQUEST, "GeoJSON. Missing parameter: '"+FIELD+"'"); + } + + SchemaField sf = req.getSchema().getFieldOrNull(geofield); + if(sf==null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "GeoJSON. Unknown field: '"+FIELD+"'="+geofield); + } + + SupportedFormats formats = null; + if(sf.getType() instanceof AbstractSpatialFieldType) { + SpatialContext ctx = ((AbstractSpatialFieldType)sf.getType()).getSpatialContext(); + formats = ctx.getFormats(); + } + + JSONWriter w = new GeoJSONWriter(writer, req, rsp, + geofield, + formats); + + try { + w.writeResponse(); + } finally { + w.close(); + } + } +} + +class GeoJSONWriter extends JSONWriter { + + final SupportedFormats formats; + final ShapeWriter geowriter; + final String geofield; + + public GeoJSONWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp, + String geofield, SupportedFormats formats) { + super(writer, req, rsp); + this.geofield = geofield; + this.formats = formats; + if(formats==null) { + this.geowriter = null; + } + else { + this.geowriter = formats.getGeoJsonWriter(); + } + } + + @Override + public void writeResponse() throws IOException { + if(req.getParams().getBool(CommonParams.OMIT_HEADER, false)) { + if(wrapperFunction!=null) { + writer.write(wrapperFunction + "("); + } + rsp.removeResponseHeader(); + + NamedList vals = rsp.getValues(); + Object response = vals.remove("response"); + if(vals.size()==0) { + writeVal(null, response); + } + else { + throw new SolrException(ErrorCode.BAD_REQUEST, + "GeoJSON with "+CommonParams.OMIT_HEADER + + " can not return more than a result set"); + } + + if(wrapperFunction!=null) { + writer.write(')'); + } + writer.write('\n'); // ending with a newline looks much better from the command line + } + else { + super.writeResponse(); + } + } + + @Override + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException { + if( idx > 0 ) { + writeArraySeparator(); + } + + indent(); + writeMapOpener(-1); + incLevel(); + + writeKey("type", false); + writeVal(null, "Feature"); + + Object val = doc.getFieldValue(geofield); + if(val != null) { + writeFeatureGeometry(val); + } + + boolean first=true; + for (String fname : doc.getFieldNames()) { + if (fname.equals(geofield) || ((returnFields!= null && !returnFields.wantsField(fname)))) { + continue; + } + writeMapSeparator(); + if (first) { + indent(); + writeKey("properties", false); + writeMapOpener(-1); + incLevel(); + + first=false; + } + + indent(); + writeKey(fname, true); + val = doc.getFieldValue(fname); + + // SolrDocument will now have multiValued fields represented as a Collection, + // even if only a single value is returned for this document. + if (val instanceof List) { + // shortcut this common case instead of going through writeVal again + writeArray(name,((Iterable)val).iterator()); + } else { + writeVal(fname, val); + } + } + + // GeoJSON does not really support nested FeatureCollections + if(doc.hasChildDocuments()) { + if(first == false) { + writeMapSeparator(); + indent(); + } + writeKey("_childDocuments_", true); + writeArrayOpener(doc.getChildDocumentCount()); + List childDocs = doc.getChildDocuments(); + for(int i=0; i + *
  • GeoJSON
    • + *
  • WKT
    • + *
  • Polyshape
    • + * + * For more information see: spatial4j/FORMATS.md + * + * The shape is either read from a stored field, or a ValueSource. + * + * This transformer is useful when: + *
      + *
    • You want to return a format different than the stored encoding (WKT vs GeoJSON)
      • + *
    • The {@link Shape} is stored in a {@link ValueSource}, not a stored field
      • + *
    • the value is not stored in a format the output understands (ie, raw GeoJSON)
      • + *
    + * + */ +public class GeoTransformerFactory extends TransformerFactory +{ + @Override + public DocTransformer create(String display, SolrParams params, SolrQueryRequest req) { + + String fname = params.get("f", display); + if(fname.startsWith("[") && fname.endsWith("]")) { + fname = display.substring(1,display.length()-1); + } + SchemaField sf = req.getSchema().getFieldOrNull(fname); + if(sf==null) { + throw new SolrException(ErrorCode.BAD_REQUEST, + this.getClass().getSimpleName() +" using unknown field: "+fname); + } + if(!(sf.getType() instanceof AbstractSpatialFieldType)) { + throw new SolrException(ErrorCode.BAD_REQUEST, + "GeoTransformer requested non-spatial field: "+fname + " ("+sf.getType().getClass().getSimpleName()+")"); + } + + final GeoFieldUpdater updater = new GeoFieldUpdater(); + updater.field = fname; + updater.display = display; + updater.display_error = display+"_error"; + + ValueSource shapes = null; + AbstractSpatialFieldType sdv = (AbstractSpatialFieldType)sf.getType(); + SpatialStrategy strategy = sdv.getStrategy(fname); + if(strategy instanceof CompositeSpatialStrategy) { + shapes = ((CompositeSpatialStrategy)strategy) + .getGeometryStrategy().makeShapeValueSource(); + } + else if(strategy instanceof SerializedDVStrategy) { + shapes = ((SerializedDVStrategy)strategy) + .makeShapeValueSource(); + } + + + String writerName = params.get("w", "GeoJSON"); + updater.formats = strategy.getSpatialContext().getFormats(); + updater.writer = updater.formats.getWriter(writerName); + if(updater.writer==null) { + StringBuilder str = new StringBuilder(); + str.append( "Unknown Spatial Writer: " ).append(writerName); + str.append(" ["); + for(ShapeWriter w : updater.formats.getWriters()) { + str.append(w.getFormatName()).append(' '); + } + str.append("]"); + throw new SolrException(ErrorCode.BAD_REQUEST, str.toString()); + } + + QueryResponseWriter qw = req.getCore().getQueryResponseWriter(req); + updater.isJSON = + (qw.getClass() == JSONResponseWriter.class) && + (updater.writer instanceof GeoJSONWriter); + + + // Using ValueSource + if(shapes!=null) { + // we don't really need the qparser... just so we can reuse valueSource + QParser parser = new QParser(null,null,params, req) { + @Override + public Query parse() throws SyntaxError { + return new MatchAllDocsQuery(); + } + }; + + return new ValueSourceAugmenter(display, parser, shapes) { + @Override + protected void setValue(SolrDocument doc, Object val) { + updater.setValue(doc, val); + } + }; + } + + // Using the raw stored values + return new DocTransformer() { + + @Override + public void transform(SolrDocument doc, int docid, float score) throws IOException { + Object val = doc.remove(updater.field); + if(val!=null) { + updater.setValue(doc, val); + } + } + + @Override + public String getName() { + return updater.display; + } + + @Override + public String[] getExtraRequestFields() { + return new String[] {updater.field}; + } + }; + } +} + +class GeoFieldUpdater { + String field; + String display; + String display_error; + + boolean isJSON; + ShapeWriter writer; + SupportedFormats formats; + + void addShape(SolrDocument doc, Shape shape) { + if(isJSON) { + doc.addField(display, new WriteableGeoJSON(shape, writer)); + } + else { + doc.addField(display, writer.toString(shape)); + } + } + + void setValue(SolrDocument doc, Object val) { + doc.remove(display); + if(val != null) { + if(val instanceof Iterable) { + Iterator iter = ((Iterable)val).iterator(); + while(iter.hasNext()) { + addValue(doc, iter.next()); + } + } + else { + addValue(doc, val); + } + } + } + + void addValue(SolrDocument doc, Object val) { + if(val == null) { + return; + } + + if(val instanceof Shape) { + addShape(doc, (Shape)val); + } + // Don't explode on 'InvalidShpae' + else if( val instanceof Exception) { + doc.setField( display_error, ((Exception)val).toString() ); + } + else { + // Use the stored value + if(val instanceof IndexableField) { + val = ((IndexableField)val).stringValue(); + } + try { + addShape(doc, formats.read(val.toString())); + } + catch(Exception ex) { + doc.setField( display_error, ex.toString() ); + } + } + } +} + diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java index a600adfcc4ac..6e7a3dd986e7 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java @@ -49,5 +49,6 @@ public void init(NamedList args) { defaultFactories.put( "child", new ChildDocTransformerFactory() ); defaultFactories.put( "json", new RawValueTransformerFactory("json") ); defaultFactories.put( "xml", new RawValueTransformerFactory("xml") ); + defaultFactories.put( "geo", new GeoTransformerFactory() ); } } diff --git a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java new file mode 100644 index 000000000000..40acebf2a7bc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import java.io.IOException; + +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.response.WriteableValue; +import org.locationtech.spatial4j.io.ShapeWriter; +import org.locationtech.spatial4j.shape.Shape; + +/** + * This will let the writer add values to the response directly + */ +public class WriteableGeoJSON extends WriteableValue { + + public final Shape shape; + public final ShapeWriter jsonWriter; + + public WriteableGeoJSON(Shape shape, ShapeWriter jsonWriter) { + this.shape = shape; + this.jsonWriter = jsonWriter; + } + + @Override + public Object resolve(Object o, JavaBinCodec codec) throws IOException { + codec.writeStr(jsonWriter.toString(shape)); + return null; // this means we wrote it + } + + @Override + public void write(String name, TextResponseWriter writer) throws IOException { + jsonWriter.write(writer.getWriter(), shape); + } + + @Override + public String toString() { + return jsonWriter.toString(shape); + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index 222f0b83eb69..e5fd8c63d957 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -390,6 +390,13 @@ public T call() throws Exception { } } + /** + * @return The Spatial Context for this field type + */ + public SpatialContext getSpatialContext() { + return ctx; + } + @Override public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { writer.writeStr(name, f.stringValue(), true); diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml index 2c1ca1f943bf..15837f348ae1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml @@ -70,6 +70,7 @@ + diff --git a/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java new file mode 100644 index 000000000000..191136bc2a36 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/response/TestGeoJSONResponseWriter.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response; + +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Map; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.BeforeClass; +import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.io.SupportedFormats; +import org.locationtech.spatial4j.shape.Shape; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class TestGeoJSONResponseWriter extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + final ObjectMapper jsonmapper = new ObjectMapper(); + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-basic.xml","schema-spatial.xml"); + createIndex(); + } + + public static void createIndex() { + + +// +// +// +// + + // multiple valued field + assertU(adoc("id","H.A", "srpt_geohash","POINT( 1 2 )")); + assertU(adoc("id","H.B", "srpt_geohash","POINT( 1 2 )", + "srpt_geohash","POINT( 3 4 )")); + assertU(adoc("id","H.C", "srpt_geohash","LINESTRING (30 10, 10 30, 40 40)")); + + assertU(adoc("id","Q.A", "srpt_quad","POINT( 1 2 )")); + assertU(adoc("id","Q.B", "srpt_quad","POINT( 1 2 )", + "srpt_quad","POINT( 3 4 )")); + assertU(adoc("id","Q.C", "srpt_quad","LINESTRING (30 10, 10 30, 40 40)")); + + assertU(adoc("id","P.A", "srpt_packedquad","POINT( 1 2 )")); + assertU(adoc("id","P.B", "srpt_packedquad","POINT( 1 2 )", + "srpt_packedquad","POINT( 3 4 )")); + assertU(adoc("id","P.C", "srpt_packedquad","LINESTRING (30 10, 10 30, 40 40)")); + + + // single valued field + assertU(adoc("id","R.A", "srptgeom","POINT( 1 2 )")); + + // non-spatial field + assertU(adoc("id","S.X", "str_shape","POINT( 1 2 )")); + assertU(adoc("id","S.A", "str_shape","{\"type\":\"Point\",\"coordinates\":[1,2]}")); + + + assertU(commit()); + } + + protected Map readJSON(String json) { + try { + return jsonmapper.readValue(json, Map.class); + } + catch(Exception ex) { + log.warn("Unable to read GeoJSON From: {}", json); + log.warn("Error", ex); + fail("Unable to parse JSON GeoJSON Response"); + } + return null; + } + + protected Map getFirstFeatureGeometry(Map json) + { + Map rsp = (Map)json.get("response"); + assertEquals("FeatureCollection", rsp.get("type")); + List vals = (List)rsp.get("features"); + assertEquals(1, vals.size()); + Map feature = (Map)vals.get(0); + assertEquals("Feature", feature.get("type")); + return (Map)feature.get("geometry"); + } + + @Test + public void testRequestExceptions() throws Exception { + + // Make sure we select the field + try { + h.query(req( + "q","*:*", + "wt","geojson", + "fl","*")); + fail("should Require a parameter to select the field"); + } + catch(SolrException ex) {} + + + // non-spatial fields *must* be stored as JSON + try { + h.query(req( + "q","id:S.X", + "wt","geojson", + "fl","*", + "geojson.field", "str_shape")); + fail("should complain about bad shape config"); + } + catch(SolrException ex) {} + + } + + @Test + public void testGeoJSONAtRoot() throws Exception { + + // Try reading the whole resposne + String json = h.query(req( + "q","*:*", + "wt","geojson", + "rows","2", + "fl","*", + "geojson.field", "stqpt_geohash", + "indent","true")); + + // Check that we have a normal solr response with 'responseHeader' and 'response' + Map rsp = readJSON(json); + assertNotNull(rsp.get("responseHeader")); + assertNotNull(rsp.get("response")); + + json = h.query(req( + "q","*:*", + "wt","geojson", + "rows","2", + "fl","*", + "omitHeader", "true", + "geojson.field", "stqpt_geohash", + "indent","true")); + + // Check that we have a normal solr response with 'responseHeader' and 'response' + rsp = readJSON(json); + assertNull(rsp.get("responseHeader")); + assertNull(rsp.get("response")); + assertEquals("FeatureCollection", rsp.get("type")); + assertNotNull(rsp.get("features")); + } + + @Test + public void testGeoJSONOutput() throws Exception { + + // Try reading the whole resposne + readJSON(h.query(req( + "q","*:*", + "wt","geojson", + "fl","*", + "geojson.field", "stqpt_geohash", + "indent","true"))); + + // Multivalued Valued Point + Map json = readJSON(h.query(req( + "q","id:H.B", + "wt","geojson", + "fl","*", + "geojson.field", "srpt_geohash", + "indent","true"))); + + Map geo = getFirstFeatureGeometry(json); + assertEquals( // NOTE: not actual JSON, it is Map.toString()! + "{type=GeometryCollection, geometries=[" + + "{type=Point, coordinates=[1, 2]}, " + + "{type=Point, coordinates=[3, 4]}]}", ""+geo); + + + // Check the same value encoded on different field types + String[][] check = new String[][] { + { "id:H.A", "srpt_geohash" }, + { "id:Q.A", "srpt_quad" }, + { "id:P.A", "srpt_packedquad" }, + { "id:R.A", "srptgeom" }, + { "id:S.A", "str_shape" }, + }; + + for(String[] args : check) { + json = readJSON(h.query(req( + "q",args[0], + "wt","geojson", + "fl","*", + "geojson.field", args[1]))); + + geo = getFirstFeatureGeometry(json); + assertEquals( + "Error reading point from: "+args[1] + " ("+args[0]+")", + // NOTE: not actual JSON, it is Map.toString()! + "{type=Point, coordinates=[1, 2]}", ""+geo); + } + } + + protected Map readFirstDoc(String json) + { + List docs = (List)((Map)readJSON(json).get("response")).get("docs"); + return (Map)docs.get(0); + } + + public static String normalizeMapToJSON(String val) { + val = val.replace("\"", ""); // remove quotes + val = val.replace(':', '='); + val = val.replace(", ", ","); + return val; + } + + @Test + public void testTransformToAllFormats() throws Exception { + + String wkt = "POINT( 1 2 )"; + SupportedFormats fmts = SpatialContext.GEO.getFormats(); + Shape shape = fmts.read(wkt); + + String[] check = new String[] { + "srpt_geohash", + "srpt_geohash", + "srpt_quad", + "srpt_packedquad", + "srptgeom", + // "str_shape", // NEEDS TO BE A SpatialField! + }; + + String[] checkFormats = new String[] { + "GeoJSON", + "WKT", + "POLY" + }; + + for(String field : check) { + // Add a document with the given field + assertU(adoc("id","test", + field, wkt)); + assertU(commit()); + + + for(String fmt : checkFormats) { + String json = h.query(req( + "q","id:test", + "wt","json", + "indent", "true", + "fl","xxx:[geo f="+field+" w="+fmt+"]" + )); + + Map doc = readFirstDoc(json); + Object v = doc.get("xxx"); + String expect = fmts.getWriter(fmt).toString(shape); + + if(!(v instanceof String)) { + v = normalizeMapToJSON(v.toString()); + expect = normalizeMapToJSON(expect); + } + + assertEquals("Bad result: "+field+"/"+fmt, expect, v.toString()); + } + } + } +} From 7687667b5ff7867249762d104707a91834d30ce3 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 16 Mar 2016 12:52:00 -0700 Subject: [PATCH 0148/1113] SOLR-8859: read/write Shapes to String --- solr/CHANGES.txt | 2 + .../solr/schema/AbstractSpatialFieldType.java | 80 +++++++++++++------ .../apache/solr/schema/DateRangeField.java | 4 +- .../solr/schema/SpatialRPTFieldTypeTest.java | 38 ++++++++- 4 files changed, 95 insertions(+), 29 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6adee44b5613..b89aae1a6935 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -42,6 +42,8 @@ New Features https://github.com/locationtech/spatial4j/blob/master/FORMATS.md To return the FeatureCollection as the root element, add '&omitHeader=true" (ryan) +* SOLR-8859: AbstractSpatialFieldType will now convert Shapes to/from Strings + using the SpatialContext. (ryan) Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index e5fd8c63d957..7addb20254d7 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -28,7 +28,6 @@ import java.util.TreeSet; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; - import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; @@ -44,6 +43,7 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.SolrParams; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -60,6 +60,9 @@ import org.locationtech.spatial4j.context.SpatialContext; import org.locationtech.spatial4j.context.SpatialContextFactory; import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.io.ShapeReader; +import org.locationtech.spatial4j.io.ShapeWriter; +import org.locationtech.spatial4j.io.SupportedFormats; import org.locationtech.spatial4j.shape.Point; import org.locationtech.spatial4j.shape.Rectangle; import org.locationtech.spatial4j.shape.Shape; @@ -83,11 +86,17 @@ public abstract class AbstractSpatialFieldType extend public static final String RECIP_DISTANCE = "recipDistance"; public static final String NONE = "none"; + /** Optional param to pick the string conversion */ + public static final String FORMAT = "format"; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected SpatialContext ctx; protected SpatialArgsParser argsParser; + protected ShapeWriter shapeWriter; + protected ShapeReader shapeReader; + private final Cache fieldStrategyCache = CacheBuilder.newBuilder().build(); protected DistanceUnits distanceUnits; @@ -130,6 +139,25 @@ protected void init(IndexSchema schema, Map args) { " on field types with class "+getClass().getSimpleName()); } + final SupportedFormats fmts = ctx.getFormats(); + final String format = args.remove(FORMAT); + if (format != null) { + shapeWriter = fmts.getWriter(format); + shapeReader = fmts.getReader(format); + if(shapeWriter==null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unknown Shape Format: "+ format); + } + if(shapeReader==null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unknown Shape Format: "+ format); + } + } + else { + // Otherwise, pick the first supported reader/writer + shapeWriter = fmts.getWriters().get(0); + shapeReader = fmts.getReaders().get(0); + } argsParser = newSpatialArgsParser(); } @@ -203,38 +231,38 @@ protected String getStoredValue(Shape shape, String shapeStr) { return (shapeStr == null) ? shapeToString(shape) : shapeStr; } - protected Shape parseShape(String str) { + /** Create a {@link Shape} from the input string */ + public Shape parseShape(String str) { if (str.length() == 0) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty string shape"); - if (Character.isLetter(str.charAt(0))) {//WKT starts with a letter - try { - return ctx.readShapeFromWkt(str); - } catch (Exception e) { - String message = e.getMessage(); - if (!message.contains(str)) - message = "Couldn't parse shape '" + str + "' because: " + message; - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message, e); - } - } else { - return SpatialUtils.parsePointSolrException(str, ctx); + + Shape shape = null; + if(shapeReader!=null) { + shape = shapeReader.readIfSupported(str); + } + + if(shape==null) { + // Try all supported formats + shape = ctx.getFormats().read(str); + } + + if(shape==null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to parse shape from: "+str); } + return shape; } /** - * Returns a String version of a shape to be used for the stored value. This method in Solr is only called if for some - * reason a Shape object is passed to the field type (perhaps via a custom UpdateRequestProcessor), - * *and* the field is marked as stored. The default implementation throws an exception. - *

    - * Spatial4j 0.4 is probably the last release to support SpatialContext.toString(shape) but it's deprecated with no - * planned replacement. Shapes do have a toString() method but they are generally internal/diagnostic and not - * standard WKT. - * The solution is subclassing and calling ctx.toString(shape) or directly using LegacyShapeReadWriterFormat or - * passing in some sort of custom wrapped shape that holds a reference to a String or can generate it. + * Returns a String version of a shape to be used for the stored value. + * + * The format can be selected using the initParam format={WKT|GeoJSON} */ - protected String shapeToString(Shape shape) { -// return ctx.toString(shape); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Getting a String from a Shape is no longer possible. See javadocs for commentary."); + public String shapeToString(Shape shape) { + if(shapeWriter!=null) { + return shapeWriter.toString(shape); + } + // This will only happen if the context does not have any writers + throw new SolrException(ErrorCode.SERVER_ERROR, "ShapeWriter not configured"); } /** Called from {@link #getStrategy(String)} upon first use by fieldName. } */ diff --git a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java index 95b441a9ed1d..faf049bb5d5c 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java @@ -82,7 +82,7 @@ protected String getStoredValue(Shape shape, String shapeStr) { } @Override - protected NRShape parseShape(String str) { + public NRShape parseShape(String str) { if (str.contains(" TO ")) { //TODO parsing range syntax doesn't support DateMath on either side or exclusive/inclusive try { @@ -121,7 +121,7 @@ public Date parseMath(Date now, String rawval) { } @Override - protected String shapeToString(Shape shape) { + public String shapeToString(Shape shape) { if (shape instanceof UnitNRShape) { UnitNRShape unitShape = (UnitNRShape) shape; if (unitShape.getLevel() == tree.getMaxLevels()) { diff --git a/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java b/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java index 479a7fe62a0f..f34183252adf 100644 --- a/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java +++ b/solr/core/src/test/org/apache/solr/schema/SpatialRPTFieldTypeTest.java @@ -24,6 +24,7 @@ import org.apache.solr.core.AbstractBadConfigTestBase; import org.junit.After; import org.junit.Before; +import org.locationtech.spatial4j.shape.Shape; public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase { @@ -201,7 +202,35 @@ public void testGeoDistanceFunctionWithMiles() throws Exception { ); } - private void setupRPTField(String distanceUnits, String geo) throws Exception { + public void testShapeToFromStringWKT() throws Exception { + // Check WKT + setupRPTField("miles", "true", "WKT"); + + AbstractSpatialFieldType ftype = (AbstractSpatialFieldType) + h.getCore().getLatestSchema().getField("geo").getType(); + + String wkt = "POINT (1 2)"; + Shape shape = ftype.parseShape(wkt); + String out = ftype.shapeToString(shape); + + assertEquals(wkt, out); + } + + public void testShapeToFromStringGeoJSON() throws Exception { + // Check WKT + setupRPTField("miles", "true", "GeoJSON"); + + AbstractSpatialFieldType ftype = (AbstractSpatialFieldType) + h.getCore().getLatestSchema().getField("geo").getType(); + + String json = "{\"type\":\"Point\",\"coordinates\":[1,2]}"; + Shape shape = ftype.parseShape(json); + String out = ftype.shapeToString(shape); + + assertEquals(json, out); + } + + private void setupRPTField(String distanceUnits, String geo, String format) throws Exception { deleteCore(); File managedSchemaFile = new File(tmpConfDir, "managed-schema"); Files.delete(managedSchemaFile.toPath()); // Delete managed-schema so it won't block parsing a new schema @@ -220,6 +249,9 @@ private void setupRPTField(String distanceUnits, String geo) throws Exception { rptMap.put("distanceUnits", distanceUnits); if(geo!=null) rptMap.put("geo", geo); + if(format!=null) { + rptMap.put("format", format); + } rptFieldType.init(oldSchema, rptMap); rptFieldType.setTypeName("location_rpt"); SchemaField newField = new SchemaField("geo", rptFieldType, SchemaField.STORED | SchemaField.INDEXED, null); @@ -229,4 +261,8 @@ private void setupRPTField(String distanceUnits, String geo) throws Exception { assertU(delQ("*:*")); } + + private void setupRPTField(String distanceUnits, String geo) throws Exception { + setupRPTField(distanceUnits, geo, null); + } } From 14752476f445436944618a6f1dde9bd787a1f3c9 Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 16 Mar 2016 18:51:50 -0400 Subject: [PATCH 0149/1113] SOLR-8740: use docValues for non-text fields in schema templates --- solr/CHANGES.txt | 9 ++++ .../basic_configs/conf/managed-schema | 32 +++++------ .../conf/managed-schema | 53 +++++++++---------- .../conf/managed-schema | 43 ++++++--------- 4 files changed, 66 insertions(+), 71 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b89aae1a6935..b5b1a04891fb 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -127,6 +127,12 @@ Upgrading from Solr 5.x * SOLR-8736: The deprecated GET methods for schema are now accessible through the bulk API. The output has less details and is not backward compatible. +* In the past, Solr guaranteed that retrieval of multi-valued fields would preserve the order of values. + Because values may now be retrieved from column-stored fields (docValues="true"), in conjunction with the + fact that docValues do not currently preserve order, means that users should set useDocValuesAsStored="false" + to prevent future optizations from using the column-stored values over the row-stored values when + fields have both stored="true" and docValues="true". + Detailed Change List ---------------------- @@ -454,6 +460,9 @@ Other Changes * SOLR-8836: Return 400, and a SolrException when an invalid json is provided to the update handler instead of 500. (Jason Gerlowski via Anshum Gupta) +* SOLR-8740: docValues are now enabled by default for most non-text (string, date, and numeric) fields + in the schema templates. (yonik) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/server/solr/configsets/basic_configs/conf/managed-schema b/solr/server/solr/configsets/basic_configs/conf/managed-schema index 2599a28dffd7..3ce6b73e7ffd 100644 --- a/solr/server/solr/configsets/basic_configs/conf/managed-schema +++ b/solr/server/solr/configsets/basic_configs/conf/managed-schema @@ -94,12 +94,12 @@ - + - + - + @@ -187,7 +187,7 @@ It supports doc values but in that case the field needs to be single-valued and either required or have a default value. --> - + @@ -214,10 +214,10 @@ These fields support doc values, but they require the field to be single-valued and either be required or have a default value. --> - - - - + + + + - - - - + + + + - + - + @@ -487,7 +487,7 @@ - + - + - - + + @@ -146,7 +146,7 @@ - + @@ -205,8 +205,8 @@ It supports doc values but in that case the field needs to be single-valued and either required or have a default value. --> - - + + @@ -235,15 +235,15 @@ These fields support doc values, but they require the field to be single-valued and either be required or have a default value. --> - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - + - + @@ -532,7 +532,7 @@ - + -false diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema index ddba48351551..1000d9a36235 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema @@ -112,12 +112,12 @@ - + - + - + - - - + @@ -351,10 +338,10 @@ These fields support doc values, but they require the field to be single-valued and either be required or have a default value. --> - - - - + + + + - - - - + + + + - + - + @@ -723,7 +710,7 @@ relevancy. --> - + - + + + + diff --git a/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java b/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java index 6c9487a53609..9a43b423f92e 100644 --- a/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java +++ b/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java @@ -42,7 +42,8 @@ public class SolrCellMorphlineTest extends AbstractSolrMorphlineTestBase { @BeforeClass public static void beforeClass2() { assumeFalse("FIXME: Morphlines currently has issues with Windows paths", Constants.WINDOWS); - assumeFalse("This test fails with Java 9 (https://issues.apache.org/jira/browse/PDFBOX-3155)", Constants.JRE_IS_MINIMUM_JAVA9); + assumeFalse("This test fails with Java 9 (https://issues.apache.org/jira/browse/PDFBOX-3155, https://issues.apache.org/jira/browse/SOLR-8876)", + Constants.JRE_IS_MINIMUM_JAVA9); } @Before diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java index cf0312870b99..9c79c07d2699 100644 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java +++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java @@ -21,6 +21,7 @@ import com.google.common.io.Files; import com.typesafe.config.Config; import org.apache.commons.io.FileUtils; +import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; @@ -85,6 +86,8 @@ public class AbstractSolrMorphlineTestBase extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { + assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); + // TODO: test doesn't work with some Locales, see SOLR-6458 savedLocale = Locale.getDefault(); Locale.setDefault(Locale.ENGLISH); diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java index 6c76ecb9d784..4d95a4f9198c 100644 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java +++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineZkTestBase.java @@ -25,6 +25,7 @@ import com.google.common.collect.ListMultimap; import com.typesafe.config.Config; import org.apache.commons.io.FileUtils; +import org.apache.lucene.util.Constants; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.AbstractZkTestCase; @@ -63,6 +64,8 @@ public AbstractSolrMorphlineZkTestBase() { @BeforeClass public static void setupClass() throws Exception { + assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); + assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); solrHomeDirectory = createTempDir().toFile(); diff --git a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java index 9d457468147f..9381c0319711 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java @@ -56,6 +56,8 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() { + assumeFalse("FIXME: This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8052)", Constants.JRE_IS_MINIMUM_JAVA9); + assumeFalse("FIXME: SOLR-7040: This test fails under IBM J9", Constants.JAVA_VENDOR.startsWith("IBM")); System.setProperty("solrcloud.skip.autorecovery", "true"); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java index f4436eb9e65b..27818966c035 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java @@ -19,6 +19,8 @@ import java.io.File; import java.lang.invoke.MethodHandles; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -91,10 +93,10 @@ public class TestCloudDeleteByQuery extends SolrCloudTestCase { private static void createMiniSolrCloudCluster() throws Exception { final String configName = "solrCloudCollectionConfig"; - File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf"); + final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf"); configureCluster(NUM_SERVERS) - .addConfig(configName, configDir.toPath()) + .addConfig(configName, configDir) .configure(); Map collectionProperties = new HashMap<>(); diff --git a/solr/core/src/test/org/apache/solr/core/TestImplicitCoreProperties.java b/solr/core/src/test/org/apache/solr/core/TestImplicitCoreProperties.java index 149d2870f80d..885ad71a7154 100644 --- a/solr/core/src/test/org/apache/solr/core/TestImplicitCoreProperties.java +++ b/solr/core/src/test/org/apache/solr/core/TestImplicitCoreProperties.java @@ -34,8 +34,10 @@ public static void setupContainer() { @AfterClass public static void teardownContainer() { - if (cc != null) + if (cc != null) { cc.shutdown(); + } + cc = null; } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java index cf41dc5a9a37..41807c767268 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java @@ -58,6 +58,7 @@ public static void beforeClass() throws Exception { @AfterClass public static void afterClass() throws Exception { admin = null; // Release it or the test harness complains. + solrHomeDirectory = null; } private static void setupCore(String coreName, boolean blivet) throws IOException { diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index c3ce47de93ce..6e511d7e4d65 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -265,6 +265,7 @@ private static void afterClass() throws Exception { // clean up static sslConfig = null; + testSolrHome = null; } IpTables.unblockAllPorts(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 615365e585ac..386419fb71c7 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -150,8 +150,10 @@ protected static Builder configureCluster(int nodeCount) { @AfterClass public static void shutdownCluster() throws Exception { - if (cluster != null) + if (cluster != null) { cluster.shutdown(); + } + cluster = null; } @Before From 4d20feeeae504b1d4acba2214d7e25df40239c64 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sat, 19 Mar 2016 15:58:31 +0100 Subject: [PATCH 0166/1113] SOLR-8877: Disable test on environments with whitespace --- .../test/org/apache/solr/util/TestSolrCLIRunExample.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java index e80669270846..1a07d3124aa5 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java +++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java @@ -47,6 +47,7 @@ import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.SolrInputDocument; import org.junit.After; +import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; @@ -60,6 +61,12 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void beforeClass() throws IOException { + assumeFalse("FIXME: This test does not work with whitespace in CWD (https://issues.apache.org/jira/browse/SOLR-8877)", + Paths.get(".").toAbsolutePath().toString().contains(" ")); + } /** * Overrides the call to exec bin/solr to start Solr nodes to start them using the Solr test-framework From 19b4168b3f6ef7c6614ece040948cab9a05be32b Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 20 Mar 2016 10:41:05 +0100 Subject: [PATCH 0167/1113] SOLR-8874, SOLR-8876: Disable more Hadoop tests with Java 9 --- .../src/test/org/apache/solr/hadoop/MRUnitBase.java | 2 ++ .../solr/hadoop/MapReduceIndexerToolArgumentParserTest.java | 1 + .../org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java | 1 + .../org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java | 5 ----- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MRUnitBase.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MRUnitBase.java index 558d66255f21..73323b9c8330 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MRUnitBase.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MRUnitBase.java @@ -23,6 +23,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.hadoop.morphline.MorphlineMapRunner; import org.apache.solr.morphlines.solr.AbstractSolrMorphlineTestBase; @@ -37,6 +38,7 @@ public abstract class MRUnitBase extends SolrTestCaseJ4 { @BeforeClass public static void setupClass() throws Exception { + assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); solrHomeZip = SolrOutputFormat.createSolrHomeZip(new File(RESOURCES_DIR + "/solr/mrunit")); diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java index 1aebcf749b13..5dfb5cc822e5 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java @@ -57,6 +57,7 @@ public class MapReduceIndexerToolArgumentParserTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() { + assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); assumeFalse("Does not work on Windows, because it uses UNIX shell commands or POSIX paths", Constants.WINDOWS); assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java index 147d5019c6b2..8d39906fd840 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java @@ -119,6 +119,7 @@ public static void setupClass() throws Exception { assumeFalse("HDFS tests were disabled by -Dtests.disableHdfs", Boolean.parseBoolean(System.getProperty("tests.disableHdfs", "false"))); + assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); assumeFalse("FIXME: This test does not work with Windows because of native library requirements", Constants.WINDOWS); AbstractZkTestCase.SOLRHOME = solrHomeDirectory; diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java index 2687d50c8bc0..97c6dfb77d4b 100644 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java +++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkTest.java @@ -29,12 +29,7 @@ import org.kitesdk.morphline.base.Fields; import org.kitesdk.morphline.base.Notifications; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence; @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) From edf12616bb3418138463b5f847e63a94015a2494 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 20 Mar 2016 11:59:56 +0100 Subject: [PATCH 0168/1113] Update tools for maximum Java 9 compatibility (especially Groovy) --- lucene/common-build.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/common-build.xml b/lucene/common-build.xml index 145f8d2e934d..634887506dbc 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -2342,7 +2342,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list} - - Date: Sun, 20 Mar 2016 13:25:38 +0100 Subject: [PATCH 0169/1113] SOLR-8874: Update Maven config to correctly set tests.disableHdfs --- dev-tools/maven/solr/pom.xml.template | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dev-tools/maven/solr/pom.xml.template b/dev-tools/maven/solr/pom.xml.template index f3c8c8165c43..eb32f6211bab 100644 --- a/dev-tools/maven/solr/pom.xml.template +++ b/dev-tools/maven/solr/pom.xml.template @@ -173,6 +173,15 @@ + + java9-tests-disableHdfs + + [9,) + + + true + + windows-tests-disableHdfs From cc01ba39cd98bf2274f2ac98c44bcd60028697b3 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 20 Mar 2016 09:16:43 -0400 Subject: [PATCH 0170/1113] LUCENE-7121: don't write ord for single-valued points, saving 4 bytes per point --- .../simpletext/SimpleTextPointsWriter.java | 5 +- .../apache/lucene/codecs/PointsWriter.java | 5 +- .../codecs/lucene60/Lucene60PointsWriter.java | 11 +++- .../lucene/index/PointValuesWriter.java | 8 ++- .../org/apache/lucene/util/bkd/BKDWriter.java | 65 +++++++++++++------ .../lucene/util/bkd/HeapPointReader.java | 8 ++- .../lucene/util/bkd/HeapPointWriter.java | 53 +++++++++------ .../lucene/util/bkd/OfflinePointReader.java | 39 +++++++---- .../lucene/util/bkd/OfflinePointWriter.java | 22 ++++--- .../lucene/util/bkd/Test2BBKDPoints.java | 19 +----- .../org/apache/lucene/util/bkd/TestBKD.java | 18 ++--- .../index/BasePointsFormatTestCase.java | 2 +- 12 files changed, 158 insertions(+), 97 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java index 33af33e688e4..e54e20abe6a2 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java @@ -69,6 +69,8 @@ public SimpleTextPointsWriter(SegmentWriteState writeState) throws IOException { @Override public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); + // We use the normal BKDWriter, but subclass to customize how it writes the index and blocks to disk: try (BKDWriter writer = new BKDWriter(writeState.segmentInfo.maxDoc(), writeState.directory, @@ -77,7 +79,8 @@ public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOExcept fieldInfo.getPointNumBytes(), BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, - values.size(fieldInfo.name)) { + values.size(fieldInfo.name), + singleValuePerDoc) { @Override protected void writeIndex(IndexOutput out, long[] leafBlockFPs, byte[] splitPackedValues) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java index 000d713fa9a4..43b4416fedb2 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java @@ -42,16 +42,19 @@ protected PointsWriter() { * a faster but more complex implementation. */ protected void mergeOneField(MergeState mergeState, FieldInfo fieldInfo) throws IOException { long maxPointCount = 0; + int docCount = 0; for (int i=0;i bkdReaders = new ArrayList<>(); List docMaps = new ArrayList<>(); List docIDBases = new ArrayList<>(); diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java index 694ccf6ddad0..f259c5b99626 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java @@ -32,6 +32,8 @@ class PointValuesWriter { private final Counter iwBytesUsed; private int[] docIDs; private int numPoints; + private int numDocs; + private int lastDocID = -1; private final byte[] packedValue; public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo) { @@ -57,6 +59,10 @@ public void addPackedValue(int docID, BytesRef value) { } bytes.append(value); docIDs[numPoints] = docID; + if (docID != lastDocID) { + numDocs++; + lastDocID = docID; + } numPoints++; } @@ -116,7 +122,7 @@ public long size(String fieldName) { @Override public int getDocCount(String fieldName) { - throw new UnsupportedOperationException(); + return numDocs; } }); } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 796f611dd9b6..488fa43af8b4 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -134,11 +134,16 @@ public class BKDWriter implements Closeable { /** An upper bound on how many points the caller will add (includes deletions) */ private final long totalPointCount; - public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, long totalPointCount) throws IOException { - this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP, totalPointCount); + /** True if every document has at most one value. We specialize this case by not bothering to store the ord since it's redundant with docID. */ + private final boolean singleValuePerDoc; + + private final int maxDoc; + + public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, long totalPointCount, boolean singleValuePerDoc) throws IOException { + this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP, totalPointCount, singleValuePerDoc); } - public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount) throws IOException { + public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException { verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount); // We use tracking dir to deal with removing files on exception, so each place that // creates temp files doesn't need crazy try/finally/sucess logic: @@ -148,6 +153,7 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n this.numDims = numDims; this.bytesPerDim = bytesPerDim; this.totalPointCount = totalPointCount; + this.maxDoc = maxDoc; docsSeen = new FixedBitSet(maxDoc); packedBytesLength = numDims * bytesPerDim; @@ -162,9 +168,14 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n // If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes). longOrds = totalPointCount > Integer.MAX_VALUE; + this.singleValuePerDoc = singleValuePerDoc; // dimensional values (numDims * bytesPerDim) + ord (int or long) + docID (int) - if (longOrds) { + if (singleValuePerDoc) { + // Lucene only supports up to 2.1 docs, so we better not need longOrds in this case: + assert longOrds == false; + bytesPerDoc = packedBytesLength + Integer.BYTES; + } else if (longOrds) { bytesPerDoc = packedBytesLength + Long.BYTES + Integer.BYTES; } else { bytesPerDoc = packedBytesLength + Integer.BYTES + Integer.BYTES; @@ -187,7 +198,7 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n } // We write first maxPointsSortInHeap in heap, then cutover to offline for additional points: - heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength, longOrds); + heapPointWriter = new HeapPointWriter(16, maxPointsSortInHeap, packedBytesLength, longOrds, singleValuePerDoc); this.maxMBSortInHeap = maxMBSortInHeap; } @@ -216,7 +227,7 @@ public static void verifyParams(int numDims, int maxPointsInLeafNode, double max private void spillToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: - offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill"); + offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill", singleValuePerDoc); tempInput = offlinePointWriter.out; PointReader reader = heapPointWriter.getReader(0, pointCount); for(int i=0;i 1) { - ordBitSet = new LongBitSet(pointCount); + if (singleValuePerDoc) { + ordBitSet = new LongBitSet(maxDoc); + } else { + ordBitSet = new LongBitSet(pointCount); + } } else { ordBitSet = null; } @@ -867,7 +886,11 @@ public long finish(IndexOutput out) throws IOException { success = true; } finally { if (success == false) { + if (tempInput != null) { + IOUtils.closeWhileHandlingException(tempInput); + } IOUtils.deleteFilesIgnoringExceptions(tempDir, tempDir.getCreatedFiles()); + tempInput = null; } } @@ -1049,7 +1072,7 @@ private int split(byte[] minPackedValue, byte[] maxPackedValue) { private PathSlice switchToHeap(PathSlice source) throws IOException { int count = Math.toIntExact(source.count); try ( - PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds); + PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds, singleValuePerDoc); PointReader reader = source.writer.getReader(source.start, source.count); ) { for(int i=0;i blocks, int valuesPerBlock, int packedBytesLength, int[] ords, long[] ordsLong, int[] docIDs, int start, int end) { + HeapPointReader(List blocks, int valuesPerBlock, int packedBytesLength, int[] ords, long[] ordsLong, int[] docIDs, int start, int end, boolean singleValuePerDoc) { this.blocks = blocks; this.valuesPerBlock = valuesPerBlock; + this.singleValuePerDoc = singleValuePerDoc; this.ords = ords; this.ordsLong = ordsLong; this.docIDs = docIDs; @@ -75,7 +77,9 @@ public int docID() { @Override public long ord() { - if (ordsLong != null) { + if (singleValuePerDoc) { + return docIDs[curRead]; + } else if (ordsLong != null) { return ordsLong[curRead]; } else { return ords[curRead]; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java index a4aedd0be3e0..1bcf836e42e3 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java @@ -31,17 +31,24 @@ final class HeapPointWriter implements PointWriter { final int maxSize; final int valuesPerBlock; final int packedBytesLength; + final boolean singleValuePerDoc; // NOTE: can't use ByteBlockPool because we need random-write access when sorting in heap final List blocks = new ArrayList<>(); - public HeapPointWriter(int initSize, int maxSize, int packedBytesLength, boolean longOrds) { + public HeapPointWriter(int initSize, int maxSize, int packedBytesLength, boolean longOrds, boolean singleValuePerDoc) { docIDs = new int[initSize]; this.maxSize = maxSize; this.packedBytesLength = packedBytesLength; - if (longOrds) { - this.ordsLong = new long[initSize]; + this.singleValuePerDoc = singleValuePerDoc; + if (singleValuePerDoc) { + this.ordsLong = null; + this.ords = null; } else { - this.ords = new int[initSize]; + if (longOrds) { + this.ordsLong = new long[initSize]; + } else { + this.ords = new int[initSize]; + } } // 4K per page, unless each value is > 4K: valuesPerBlock = Math.max(1, 4096/packedBytesLength); @@ -52,12 +59,14 @@ public void copyFrom(HeapPointWriter other) { throw new IllegalStateException("docIDs.length=" + docIDs.length + " other.nextWrite=" + other.nextWrite); } System.arraycopy(other.docIDs, 0, docIDs, 0, other.nextWrite); - if (other.ords != null) { - assert this.ords != null; - System.arraycopy(other.ords, 0, ords, 0, other.nextWrite); - } else { - assert this.ordsLong != null; - System.arraycopy(other.ordsLong, 0, ordsLong, 0, other.nextWrite); + if (singleValuePerDoc == false) { + if (other.ords != null) { + assert this.ords != null; + System.arraycopy(other.ords, 0, ords, 0, other.nextWrite); + } else { + assert this.ordsLong != null; + System.arraycopy(other.ordsLong, 0, ordsLong, 0, other.nextWrite); + } } for(byte[] block : other.blocks) { @@ -117,18 +126,22 @@ public void append(byte[] packedValue, long ord, int docID) { int nextSize = Math.min(maxSize, ArrayUtil.oversize(nextWrite+1, Integer.BYTES)); assert nextSize > nextWrite: "nextSize=" + nextSize + " vs nextWrite=" + nextWrite; docIDs = growExact(docIDs, nextSize); - if (ordsLong != null) { - ordsLong = growExact(ordsLong, nextSize); - } else { - ords = growExact(ords, nextSize); + if (singleValuePerDoc == false) { + if (ordsLong != null) { + ordsLong = growExact(ordsLong, nextSize); + } else { + ords = growExact(ords, nextSize); + } } } writePackedValue(nextWrite, packedValue); - if (ordsLong != null) { - ordsLong[nextWrite] = ord; - } else { - assert ord <= Integer.MAX_VALUE; - ords[nextWrite] = (int) ord; + if (singleValuePerDoc == false) { + if (ordsLong != null) { + ordsLong[nextWrite] = ord; + } else { + assert ord <= Integer.MAX_VALUE; + ords[nextWrite] = (int) ord; + } } docIDs[nextWrite] = docID; nextWrite++; @@ -137,7 +150,7 @@ public void append(byte[] packedValue, long ord, int docID) { @Override public PointReader getReader(long start, long length) { assert start + length <= docIDs.length: "start=" + start + " length=" + length + " docIDs.length=" + docIDs.length; - return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, ordsLong, docIDs, (int) start, nextWrite); + return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, ordsLong, docIDs, (int) start, nextWrite, singleValuePerDoc); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 15274fb77615..18afda4659ce 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -32,18 +32,23 @@ final class OfflinePointReader extends PointReader { long countLeft; final IndexInput in; private final byte[] packedValue; + final boolean singleValuePerDoc; private long ord; private int docID; // true if ords are written as long (8 bytes), else 4 bytes private boolean longOrds; private boolean checked; - OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, boolean longOrds) throws IOException { + OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, + boolean longOrds, boolean singleValuePerDoc) throws IOException { + this.singleValuePerDoc = singleValuePerDoc; int bytesPerDoc = packedBytesLength + Integer.BYTES; - if (longOrds) { - bytesPerDoc += Long.BYTES; - } else { - bytesPerDoc += Integer.BYTES; + if (singleValuePerDoc == false) { + if (longOrds) { + bytesPerDoc += Long.BYTES; + } else { + bytesPerDoc += Integer.BYTES; + } } if ((start + length) * bytesPerDoc + CodecUtil.footerLength() > tempDir.fileLength(tempFileName)) { @@ -84,12 +89,16 @@ public boolean next() throws IOException { assert countLeft == -1; return false; } - if (longOrds) { - ord = in.readLong(); + if (singleValuePerDoc == false) { + if (longOrds) { + ord = in.readLong(); + } else { + ord = in.readInt(); + } + docID = in.readInt(); } else { - ord = in.readInt(); + ord = docID = in.readInt(); } - docID = in.readInt(); return true; } @@ -135,10 +144,12 @@ public long split(long count, LongBitSet rightTree, PointWriter left, PointWrite int packedBytesLength = packedValue.length; int bytesPerDoc = packedBytesLength + Integer.BYTES; - if (longOrds) { - bytesPerDoc += Long.BYTES; - } else { - bytesPerDoc += Integer.BYTES; + if (singleValuePerDoc == false) { + if (longOrds) { + bytesPerDoc += Long.BYTES; + } else { + bytesPerDoc += Integer.BYTES; + } } long rightCount = 0; @@ -159,8 +170,10 @@ public long split(long count, LongBitSet rightTree, PointWriter left, PointWrite if (longOrds) { ord = readLong(buffer, packedBytesLength); } else { + // This is either ord (multi-valued case) or docID (which we use as ord in the single valued case): ord = readInt(buffer, packedBytesLength); } + if (rightTree.get(ord)) { rightOut.writeBytes(buffer, 0, bytesPerDoc); if (doClearBits) { diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index af974c7b1474..53148769fe8d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -30,37 +29,42 @@ final class OfflinePointWriter implements PointWriter { final Directory tempDir; final IndexOutput out; final int packedBytesLength; + final boolean singleValuePerDoc; long count; private boolean closed; // true if ords are written as long (8 bytes), else 4 bytes private boolean longOrds; - public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds, String desc) throws IOException { + public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds, String desc, boolean singleValuePerDoc) throws IOException { this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT); this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.longOrds = longOrds; + this.singleValuePerDoc = singleValuePerDoc; } /** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */ - public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count, boolean longOrds) { + public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count, boolean longOrds, boolean singleValuePerDoc) { this.out = out; this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.count = count; closed = true; this.longOrds = longOrds; + this.singleValuePerDoc = singleValuePerDoc; } @Override public void append(byte[] packedValue, long ord, int docID) throws IOException { assert packedValue.length == packedBytesLength; out.writeBytes(packedValue, 0, packedValue.length); - if (longOrds) { - out.writeLong(ord); - } else { - assert ord <= Integer.MAX_VALUE; - out.writeInt((int) ord); + if (singleValuePerDoc == false) { + if (longOrds) { + out.writeLong(ord); + } else { + assert ord <= Integer.MAX_VALUE; + out.writeInt((int) ord); + } } out.writeInt(docID); count++; @@ -70,7 +74,7 @@ public void append(byte[] packedValue, long ord, int docID) throws IOException { public PointReader getReader(long start, long length) throws IOException { assert closed; assert start + length <= count: "start=" + start + " length=" + length + " count=" + count; - return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, length, longOrds); + return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, length, longOrds, singleValuePerDoc); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java index cfb98b09c78a..ffcbcf83cc59 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -16,29 +16,14 @@ */ package org.apache.lucene.util.bkd; -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PointsReader; -import org.apache.lucene.codecs.PointsWriter; -import org.apache.lucene.codecs.lucene60.Lucene60PointsReader; -import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.LuceneTestCase.Monster; -import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TimeUnits; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; @@ -55,7 +40,7 @@ public void test1D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 26L * numDocs, false); int counter = 0; byte[] packedBytes = new byte[Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { @@ -88,7 +73,7 @@ public void test2D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs, false); int counter = 0; byte[] packedBytes = new byte[2*Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index acc049ca9226..dc7f8f7e3cf4 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -43,8 +43,8 @@ public class TestBKD extends LuceneTestCase { - private long randomPointCount() { - if (random().nextBoolean()) { + private long randomPointCount(boolean singleValuePerDoc) { + if (singleValuePerDoc || random().nextBoolean()) { return random().nextInt(Integer.MAX_VALUE); } else { return random().nextLong() & Long.MAX_VALUE; @@ -53,7 +53,7 @@ private long randomPointCount() { public void testBasicInts1D() throws Exception { try (Directory dir = getDirectory(100)) { - BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, randomPointCount()); + BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, randomPointCount(true), true); byte[] scratch = new byte[4]; for(int docID=0;docID<100;docID++) { NumericUtils.intToSortableBytes(docID, scratch, 0); @@ -128,7 +128,7 @@ public void testRandomIntsNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, randomPointCount()); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, randomPointCount(true), true); if (VERBOSE) { System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs); @@ -269,7 +269,7 @@ public void testBigIntNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(true), true); BigInteger[][] docs = new BigInteger[numDocs][]; byte[] scratch = new byte[numBytesPerDim*numDims]; @@ -442,7 +442,7 @@ public void testRandomBinaryBig() throws Exception { public void testTooLittleHeap() throws Exception { try (Directory dir = getDirectory(0)) { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, randomPointCount()); + new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, randomPointCount(true), true); }); assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode")); } @@ -565,7 +565,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi List docIDBases = null; int seg = 0; - BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); + BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); IndexInput in = null; @@ -619,7 +619,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi seg++; maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000); maxMB = (float) 3.0 + (3*random().nextDouble()); - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); lastDocIDBase = docID; } } @@ -634,7 +634,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi out.close(); in = dir.openInput("bkd", IOContext.DEFAULT); seg++; - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount()); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); List readers = new ArrayList<>(); for(long fp : toMerge) { in.seek(fp); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java index ecb3a616a0ff..7c42d1c75f16 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java @@ -229,7 +229,7 @@ public void testWithExceptions() throws Exception { throw ise; } } catch (AssertionError ae) { - if (ae.getMessage().contains("does not exist; files=")) { + if (ae.getMessage() != null && ae.getMessage().contains("does not exist; files=")) { // OK: likely we threw the random IOExc when IW was asserting the commit files exist done = true; } else { From 51b109620be7d565ae816f9e327813798c001611 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 20 Mar 2016 16:05:17 -0400 Subject: [PATCH 0171/1113] LUCENE-7117, LUCENE-7118: Remove multidimensional arrays from PointRangeQuery --- .../apache/lucene/document/BinaryPoint.java | 2 +- .../apache/lucene/document/DoublePoint.java | 12 +- .../apache/lucene/document/FloatPoint.java | 12 +- .../org/apache/lucene/document/IntPoint.java | 12 +- .../org/apache/lucene/document/LongPoint.java | 12 +- .../apache/lucene/search/PointRangeQuery.java | 88 ++++------- .../lucene/search/TestPointQueries.java | 138 +++++++++++------- .../lucene/document/BigIntegerPoint.java | 12 +- .../lucene/document/InetAddressPoint.java | 6 +- .../apache/lucene/document/LatLonPoint.java | 32 ++-- .../lucene/document/TestBigIntegerPoint.java | 24 +-- .../lucene/document/TestInetAddressPoint.java | 34 +++-- .../lucene/document/TestLatLonPoint.java | 28 ++-- 13 files changed, 188 insertions(+), 224 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java index 4e27d8127c42..60fd9d636e64 100644 --- a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java @@ -173,7 +173,7 @@ public static Query newRangeQuery(String field, byte[] lowerValue, byte[] upperV * @return a query matching documents within this range. */ public static Query newRangeQuery(String field, byte[][] lowerValue, byte[][] upperValue) { - return new PointRangeQuery(field, lowerValue, upperValue) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { assert value != null; diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java index 1eb1d7eb2ee6..53de1279cba4 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java @@ -126,16 +126,6 @@ public String toString() { return result.toString(); } - /** Encode n-dimensional double point into binary encoding */ - private static byte[][] encode(double value[]) { - byte[][] encoded = new byte[value.length][]; - for (int i = 0; i < value.length; i++) { - encoded[i] = new byte[Double.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } - return encoded; - } - // public helper methods (e.g. for queries) /** Encode single double dimension */ @@ -207,7 +197,7 @@ public static Query newRangeQuery(String field, double lowerValue, double upperV */ public static Query newRangeQuery(String field, double[] lowerValue, double[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { return Double.toString(decodeDimension(value, 0)); diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java index 08192e5cac1e..ab209288a860 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java @@ -126,16 +126,6 @@ public String toString() { return result.toString(); } - /** Encode n-dimensional float values into binary encoding */ - private static byte[][] encode(float value[]) { - byte[][] encoded = new byte[value.length][]; - for (int i = 0; i < value.length; i++) { - encoded[i] = new byte[Float.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } - return encoded; - } - // public helper methods (e.g. for queries) /** Encode single float dimension */ @@ -207,7 +197,7 @@ public static Query newRangeQuery(String field, float lowerValue, float upperVal */ public static Query newRangeQuery(String field, float[] lowerValue, float[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { return Float.toString(decodeDimension(value, 0)); diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java index c40a4b73a1dc..db24fc974800 100644 --- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java @@ -125,16 +125,6 @@ public String toString() { result.append('>'); return result.toString(); } - - /** Encode n-dimensional integer values into binary encoding */ - private static byte[][] encode(int value[]) { - byte[][] encoded = new byte[value.length][]; - for (int i = 0; i < value.length; i++) { - encoded[i] = new byte[Integer.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } - return encoded; - } // public helper methods (e.g. for queries) @@ -205,7 +195,7 @@ public static Query newRangeQuery(String field, int lowerValue, int upperValue) */ public static Query newRangeQuery(String field, int[] lowerValue, int[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { return Integer.toString(decodeDimension(value, 0)); diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java index 46bf93cd6eea..ab2cb6fd8b4b 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java @@ -126,16 +126,6 @@ public String toString() { return result.toString(); } - /** Encode n-dimensional long values into binary encoding */ - private static byte[][] encode(long value[]) { - byte[][] encoded = new byte[value.length][]; - for (int i = 0; i < value.length; i++) { - encoded[i] = new byte[Long.BYTES]; - encodeDimension(value[i], encoded[i], 0); - } - return encoded; - } - // public helper methods (e.g. for queries) /** Encode single long dimension */ @@ -205,7 +195,7 @@ public static Query newRangeQuery(String field, long lowerValue, long upperValue */ public static Query newRangeQuery(String field, long[] lowerValue, long[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, encode(lowerValue), encode(upperValue)) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { return Long.toString(decodeDimension(value, 0)); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 9384d235d3bc..37580e02b440 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -47,8 +47,8 @@ public abstract class PointRangeQuery extends Query { final String field; final int numDims; final int bytesPerDim; - final byte[][] lowerPoint; - final byte[][] upperPoint; + final byte[] lowerPoint; + final byte[] upperPoint; /** * Expert: create a multidimensional range query for point values. @@ -56,40 +56,26 @@ public abstract class PointRangeQuery extends Query { * @param field field name. must not be {@code null}. * @param lowerPoint lower portion of the range (inclusive). * @param upperPoint upper portion of the range (inclusive). + * @param numDims number of dimensions. * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} */ - protected PointRangeQuery(String field, byte[][] lowerPoint, byte[][] upperPoint) { + protected PointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, int numDims) { checkArgs(field, lowerPoint, upperPoint); this.field = field; if (lowerPoint.length == 0) { throw new IllegalArgumentException("lowerPoint has length of zero"); } - this.numDims = lowerPoint.length; - - if (upperPoint.length != numDims) { + if (lowerPoint.length % numDims != 0) { + throw new IllegalArgumentException("lowerPoint is not a fixed multiple of numDims"); + } + if (upperPoint.length != upperPoint.length) { throw new IllegalArgumentException("lowerPoint has length=" + numDims + " but upperPoint has different length=" + upperPoint.length); } + this.numDims = numDims; + this.bytesPerDim = lowerPoint.length / numDims; + this.lowerPoint = lowerPoint; this.upperPoint = upperPoint; - - if (lowerPoint[0] == null) { - throw new IllegalArgumentException("lowerPoint[0] is null"); - } - this.bytesPerDim = lowerPoint[0].length; - for (int i = 0; i < numDims; i++) { - if (lowerPoint[i] == null) { - throw new IllegalArgumentException("lowerPoint[" + i + "] is null"); - } - if (upperPoint[i] == null) { - throw new IllegalArgumentException("upperPoint[" + i + "] is null"); - } - if (lowerPoint[i].length != bytesPerDim) { - throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + lowerPoint[i].length); - } - if (upperPoint[i].length != bytesPerDim) { - throw new IllegalArgumentException("all dimensions must have same bytes length, but saw " + bytesPerDim + " and " + upperPoint[i].length); - } - } } /** @@ -116,8 +102,7 @@ public final Weight createWeight(IndexSearcher searcher, boolean needsScores) th return new ConstantScoreWeight(this) { - private DocIdSet buildMatchingDocIdSet(LeafReader reader, PointValues values, - byte[] packedLower, byte[] packedUpper) throws IOException { + private DocIdSet buildMatchingDocIdSet(LeafReader reader, PointValues values) throws IOException { DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc()); values.intersect(field, @@ -137,11 +122,11 @@ public void visit(int docID) { public void visit(int docID, byte[] packedValue) { for(int dim=0;dim 0) { + if (StringHelper.compare(bytesPerDim, packedValue, offset, upperPoint, offset) > 0) { // Doc's value is too high, in this dimension return; } @@ -159,13 +144,13 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { for(int dim=0;dim 0 || - StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedLower, offset) < 0) { + if (StringHelper.compare(bytesPerDim, minPackedValue, offset, upperPoint, offset) > 0 || + StringHelper.compare(bytesPerDim, maxPackedValue, offset, lowerPoint, offset) < 0) { return Relation.CELL_OUTSIDE_QUERY; } - crosses |= StringHelper.compare(bytesPerDim, minPackedValue, offset, packedLower, offset) < 0 || - StringHelper.compare(bytesPerDim, maxPackedValue, offset, packedUpper, offset) > 0; + crosses |= StringHelper.compare(bytesPerDim, minPackedValue, offset, lowerPoint, offset) < 0 || + StringHelper.compare(bytesPerDim, maxPackedValue, offset, upperPoint, offset) > 0; } if (crosses) { @@ -197,16 +182,6 @@ public Scorer scorer(LeafReaderContext context) throws IOException { if (bytesPerDim != fieldInfo.getPointNumBytes()) { throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + fieldInfo.getPointNumBytes() + " but this query has bytesPerDim=" + bytesPerDim); } - int bytesPerDim = fieldInfo.getPointNumBytes(); - - byte[] packedLower = new byte[numDims * bytesPerDim]; - byte[] packedUpper = new byte[numDims * bytesPerDim]; - - // Carefully pack lower and upper bounds - for(int dim=0;dim 0 - || StringHelper.compare(bytesPerDim, packedUpper, offset, fieldPackedUpper, offset) < 0) { + if (StringHelper.compare(bytesPerDim, lowerPoint, offset, fieldPackedLower, offset) > 0 + || StringHelper.compare(bytesPerDim, upperPoint, offset, fieldPackedUpper, offset) < 0) { allDocsMatch = false; break; } @@ -230,7 +205,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { // all docs have a value and all points are within bounds, so everything matches iterator = DocIdSetIterator.all(reader.maxDoc()); } else { - iterator = buildMatchingDocIdSet(reader, values, packedLower, packedUpper).iterator(); + iterator = buildMatchingDocIdSet(reader, values).iterator(); } return new ConstantScoreScorer(this, score(), iterator); @@ -263,15 +238,12 @@ public final boolean equals(Object other) { return false; } - // Cannot use Arrays.equals here, because it in turn uses byte[].equals - // to compare each value, which only uses "==" - for(int dim=0;dim 0) { sb.append(','); } + + int startOffset = bytesPerDim * i; sb.append('['); - sb.append(toString(i, lowerPoint[i])); + sb.append(toString(i, Arrays.copyOfRange(lowerPoint, startOffset, startOffset + bytesPerDim))); sb.append(" TO "); - sb.append(toString(i, upperPoint[i])); + sb.append(toString(i, Arrays.copyOfRange(upperPoint, startOffset, startOffset + bytesPerDim))); sb.append(']'); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index c72ab44ac398..b4a586d6b376 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -1896,82 +1896,112 @@ public void testRangeOptimizesIfAllPointsMatch() throws IOException { } public void testPointRangeEquals() { - Query q = IntPoint.newRangeQuery("a", 0, 1000); - assertEquals(q, IntPoint.newRangeQuery("a", 0, 1000)); - assertFalse(q.equals(IntPoint.newRangeQuery("a", 1, 1000))); - - q = LongPoint.newRangeQuery("a", 0, 1000); - assertEquals(q, LongPoint.newRangeQuery("a", 0, 1000)); - assertFalse(q.equals(LongPoint.newRangeQuery("a", 1, 1000))); - - q = FloatPoint.newRangeQuery("a", 0, 1000); - assertEquals(q, FloatPoint.newRangeQuery("a", 0, 1000)); - assertFalse(q.equals(FloatPoint.newRangeQuery("a", 1, 1000))); - - q = DoublePoint.newRangeQuery("a", 0, 1000); - assertEquals(q, DoublePoint.newRangeQuery("a", 0, 1000)); - assertFalse(q.equals(DoublePoint.newRangeQuery("a", 1, 1000))); + Query q1 = IntPoint.newRangeQuery("a", 0, 1000); + Query q2 = IntPoint.newRangeQuery("a", 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(IntPoint.newRangeQuery("a", 1, 1000))); + + q1 = LongPoint.newRangeQuery("a", 0, 1000); + q2 = LongPoint.newRangeQuery("a", 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LongPoint.newRangeQuery("a", 1, 1000))); + + q1 = FloatPoint.newRangeQuery("a", 0, 1000); + q2 = FloatPoint.newRangeQuery("a", 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(FloatPoint.newRangeQuery("a", 1, 1000))); + + q1 = DoublePoint.newRangeQuery("a", 0, 1000); + q2 = DoublePoint.newRangeQuery("a", 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(DoublePoint.newRangeQuery("a", 1, 1000))); byte[] zeros = new byte[5]; byte[] ones = new byte[5]; Arrays.fill(ones, (byte) 0xff); - q = BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {ones}); - assertEquals(q, BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {ones})); + q1 = BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {ones}); + q2 = BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {ones}); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); byte[] other = ones.clone(); other[2] = (byte) 5; - assertFalse(q.equals(BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {other}))); + assertFalse(q1.equals(BinaryPoint.newRangeQuery("a", new byte[][] {zeros}, new byte[][] {other}))); } public void testPointExactEquals() { - Query q = IntPoint.newExactQuery("a", 1000); - assertEquals(q, IntPoint.newExactQuery("a", 1000)); - assertFalse(q.equals(IntPoint.newExactQuery("a", 1))); - - q = LongPoint.newExactQuery("a", 1000); - assertEquals(q, LongPoint.newExactQuery("a", 1000)); - assertFalse(q.equals(LongPoint.newExactQuery("a", 1))); - - q = FloatPoint.newExactQuery("a", 1000); - assertEquals(q, FloatPoint.newExactQuery("a", 1000)); - assertFalse(q.equals(FloatPoint.newExactQuery("a", 1))); - - q = DoublePoint.newExactQuery("a", 1000); - assertEquals(q, DoublePoint.newExactQuery("a", 1000)); - assertFalse(q.equals(DoublePoint.newExactQuery("a", 1))); + Query q1 = IntPoint.newExactQuery("a", 1000); + Query q2 = IntPoint.newExactQuery("a", 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(IntPoint.newExactQuery("a", 1))); + + q1 = LongPoint.newExactQuery("a", 1000); + q2 = LongPoint.newExactQuery("a", 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LongPoint.newExactQuery("a", 1))); + + q1 = FloatPoint.newExactQuery("a", 1000); + q2 = FloatPoint.newExactQuery("a", 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(FloatPoint.newExactQuery("a", 1))); + + q1 = DoublePoint.newExactQuery("a", 1000); + q2 = DoublePoint.newExactQuery("a", 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(DoublePoint.newExactQuery("a", 1))); byte[] ones = new byte[5]; Arrays.fill(ones, (byte) 0xff); - q = BinaryPoint.newExactQuery("a", ones); - assertEquals(q, BinaryPoint.newExactQuery("a", ones)); + q1 = BinaryPoint.newExactQuery("a", ones); + q2 = BinaryPoint.newExactQuery("a", ones); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); byte[] other = ones.clone(); other[2] = (byte) 5; - assertFalse(q.equals(BinaryPoint.newExactQuery("a", other))); + assertFalse(q1.equals(BinaryPoint.newExactQuery("a", other))); } public void testPointInSetEquals() { - Query q = IntPoint.newSetQuery("a", 0, 1000, 17); - assertEquals(q, IntPoint.newSetQuery("a", 17, 0, 1000)); - assertFalse(q.equals(IntPoint.newSetQuery("a", 1, 17, 1000))); - - q = LongPoint.newSetQuery("a", 0, 1000, 17); - assertEquals(q, LongPoint.newSetQuery("a", 17, 0, 1000)); - assertFalse(q.equals(LongPoint.newSetQuery("a", 1, 17, 1000))); - - q = FloatPoint.newSetQuery("a", 0, 1000, 17); - assertEquals(q, FloatPoint.newSetQuery("a", 17, 0, 1000)); - assertFalse(q.equals(FloatPoint.newSetQuery("a", 1, 17, 1000))); - - q = DoublePoint.newSetQuery("a", 0, 1000, 17); - assertEquals(q, DoublePoint.newSetQuery("a", 17, 0, 1000)); - assertFalse(q.equals(DoublePoint.newSetQuery("a", 1, 17, 1000))); + Query q1 = IntPoint.newSetQuery("a", 0, 1000, 17); + Query q2 = IntPoint.newSetQuery("a", 17, 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(IntPoint.newSetQuery("a", 1, 17, 1000))); + + q1 = LongPoint.newSetQuery("a", 0, 1000, 17); + q2 = LongPoint.newSetQuery("a", 17, 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LongPoint.newSetQuery("a", 1, 17, 1000))); + + q1 = FloatPoint.newSetQuery("a", 0, 1000, 17); + q2 = FloatPoint.newSetQuery("a", 17, 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(FloatPoint.newSetQuery("a", 1, 17, 1000))); + + q1 = DoublePoint.newSetQuery("a", 0, 1000, 17); + q2 = DoublePoint.newSetQuery("a", 17, 0, 1000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(DoublePoint.newSetQuery("a", 1, 17, 1000))); byte[] zeros = new byte[5]; byte[] ones = new byte[5]; Arrays.fill(ones, (byte) 0xff); - q = BinaryPoint.newSetQuery("a", new byte[][] {zeros, ones}); - assertEquals(q, BinaryPoint.newSetQuery("a", new byte[][] {zeros, ones})); + q1 = BinaryPoint.newSetQuery("a", new byte[][] {zeros, ones}); + q2 = BinaryPoint.newSetQuery("a", new byte[][] {zeros, ones}); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); byte[] other = ones.clone(); other[2] = (byte) 5; - assertFalse(q.equals(BinaryPoint.newSetQuery("a", new byte[][] {zeros, other}))); + assertFalse(q1.equals(BinaryPoint.newSetQuery("a", new byte[][] {zeros, other}))); } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java index 70445d6d2fd7..efbb37c0c867 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java @@ -129,16 +129,6 @@ public String toString() { result.append('>'); return result.toString(); } - - /** sugar: Encode n-dimensional BigInteger values into binary encoding */ - private static byte[][] encode(BigInteger value[]) { - byte[][] encoded = new byte[value.length][]; - for (int i = 0; i < value.length; i++) { - encoded[i] = new byte[BYTES]; - encodeDimension(value[i], encoded[i], 0); - } - return encoded; - } // public helper methods (e.g. for queries) @@ -212,7 +202,7 @@ public static Query newRangeQuery(String field, BigInteger lowerValue, BigIntege */ public static Query newRangeQuery(String field, BigInteger[] lowerValue, BigInteger[] upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - return new PointRangeQuery(field, BigIntegerPoint.encode(lowerValue), BigIntegerPoint.encode(upperValue)) { + return new PointRangeQuery(field, pack(lowerValue).bytes, pack(upperValue).bytes, lowerValue.length) { @Override protected String toString(int dimension, byte[] value) { return BigIntegerPoint.decodeDimension(value, 0).toString(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java index 7abc1b695acd..c0739a40e9ea 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java @@ -196,11 +196,7 @@ public static Query newPrefixQuery(String field, InetAddress value, int prefixLe */ public static Query newRangeQuery(String field, InetAddress lowerValue, InetAddress upperValue) { PointRangeQuery.checkArgs(field, lowerValue, upperValue); - byte[][] lowerBytes = new byte[1][]; - lowerBytes[0] = encode(lowerValue); - byte[][] upperBytes = new byte[1][]; - upperBytes[0] = encode(upperValue); - return new PointRangeQuery(field, lowerBytes, upperBytes) { + return new PointRangeQuery(field, encode(lowerValue), encode(upperValue), 1) { @Override protected String toString(int dimension, byte[] value) { return decode(value).getHostAddress(); // for ranges, the range itself is already bracketed diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 7c056e252271..429f9792b97f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -206,13 +206,11 @@ public static double decodeLongitude(byte[] src, int offset) { return decodeLongitude(NumericUtils.sortableBytesToInt(src, offset)); } - /** sugar encodes a single point as a 2D byte array */ - private static byte[][] encode(double latitude, double longitude) { - byte[][] bytes = new byte[2][]; - bytes[0] = new byte[4]; - NumericUtils.intToSortableBytes(encodeLatitude(latitude), bytes[0], 0); - bytes[1] = new byte[4]; - NumericUtils.intToSortableBytes(encodeLongitude(longitude), bytes[1], 0); + /** sugar encodes a single point as a byte array */ + private static byte[] encode(double latitude, double longitude) { + byte[] bytes = new byte[2 * Integer.BYTES]; + NumericUtils.intToSortableBytes(encodeLatitude(latitude), bytes, 0); + NumericUtils.intToSortableBytes(encodeLongitude(longitude), bytes, Integer.BYTES); return bytes; } @@ -251,8 +249,8 @@ static void checkCompatible(FieldInfo fieldInfo) { * @throws IllegalArgumentException if {@code field} is null, or the box has invalid coordinates. */ public static Query newBoxQuery(String field, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) { - byte[][] lower = encode(minLatitude, minLongitude); - byte[][] upper = encode(maxLatitude, maxLongitude); + byte[] lower = encode(minLatitude, minLongitude); + byte[] upper = encode(maxLatitude, maxLongitude); // Crosses date line: we just rewrite into OR of two bboxes, with longitude as an open range: if (maxLongitude < minLongitude) { // Disable coord here because a multi-valued doc could match both rects and get unfairly boosted: @@ -260,19 +258,15 @@ public static Query newBoxQuery(String field, double minLatitude, double maxLati q.setDisableCoord(true); // E.g.: maxLon = -179, minLon = 179 - byte[][] leftOpen = new byte[2][]; - leftOpen[0] = lower[0]; + byte[] leftOpen = lower.clone(); // leave longitude open - leftOpen[1] = new byte[Integer.BYTES]; - NumericUtils.intToSortableBytes(Integer.MIN_VALUE, leftOpen[1], 0); + NumericUtils.intToSortableBytes(Integer.MIN_VALUE, leftOpen, Integer.BYTES); Query left = newBoxInternal(field, leftOpen, upper); q.add(new BooleanClause(left, BooleanClause.Occur.SHOULD)); - byte[][] rightOpen = new byte[2][]; - rightOpen[0] = upper[0]; + byte[] rightOpen = upper.clone(); // leave longitude open - rightOpen[1] = new byte[Integer.BYTES]; - NumericUtils.intToSortableBytes(Integer.MAX_VALUE, rightOpen[1], 0); + NumericUtils.intToSortableBytes(Integer.MAX_VALUE, rightOpen, Integer.BYTES); Query right = newBoxInternal(field, lower, rightOpen); q.add(new BooleanClause(right, BooleanClause.Occur.SHOULD)); return new ConstantScoreQuery(q.build()); @@ -281,8 +275,8 @@ public static Query newBoxQuery(String field, double minLatitude, double maxLati } } - private static Query newBoxInternal(String field, byte[][] min, byte[][] max) { - return new PointRangeQuery(field, min, max) { + private static Query newBoxInternal(String field, byte[] min, byte[] max) { + return new PointRangeQuery(field, min, max, 2) { @Override protected String toString(int dimension, byte[] value) { if (dimension == 0) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index a7a129551dbf..7d9281b77dab 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -96,16 +96,22 @@ public void testToString() throws Exception { } public void testQueryEquals() throws Exception { - Query q = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); - assertEquals(q, BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000))); - assertFalse(q.equals(BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(1000)))); + Query q1 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); + Query q2 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(1000)))); - q = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); - assertEquals(q, BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000))); - assertFalse(q.equals(BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1)))); + q1 = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); + q2 = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1)))); - q = BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000), BigInteger.valueOf(17)); - assertEquals(q, BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(17), BigInteger.valueOf(0), BigInteger.valueOf(1000))); - assertFalse(q.equals(BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(17), BigInteger.valueOf(1000)))); + q1 = BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000), BigInteger.valueOf(17)); + q2 = BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(17), BigInteger.valueOf(0), BigInteger.valueOf(1000)); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(BigIntegerPoint.newSetQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(17), BigInteger.valueOf(1000)))); } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index b0e7107fff1d..ee34ecc5b448 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -92,21 +92,29 @@ public void testToString() throws Exception { } public void testQueryEquals() throws Exception { - Query q = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); - assertEquals(q, InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5"))); - assertFalse(q.equals(InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); + Query q1 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + Query q2 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); - q = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); - assertEquals(q, InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16)); - assertFalse(q.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.1.3.5"), 16))); - assertFalse(q.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.5"), 24))); + q1 = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); + q2 = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.1.3.5"), 16))); + assertFalse(q1.equals(InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.5"), 24))); - q = InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3")); - assertEquals(q, InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3"))); - assertFalse(q.equals(InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.5")))); + q1 = InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3")); + q2 = InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.3")); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(InetAddressPoint.newExactQuery("a", InetAddress.getByName("1.2.3.5")))); - q = InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); - assertEquals(q, InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5"))); - assertFalse(q.equals(InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); + q1 = InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + q2 = InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(InetAddressPoint.newSetQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index ff4af122234f..1b6a6d23db1d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -186,21 +186,27 @@ public void testEncodeDecodeIsStable() throws Exception { } public void testQueryEquals() throws Exception { - Query q = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); - assertEquals(q, LatLonPoint.newBoxQuery("field", 50, 70, -40, 20)); - assertFalse(q.equals(LatLonPoint.newBoxQuery("field", 50, 70, -40, 10))); - - q = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); - assertEquals(q, LatLonPoint.newDistanceQuery("field", 50, 70, 10000)); - assertFalse(q.equals(LatLonPoint.newDistanceQuery("field", 50, 70, 11000))); - assertFalse(q.equals(LatLonPoint.newDistanceQuery("field", 50, 60, 10000))); + Query q1 = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); + Query q2 = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LatLonPoint.newBoxQuery("field", 50, 70, -40, 10))); + + q1 = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); + q2 = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LatLonPoint.newDistanceQuery("field", 50, 70, 11000))); + assertFalse(q1.equals(LatLonPoint.newDistanceQuery("field", 50, 60, 10000))); double[] polyLats1 = new double[] {30, 40, 40, 30, 30}; double[] polyLons1 = new double[] {90, 90, -40, -40, 90}; double[] polyLats2 = new double[] {20, 40, 40, 20, 20}; - q = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); - assertEquals(q, LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1)); - assertFalse(q.equals(LatLonPoint.newPolygonQuery("field", polyLats2, polyLons1))); + q1 = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); + q2 = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); + assertEquals(q1, q2); + assertEquals(q1.hashCode(), q2.hashCode()); + assertFalse(q1.equals(LatLonPoint.newPolygonQuery("field", polyLats2, polyLons1))); } } From 38673877aef4f89dbb437249eaf201f963477f80 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Sun, 20 Mar 2016 13:34:22 -0700 Subject: [PATCH 0172/1113] LUCENE-7118: Fix packed points upper/lower bound length check --- .../src/java/org/apache/lucene/search/PointRangeQuery.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index 37580e02b440..d798c2202b87 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -68,9 +68,12 @@ protected PointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, in if (lowerPoint.length % numDims != 0) { throw new IllegalArgumentException("lowerPoint is not a fixed multiple of numDims"); } - if (upperPoint.length != upperPoint.length) { + if (lowerPoint.length != upperPoint.length) { throw new IllegalArgumentException("lowerPoint has length=" + numDims + " but upperPoint has different length=" + upperPoint.length); } + if (numDims <= 0) { + throw new IllegalArgumentException("numDims must be positive, got " + numDims); + } this.numDims = numDims; this.bytesPerDim = lowerPoint.length / numDims; From ab821d3a087f400206f903800f1167679dccb9ac Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Sun, 20 Mar 2016 13:49:33 -0700 Subject: [PATCH 0173/1113] LUCENE-7118: Move numDims check before modulo numDims --- .../src/java/org/apache/lucene/search/PointRangeQuery.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index d798c2202b87..cfa644fea9b3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -62,6 +62,9 @@ public abstract class PointRangeQuery extends Query { protected PointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, int numDims) { checkArgs(field, lowerPoint, upperPoint); this.field = field; + if (numDims <= 0) { + throw new IllegalArgumentException("numDims must be positive, got " + numDims); + } if (lowerPoint.length == 0) { throw new IllegalArgumentException("lowerPoint has length of zero"); } @@ -71,9 +74,6 @@ protected PointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, in if (lowerPoint.length != upperPoint.length) { throw new IllegalArgumentException("lowerPoint has length=" + numDims + " but upperPoint has different length=" + upperPoint.length); } - if (numDims <= 0) { - throw new IllegalArgumentException("numDims must be positive, got " + numDims); - } this.numDims = numDims; this.bytesPerDim = lowerPoint.length / numDims; From 2668ff5abb16e867f9f770d1da65457161b52cda Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Sun, 20 Mar 2016 16:55:12 -0400 Subject: [PATCH 0174/1113] SOLR-8082: Can't query against negative float or double values when indexed='false' docValues='true' multiValued='false' --- solr/CHANGES.txt | 3 + .../org/apache/solr/schema/TrieField.java | 63 ++- .../function/ValueSourceRangeFilter.java | 6 +- .../collection1/conf/schema-docValues.xml | 2 +- .../solr/schema/DocValuesMultiTest.java | 71 ++- .../org/apache/solr/schema/DocValuesTest.java | 411 +++++++++++++++--- 6 files changed, 490 insertions(+), 66 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 23b4a385c265..b386f6a8e391 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -362,6 +362,9 @@ Optimizations * SOLR-8720: ZkController#publishAndWaitForDownStates should use #publishNodeAsDown. (Mark Miller) +* SOLR-8082: Can't query against negative float or double values when indexed="false" + docValues="true" multiValued="false". (hossman, Ishan Chattopadhyaya, yonik, Steve Rowe) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index c4899a106045..1580b0037edc 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -40,10 +40,13 @@ import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanQuery.Builder; import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.SortField; import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.lucene.util.BytesRef; @@ -56,7 +59,9 @@ import org.apache.lucene.util.mutable.MutableValueLong; import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.FunctionRangeQuery; import org.apache.solr.search.QParser; +import org.apache.solr.search.function.ValueSourceRangeFilter; import org.apache.solr.util.DateFormatUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -351,10 +356,7 @@ public Query getRangeQuery(QParser parser, SchemaField field, String min, String break; case FLOAT: if (matchOnly) { - query = DocValuesRangeQuery.newLongRange(field.getName(), - min == null ? null : (long) NumericUtils.floatToSortableInt(Float.parseFloat(min)), - max == null ? null : (long) NumericUtils.floatToSortableInt(Float.parseFloat(max)), - minInclusive, maxInclusive); + return getRangeQueryForFloatDoubleDocValues(field, min, max, minInclusive, maxInclusive); } else { query = LegacyNumericRangeQuery.newFloatRange(field.getName(), ps, min == null ? null : Float.parseFloat(min), @@ -377,10 +379,7 @@ public Query getRangeQuery(QParser parser, SchemaField field, String min, String break; case DOUBLE: if (matchOnly) { - query = DocValuesRangeQuery.newLongRange(field.getName(), - min == null ? null : NumericUtils.doubleToSortableLong(Double.parseDouble(min)), - max == null ? null : NumericUtils.doubleToSortableLong(Double.parseDouble(max)), - minInclusive, maxInclusive); + return getRangeQueryForFloatDoubleDocValues(field, min, max, minInclusive, maxInclusive); } else { query = LegacyNumericRangeQuery.newDoubleRange(field.getName(), ps, min == null ? null : Double.parseDouble(min), @@ -407,7 +406,53 @@ public Query getRangeQuery(QParser parser, SchemaField field, String min, String return query; } - + + private static long FLOAT_NEGATIVE_INFINITY_BITS = (long)Float.floatToIntBits(Float.NEGATIVE_INFINITY); + private static long DOUBLE_NEGATIVE_INFINITY_BITS = Double.doubleToLongBits(Double.NEGATIVE_INFINITY); + private static long FLOAT_POSITIVE_INFINITY_BITS = (long)Float.floatToIntBits(Float.POSITIVE_INFINITY); + private static long DOUBLE_POSITIVE_INFINITY_BITS = Double.doubleToLongBits(Double.POSITIVE_INFINITY); + private static long FLOAT_MINUS_ZERO_BITS = (long)Float.floatToIntBits(-0f); + private static long DOUBLE_MINUS_ZERO_BITS = Double.doubleToLongBits(-0d); + private static long FLOAT_ZERO_BITS = (long)Float.floatToIntBits(0f); + private static long DOUBLE_ZERO_BITS = Double.doubleToLongBits(0d); + + private Query getRangeQueryForFloatDoubleDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) { + Query query; + String fieldName = sf.getName(); + + Number minVal = min == null ? null : type == TrieTypes.FLOAT ? Float.parseFloat(min): Double.parseDouble(min); + Number maxVal = max == null ? null : type == TrieTypes.FLOAT ? Float.parseFloat(max): Double.parseDouble(max); + + Long minBits = + min == null ? null : type == TrieTypes.FLOAT ? (long) Float.floatToIntBits(minVal.floatValue()): Double.doubleToLongBits(minVal.doubleValue()); + Long maxBits = + max == null ? null : type == TrieTypes.FLOAT ? (long) Float.floatToIntBits(maxVal.floatValue()): Double.doubleToLongBits(maxVal.doubleValue()); + + long negativeInfinityBits = type == TrieTypes.FLOAT ? FLOAT_NEGATIVE_INFINITY_BITS : DOUBLE_NEGATIVE_INFINITY_BITS; + long positiveInfinityBits = type == TrieTypes.FLOAT ? FLOAT_POSITIVE_INFINITY_BITS : DOUBLE_POSITIVE_INFINITY_BITS; + long minusZeroBits = type == TrieTypes.FLOAT ? FLOAT_MINUS_ZERO_BITS : DOUBLE_MINUS_ZERO_BITS; + long zeroBits = type == TrieTypes.FLOAT ? FLOAT_ZERO_BITS : DOUBLE_ZERO_BITS; + + // If min is negative (or -0d) and max is positive (or +0d), then issue a FunctionRangeQuery + if ((minVal == null || minVal.doubleValue() < 0d || minBits == minusZeroBits) && + (maxVal == null || (maxVal.doubleValue() > 0d || maxBits == zeroBits))) { + + ValueSource vs = getValueSource(sf, null); + query = new FunctionRangeQuery(new ValueSourceRangeFilter(vs, min, max, minInclusive, maxInclusive)); + + } else { // If both max and min are negative (or -0d), then issue range query with max and min reversed + if ((minVal == null || minVal.doubleValue() < 0d || minBits == minusZeroBits) && + (maxVal != null && (maxVal.doubleValue() < 0d || maxBits == minusZeroBits))) { + query = DocValuesRangeQuery.newLongRange + (fieldName, maxBits, (min == null ? negativeInfinityBits : minBits), maxInclusive, minInclusive); + } else { // If both max and min are positive, then issue range query + query = DocValuesRangeQuery.newLongRange + (fieldName, minBits, (max == null ? positiveInfinityBits : maxBits), minInclusive, maxInclusive); + } + } + return query; + } + @Override public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { if (!field.indexed() && field.hasDocValues()) { diff --git a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java index 64211d2fb997..1fce97e957c2 100644 --- a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java +++ b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java @@ -48,8 +48,10 @@ public ValueSourceRangeFilter(ValueSource valueSource, this.valueSource = valueSource; this.lowerVal = lowerVal; this.upperVal = upperVal; - this.includeLower = lowerVal != null && includeLower; - this.includeUpper = upperVal != null && includeUpper; + this.includeLower = includeLower; + this.includeUpper = includeUpper; +// this.includeLower = lowerVal != null && includeLower; +// this.includeUpper = upperVal != null && includeUpper; } public ValueSource getValueSource() { diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml index 851ff658a9b9..cbbdf6ed0ae5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -58,7 +58,7 @@ - + diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java index b544d5311dda..7b4a5db3ea80 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java @@ -33,6 +33,18 @@ public class DocValuesMultiTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeTests() throws Exception { initCore("solrconfig-basic.xml", "schema-docValuesMulti.xml"); + + // sanity check our schema meets our expectations + final IndexSchema schema = h.getCore().getLatestSchema(); + for (String f : new String[] {"floatdv", "intdv", "doubledv", "longdv", "datedv", "stringdv"}) { + final SchemaField sf = schema.getField(f); + assertTrue(f + " is not multiValued, test is useless, who changed the schema?", + sf.multiValued()); + assertFalse(f + " is indexed, test is useless, who changed the schema?", + sf.indexed()); + assertTrue(f + " has no docValues, test is useless, who changed the schema?", + sf.hasDocValues()); + } } public void setUp() throws Exception { @@ -117,9 +129,10 @@ public void testStringDocValuesMatch() throws Exception { */ public void testFloatDocValuesMatch() throws Exception { assertU(adoc("id", "1", "floatdv", "2")); - assertU(adoc("id", "2", "floatdv", "5")); + assertU(adoc("id", "2", "floatdv", "-5")); assertU(adoc("id", "3", "floatdv", "3.0", "floatdv", "-1.3", "floatdv", "2.2")); assertU(adoc("id", "4", "floatdv", "3")); + assertU(adoc("id", "5", "floatdv", "-0.5")); assertU(commit()); // float: termquery @@ -131,10 +144,62 @@ public void testFloatDocValuesMatch() throws Exception { // float: rangequery assertQ(req("q", "floatdv:[-1 TO 2.5]", "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" + ); + + // (neg) float: rangequery + assertQ(req("q", "floatdv:[-6 TO -4]", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/str[@name='id'][.=2]" + ); + + // (neg) float: termquery + assertQ(req("q", "floatdv:\"-5\"", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/str[@name='id'][.=2]" + ); + } + + /** Tests the ability to do basic queries (without scoring, just match-only) on + * double docvalues fields that are not inverted (indexed "forward" only) + */ + public void testDoubleDocValuesMatch() throws Exception { + assertU(adoc("id", "1", "doubledv", "2")); + assertU(adoc("id", "2", "doubledv", "-5")); + assertU(adoc("id", "3", "doubledv", "3.0", "doubledv", "-1.3", "doubledv", "2.2")); + assertU(adoc("id", "4", "doubledv", "3")); + assertU(adoc("id", "5", "doubledv", "-0.5")); + assertU(commit()); + + // double: termquery + assertQ(req("q", "doubledv:3", "sort", "id asc"), "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=1]", - "//result/doc[2]/str[@name='id'][.=3]" + "//result/doc[1]/str[@name='id'][.=3]", + "//result/doc[2]/str[@name='id'][.=4]" ); + + // double: rangequery + assertQ(req("q", "doubledv:[-1 TO 2.5]", "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.=1]", + "//result/doc[2]/str[@name='id'][.=3]", + "//result/doc[3]/str[@name='id'][.=5]" + ); + + // (neg) double: rangequery + assertQ(req("q", "doubledv:[-6 TO -4]", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/str[@name='id'][.=2]" + ); + + // (neg) double: termquery + assertQ(req("q", "doubledv:\"-5\"", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/str[@name='id'][.=2]" + ); } public void testDocValuesFacetingSimple() { diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java index a9aa401cbf1b..0c3b0027b293 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.util.NumericUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrCore; import org.apache.solr.search.SolrIndexSearcher; @@ -27,12 +28,32 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Logger; public class DocValuesTest extends SolrTestCaseJ4 { + protected Logger log = Logger.getLogger(getClass().getName()); + @BeforeClass public static void beforeTests() throws Exception { initCore("solrconfig-basic.xml", "schema-docValues.xml"); + + // sanity check our schema meets our expectations + final IndexSchema schema = h.getCore().getLatestSchema(); + for (String f : new String[] {"floatdv", "intdv", "doubledv", "longdv", "datedv", "stringdv"}) { + final SchemaField sf = schema.getField(f); + assertFalse(f + " is multiValued, test is useless, who changed the schema?", + sf.multiValued()); + assertFalse(f + " is indexed, test is useless, who changed the schema?", + sf.indexed()); + assertTrue(f + " has no docValues, test is useless, who changed the schema?", + sf.hasDocValues()); + } } public void setUp() throws Exception { @@ -92,29 +113,29 @@ public void testDocValuesSorting() { assertU(adoc("id", "4")); assertU(commit()); assertQ(req("q", "*:*", "sort", "floatdv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='2']"); + "//int[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "intdv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='2']"); + "//int[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "doubledv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='1']"); + "//int[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "longdv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='1']"); + "//int[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "datedv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='2']"); + "//int[@name='id'][.='2']"); assertQ(req("q", "*:*", "sort", "stringdv desc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='4']"); + "//int[@name='id'][.='4']"); assertQ(req("q", "*:*", "sort", "floatdv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='4']"); + "//int[@name='id'][.='4']"); assertQ(req("q", "*:*", "sort", "intdv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='3']"); + "//int[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "doubledv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='3']"); + "//int[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "longdv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='3']"); + "//int[@name='id'][.='3']"); assertQ(req("q", "*:*", "sort", "datedv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='1']"); + "//int[@name='id'][.='1']"); assertQ(req("q", "*:*", "sort", "stringdv asc", "rows", "1", "fl", "id"), - "//str[@name='id'][.='2']"); + "//int[@name='id'][.='2']"); } public void testDocValuesSorting2() { @@ -122,18 +143,18 @@ public void testDocValuesSorting2() { assertU(adoc("id", "2", "doubledv", "50.567")); assertU(adoc("id", "3", "doubledv", "+0")); assertU(adoc("id", "4", "doubledv", "4.9E-324")); - assertU(adoc("id", "5", "doubledv", "-0")); + assertU(adoc("id", "5", "doubledv", "-0.1")); assertU(adoc("id", "6", "doubledv", "-5.123")); assertU(adoc("id", "7", "doubledv", "1.7976931348623157E308")); assertU(commit()); assertQ(req("fl", "id", "q", "*:*", "sort", "doubledv asc"), - "//result/doc[1]/str[@name='id'][.='6']", - "//result/doc[2]/str[@name='id'][.='5']", - "//result/doc[3]/str[@name='id'][.='3']", - "//result/doc[4]/str[@name='id'][.='4']", - "//result/doc[5]/str[@name='id'][.='1']", - "//result/doc[6]/str[@name='id'][.='2']", - "//result/doc[7]/str[@name='id'][.='7']" + "//result/doc[1]/int[@name='id'][.='6']", + "//result/doc[2]/int[@name='id'][.='5']", + "//result/doc[3]/int[@name='id'][.='3']", + "//result/doc[4]/int[@name='id'][.='4']", + "//result/doc[5]/int[@name='id'][.='1']", + "//result/doc[6]/int[@name='id'][.='2']", + "//result/doc[7]/int[@name='id'][.='7']" ); } @@ -247,87 +268,375 @@ public void testDocValuesStats() { * docvalues fields that are not inverted (indexed "forward" only) */ public void testDocValuesMatch() throws Exception { - assertU(adoc("id", "1", "floatdv", "2", "intdv", "3", "doubledv", "4", "longdv", "5", "datedv", "1995-12-31T23:59:59.999Z", "stringdv", "b")); - assertU(adoc("id", "2", "floatdv", "5", "intdv", "4", "doubledv", "3", "longdv", "2", "datedv", "1997-12-31T23:59:59.999Z", "stringdv", "a")); - assertU(adoc("id", "3", "floatdv", "3", "intdv", "1", "doubledv", "2", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "c")); - assertU(adoc("id", "4", "floatdv", "3", "intdv", "1", "doubledv", "2", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "car")); + assertU(adoc("id", "1", "floatdv", "2", "intdv", "3", "doubledv", "3.1", "longdv", "5", "datedv", "1995-12-31T23:59:59.999Z", "stringdv", "b")); + assertU(adoc("id", "2", "floatdv", "-5", "intdv", "4", "doubledv", "-4.3", "longdv", "2", "datedv", "1997-12-31T23:59:59.999Z", "stringdv", "a")); + assertU(adoc("id", "3", "floatdv", "3", "intdv", "1", "doubledv", "2.1", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "c")); + assertU(adoc("id", "4", "floatdv", "3", "intdv", "-1", "doubledv", "1.5", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "car")); assertU(commit()); - + // string: termquery assertQ(req("q", "stringdv:car", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=4]" ); // string: range query assertQ(req("q", "stringdv:[b TO d]", "sort", "id asc"), "//*[@numFound='3']", - "//result/doc[1]/str[@name='id'][.=1]", - "//result/doc[2]/str[@name='id'][.=3]", - "//result/doc[3]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=1]", + "//result/doc[2]/int[@name='id'][.=3]", + "//result/doc[3]/int[@name='id'][.=4]" ); // string: prefix query assertQ(req("q", "stringdv:c*", "sort", "id asc"), "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=3]", - "//result/doc[2]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=3]", + "//result/doc[2]/int[@name='id'][.=4]" ); // string: wildcard query assertQ(req("q", "stringdv:c?r", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=4]" ); // string: regexp query assertQ(req("q", "stringdv:/c[a-b]r/", "sort", "id asc"), "//*[@numFound='1']", - "//result/doc[1]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=4]" ); // float: termquery assertQ(req("q", "floatdv:3", "sort", "id asc"), "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=3]", - "//result/doc[2]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=3]", + "//result/doc[2]/int[@name='id'][.=4]" ); // float: rangequery assertQ(req("q", "floatdv:[2 TO 3]", "sort", "id asc"), "//*[@numFound='3']", - "//result/doc[1]/str[@name='id'][.=1]", - "//result/doc[2]/str[@name='id'][.=3]", - "//result/doc[3]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=1]", + "//result/doc[2]/int[@name='id'][.=3]", + "//result/doc[3]/int[@name='id'][.=4]" ); + // (neg) float: termquery + assertQ(req("q", "floatdv:\"-5\"", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=2]" + ); + + // (neg) float: rangequery + assertQ(req("q", "floatdv:[-6 TO -4]", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=2]" + ); + + // (cross zero bounds) float: rangequery + assertQ(req("q", "floatdv:[-6 TO 2.1]", "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=1]", + "//result/doc[2]/int[@name='id'][.=2]" + ); + // int: termquery assertQ(req("q", "intdv:1", "sort", "id asc"), - "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=3]", - "//result/doc[2]/str[@name='id'][.=4]" - ); + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=3]" + ); // int: rangequery assertQ(req("q", "intdv:[3 TO 4]", "sort", "id asc"), - "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=1]", - "//result/doc[2]/str[@name='id'][.=2]" - ); + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=1]", + "//result/doc[2]/int[@name='id'][.=2]" + ); + + // (neg) int: termquery + assertQ(req("q", "intdv:\"-1\"", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=4]" + ); + // (neg) int: rangequery + assertQ(req("q", "intdv:[-1 TO 1]", "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=3]", + "//result/doc[2]/int[@name='id'][.=4]" + ); + // long: termquery assertQ(req("q", "longdv:1", "sort", "id asc"), "//*[@numFound='2']", - "//result/doc[1]/str[@name='id'][.=3]", - "//result/doc[2]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=3]", + "//result/doc[2]/int[@name='id'][.=4]" ); // long: rangequery assertQ(req("q", "longdv:[1 TO 2]", "sort", "id asc"), "//*[@numFound='3']", - "//result/doc[1]/str[@name='id'][.=2]", - "//result/doc[2]/str[@name='id'][.=3]", - "//result/doc[3]/str[@name='id'][.=4]" + "//result/doc[1]/int[@name='id'][.=2]", + "//result/doc[2]/int[@name='id'][.=3]", + "//result/doc[3]/int[@name='id'][.=4]" ); + + // double: termquery + assertQ(req("q", "doubledv:3.1", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=1]" + ); + + // double: rangequery + assertQ(req("q", "doubledv:[2 TO 3.3]", "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=1]", + "//result/doc[2]/int[@name='id'][.=3]" + ); + + // (neg) double: termquery + assertQ(req("q", "doubledv:\"-4.3\"", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=2]" + ); + + // (neg) double: rangequery + assertQ(req("q", "doubledv:[-6 TO -4]", "sort", "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/int[@name='id'][.=2]" + ); + + // (cross zero bounds) double: rangequery + assertQ(req("q", "doubledv:[-6 TO 2.0]", "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=2]", + "//result/doc[2]/int[@name='id'][.=4]" + ); + } + + public void testFloatAndDoubleRangeQueryRandom() throws Exception { + + String fieldName[] = new String[] {"floatdv", "doubledv"}; + + Number largestNegative[] = new Number[] {0f-Float.MIN_NORMAL, 0f-Double.MIN_NORMAL}; + Number smallestPositive[] = new Number[] {Float.MIN_NORMAL, Double.MIN_NORMAL}; + Number positiveInfinity[] = new Number[] {Float.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}; + Number negativeInfinity[] = new Number[] {Float.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}; + Number largestValue[] = new Number[] {Float.MAX_VALUE, Double.MAX_VALUE}; + Number zero[] = new Number[] {0f, 0d}; + Function,Number> noNaN = (next) + -> { Number num; while (String.valueOf(num = next.get()).equals("NaN")); return num; }; + List> nextRandNoNaN = Arrays.asList( + () -> noNaN.apply(() -> Float.intBitsToFloat(random().nextInt())), + () -> noNaN.apply(() -> Double.longBitsToDouble(random().nextLong()))); + List> toSortableLong = Arrays.asList( + (num) -> (long)NumericUtils.floatToSortableInt(num.floatValue()), + (num) -> NumericUtils.doubleToSortableLong(num.doubleValue())); + + // Number minusZero[] = new Number[] {-0f, -0d}; // -0 == 0, so we should not treat them differently (and we should not guarantee that sign is preserved... we should be able to index both as 0) + + for (int i=0; i values = new ArrayList<>(); + int numDocs = 1 + random().nextInt(10); + for (int j=0; j tests = new ArrayList<>(); + int counter = 0; + + for (int k=0; k=valSortable || !maxInclusive && maxSortable>valSortable)) { + counter++; + tests.add("//result/doc["+counter+"]/int[@name='id'][.="+(k+1)+"]"); + tests.add("//result/doc["+counter+"]/float[@name='score'][.=1.0]"); + } + } + + tests.add(0, "//*[@numFound='"+counter+"']"); + + String testsArr[] = new String[tests.size()]; + for (int k=0; k Date: Mon, 21 Mar 2016 00:09:15 +0100 Subject: [PATCH 0175/1113] SOLR-8082: Fix forbidden APIs --- .../core/src/test/org/apache/solr/schema/DocValuesTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java index 0c3b0027b293..f1d0196c2262 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -26,18 +26,20 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.function.Supplier; -import java.util.logging.Logger; public class DocValuesTest extends SolrTestCaseJ4 { - protected Logger log = Logger.getLogger(getClass().getName()); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass public static void beforeTests() throws Exception { From 551cf750569d41103207d607c497233f436c32cc Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 21 Mar 2016 05:49:47 -0400 Subject: [PATCH 0176/1113] LUCENE-7120: re-use readers while recursing in BKDWriter so we get more thorough checksum verification on its temp files --- .../org/apache/lucene/util/bkd/BKDWriter.java | 47 +++++++++++-------- .../lucene/util/bkd/HeapPointWriter.java | 6 +++ .../lucene/util/bkd/OfflinePointReader.java | 5 ++ .../lucene/util/bkd/OfflinePointWriter.java | 37 ++++++++++++++- .../apache/lucene/util/bkd/PointWriter.java | 4 ++ 5 files changed, 78 insertions(+), 21 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 488fa43af8b4..5f36155efab5 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -227,7 +227,7 @@ public static void verifyParams(int numDims, int maxPointsInLeafNode, double max private void spillToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: - offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill", singleValuePerDoc); + offlinePointWriter = new OfflinePointWriter(tempDir, tempFileNamePrefix, packedBytesLength, longOrds, "spill", 0, singleValuePerDoc); tempInput = offlinePointWriter.out; PointReader reader = heapPointWriter.getReader(0, pointCount); for(int i=0;i toCloseHeroically = new ArrayList<>(); + boolean success = false; try { //long t0 = System.nanoTime(); @@ -872,7 +875,8 @@ public long finish(IndexOutput out) throws IOException { ordBitSet, out, minPackedValue, maxPackedValue, splitPackedValues, - leafBlockFPs); + leafBlockFPs, + toCloseHeroically); for(PathSlice slice : sortedPointWriters) { slice.writer.destroy(); @@ -886,11 +890,8 @@ public long finish(IndexOutput out) throws IOException { success = true; } finally { if (success == false) { - if (tempInput != null) { - IOUtils.closeWhileHandlingException(tempInput); - } IOUtils.deleteFilesIgnoringExceptions(tempDir, tempDir.getCreatedFiles()); - tempInput = null; + IOUtils.closeWhileHandlingException(toCloseHeroically); } } @@ -1010,6 +1011,8 @@ private byte[] markRightTree(long rightCount, int splitDim, PathSlice source, Lo // Now we mark ords that fall into the right half, so we can partition on all other dims that are not the split dim: // Read the split value, then mark all ords in the right tree (larger than the split value): + + // TODO: find a way to also checksum this reader? If we changed to markLeftTree, and scanned the final chunk, it could work? try (PointReader reader = source.writer.getReader(source.start + source.count - rightCount, rightCount)) { boolean result = reader.next(); assert result; @@ -1069,12 +1072,11 @@ private int split(byte[] minPackedValue, byte[] maxPackedValue) { } /** Pull a partition back into heap once the point count is low enough while recursing. */ - private PathSlice switchToHeap(PathSlice source) throws IOException { + private PathSlice switchToHeap(PathSlice source, List toCloseHeroically) throws IOException { int count = Math.toIntExact(source.count); - try ( - PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds, singleValuePerDoc); - PointReader reader = source.writer.getReader(source.start, source.count); - ) { + // Not inside the try because we don't want to close it here: + PointReader reader = source.writer.getSharedReader(source.start, source.count, toCloseHeroically); + try (PointWriter writer = new HeapPointWriter(count, count, packedBytesLength, longOrds, singleValuePerDoc)) { for(int i=0;i toCloseHeroically) throws IOException { for(PathSlice slice : slices) { assert slice.count == slices[0].count; @@ -1104,7 +1107,7 @@ private void build(int nodeID, int leafNodeOffset, if (numDims == 1 && slices[0].writer instanceof OfflinePointWriter && slices[0].count <= maxPointsSortInHeap) { // Special case for 1D, to cutover to heap once we recurse deeply enough: - slices[0] = switchToHeap(slices[0]); + slices[0] = switchToHeap(slices[0], toCloseHeroically); } if (nodeID >= leafNodeOffset) { @@ -1114,7 +1117,7 @@ private void build(int nodeID, int leafNodeOffset, if (slices[dim].writer instanceof HeapPointWriter == false) { // Adversarial cases can cause this, e.g. very lopsided data, all equal points, such that we started // offline, but then kept splitting only in one dimension, and so never had to rewrite into heap writer - slices[dim] = switchToHeap(slices[dim]); + slices[dim] = switchToHeap(slices[dim], toCloseHeroically); } PathSlice source = slices[dim]; @@ -1212,7 +1215,8 @@ private void build(int nodeID, int leafNodeOffset, for(int dim=0;dim toCloseHeroically) { + return new HeapPointReader(blocks, valuesPerBlock, packedBytesLength, ords, ordsLong, docIDs, (int) start, nextWrite, singleValuePerDoc); + } + @Override public void close() { closed = true; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 18afda4659ce..ebb2fd14a7f0 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -39,6 +39,9 @@ final class OfflinePointReader extends PointReader { private boolean longOrds; private boolean checked; + // File name we are reading + final String name; + OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, boolean longOrds, boolean singleValuePerDoc) throws IOException { this.singleValuePerDoc = singleValuePerDoc; @@ -67,6 +70,7 @@ final class OfflinePointReader extends PointReader { // at another level of the BKDWriter recursion in = tempDir.openInput(tempFileName, IOContext.READONCE); } + name = tempFileName; long seekFP = start * bytesPerDoc; in.seek(seekFP); @@ -121,6 +125,7 @@ public int docID() { public void close() throws IOException { try { if (countLeft == 0 && in instanceof ChecksumIndexInput && checked == false) { + //System.out.println("NOW CHECK: " + name); checked = true; CodecUtil.checkFooter((ChecksumIndexInput) in); } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 53148769fe8d..2261b81380d8 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -16,9 +16,12 @@ */ package org.apache.lucene.util.bkd; +import java.io.Closeable; import java.io.IOException; +import java.util.List; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -34,13 +37,19 @@ final class OfflinePointWriter implements PointWriter { private boolean closed; // true if ords are written as long (8 bytes), else 4 bytes private boolean longOrds; + private OfflinePointReader sharedReader; + private long nextSharedRead; + final long expectedCount; - public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds, String desc, boolean singleValuePerDoc) throws IOException { + /** Create a new writer with an unknown number of incoming points */ + public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, + boolean longOrds, String desc, long expectedCount, boolean singleValuePerDoc) throws IOException { this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT); this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.longOrds = longOrds; this.singleValuePerDoc = singleValuePerDoc; + this.expectedCount = expectedCount; } /** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */ @@ -52,6 +61,7 @@ public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLen closed = true; this.longOrds = longOrds; this.singleValuePerDoc = singleValuePerDoc; + this.expectedCount = 0; } @Override @@ -68,18 +78,37 @@ public void append(byte[] packedValue, long ord, int docID) throws IOException { } out.writeInt(docID); count++; + assert expectedCount == 0 || count <= expectedCount; } @Override public PointReader getReader(long start, long length) throws IOException { assert closed; assert start + length <= count: "start=" + start + " length=" + length + " count=" + count; + assert expectedCount == 0 || count == expectedCount; return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, length, longOrds, singleValuePerDoc); } + @Override + public PointReader getSharedReader(long start, long length, List toCloseHeroically) throws IOException { + if (sharedReader == null) { + assert start == 0; + assert length <= count; + sharedReader = new OfflinePointReader(tempDir, out.getName(), packedBytesLength, 0, count, longOrds, singleValuePerDoc); + toCloseHeroically.add(sharedReader); + // Make sure the OfflinePointReader intends to verify its checksum: + assert sharedReader.in instanceof ChecksumIndexInput; + } else { + assert start == nextSharedRead: "start=" + start + " length=" + length + " nextSharedRead=" + nextSharedRead; + } + nextSharedRead += length; + return sharedReader; + } + @Override public void close() throws IOException { if (closed == false) { + assert sharedReader == null; try { CodecUtil.writeFooter(out); } finally { @@ -91,6 +120,12 @@ public void close() throws IOException { @Override public void destroy() throws IOException { + if (sharedReader != null) { + // At this point, the shared reader should have done a full sweep of the file: + assert nextSharedRead == count; + sharedReader.close(); + sharedReader = null; + } tempDir.deleteFile(out.getName()); } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/PointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/PointWriter.java index 2f94967cb8a2..d19f6e547b2c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/PointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/PointWriter.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.List; /** Appends many points, and then at the end provides a {@link PointReader} to iterate * those points. This abstracts away whether we write to disk, or use simple arrays @@ -30,6 +31,9 @@ interface PointWriter extends Closeable { /** Returns a {@link PointReader} iterator to step through all previously added points */ PointReader getReader(long startPoint, long length) throws IOException; + /** Returns the single shared reader, used at multiple times during the recursion, to read previously added points */ + PointReader getSharedReader(long startPoint, long length, List toCloseHeroically) throws IOException; + /** Removes any temp files behind this writer */ void destroy() throws IOException; } From 4148cdf7446dca03e59cd5662f49b8286b3ba856 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Mon, 21 Mar 2016 08:59:43 -0400 Subject: [PATCH 0177/1113] LUCENE-7111: DocValuesRangeQuery.newLongRange behaves incorrectly for Long.MAX_VALUE and Long.MIN_VALUE --- lucene/CHANGES.txt | 3 ++ .../lucene/search/DocValuesRangeQuery.java | 6 ++++ .../search/TestDocValuesRangeQuery.java | 34 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 9d4bfa6ee653..78d7aa3c0e58 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -201,6 +201,9 @@ Bug Fixes * LUCENE-7101: OfflineSorter had O(N^2) merge cost, and used too many temporary file descriptors, for large sorts (Mike McCandless) +* LUCENE-7111: DocValuesRangeQuery.newLongRange behaves incorrectly for + Long.MAX_VALUE and Long.MIN_VALUE (Ishan Chattopadhyaya via Steve Rowe) + Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java index cbdd9ba33774..b1aca4bdc8b2 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/DocValuesRangeQuery.java @@ -157,6 +157,9 @@ protected Bits getMatchingDocs(LeafReaderContext context) throws IOException { } else if (includeLower) { min = (long) lowerVal; } else { + if ((long) lowerVal == Long.MAX_VALUE) { + return null; + } min = 1 + (long) lowerVal; } @@ -166,6 +169,9 @@ protected Bits getMatchingDocs(LeafReaderContext context) throws IOException { } else if (includeUpper) { max = (long) upperVal; } else { + if ((long) upperVal == Long.MIN_VALUE) { + return null; + } max = -1 + (long) upperVal; } diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java index 2f79679189b5..c5ca64f3ae7c 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java @@ -270,4 +270,38 @@ public void testDocValuesRangeSupportsApproximation() throws IOException { dir.close(); } + public void testLongRangeBoundaryValues() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("dv", 100l)); + iw.addDocument(doc); + + doc = new Document(); + doc.add(new SortedNumericDocValuesField("dv", 200l)); + iw.addDocument(doc); + + iw.commit(); + + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader, false); + iw.close(); + + Long min = Long.MIN_VALUE; + Long max = Long.MIN_VALUE; + Query query = DocValuesRangeQuery.newLongRange("dv", min, max, true, false); + TopDocs td = searcher.search(query, searcher.reader.maxDoc(), Sort.INDEXORDER); + assertEquals(0, td.totalHits); + + min = Long.MAX_VALUE; + max = Long.MAX_VALUE; + query = DocValuesRangeQuery.newLongRange("dv", min, max, false, true); + td = searcher.search(query, searcher.reader.maxDoc(), Sort.INDEXORDER); + assertEquals(0, td.totalHits); + + reader.close(); + dir.close(); + } + } From 0b221826a397eb54b8c9ace3c4d940818d374b28 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Mar 2016 10:15:54 -0400 Subject: [PATCH 0178/1113] LUCENE-7123: deduplicate/cleanup spatial distance --- .../org/apache/lucene/util/SloppyMath.java | 120 +++++++----------- .../apache/lucene/util/TestSloppyMath.java | 97 ++++++++++---- .../demo/facet/DistanceFacetsExample.java | 3 +- .../js/JavascriptCompiler.properties | 2 +- .../expressions/TestDemoExpressions.java | 6 +- .../js/TestJavascriptFunction.java | 2 +- .../LatLonPointDistanceComparator.java | 22 +--- .../document/LatLonPointDistanceQuery.java | 19 ++- .../TestLatLonPointDistanceQuery.java | 59 ++++++++- .../document/TestLatLonPointDistanceSort.java | 34 +---- .../lucene/search/TestLatLonPointQueries.java | 8 +- .../search/GeoPointDistanceQuery.java | 2 +- .../search/GeoPointDistanceQueryImpl.java | 4 +- .../search/GeoPointInBBoxQueryImpl.java | 2 +- .../search/GeoPointMultiTermQuery.java | 2 +- .../lucene/spatial/util/GeoDistanceUtils.java | 32 +---- .../spatial/util/GeoProjectionUtils.java | 34 ++--- .../lucene/spatial/util/GeoRelationUtils.java | 64 +++++----- .../apache/lucene/spatial/util/GeoUtils.java | 46 ++++++- .../geopoint/search/TestGeoPointQuery.java | 6 +- .../spatial/util/BaseGeoPointTestCase.java | 3 +- .../lucene/spatial/util/TestGeoUtils.java | 5 +- 22 files changed, 311 insertions(+), 261 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java index 8da8276de35a..e3ba489cfd94 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java +++ b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java @@ -32,6 +32,32 @@ /** Math functions that trade off accuracy for speed. */ public class SloppyMath { + /** + * Returns the Haversine distance in meters between two points + * specified in decimal degrees (latitude/longitude). This works correctly + * even if the dateline is between the two points. + *

    + * Error is around 1E-5 (0.01mm) from the actual haversine distance. + * + * @param lat1 Latitude of the first point. + * @param lon1 Longitude of the first point. + * @param lat2 Latitude of the second point. + * @param lon2 Longitude of the second point. + * @return distance in meters. + */ + public static double haversinMeters(double lat1, double lon1, double lat2, double lon2) { + return haversinMeters(haversinSortKey(lat1, lon1, lat2, lon2)); + } + + /** + * Returns the Haversine distance in meters between two points + * given the previous result from {@link #haversinSortKey(double, double, double, double)} + * @return distance in meters. + */ + public static double haversinMeters(double sortKey) { + return TO_METERS * 2 * asin(Math.min(1, Math.sqrt(sortKey * 0.5))); + } + /** * Returns the Haversine distance in kilometers between two points * specified in decimal degrees (latitude/longitude). This works correctly @@ -42,19 +68,26 @@ public class SloppyMath { * @param lat2 Latitude of the second point. * @param lon2 Longitude of the second point. * @return distance in kilometers. + * @deprecated Use {@link #haversinMeters(double, double, double, double) instead} */ - public static double haversin(double lat1, double lon1, double lat2, double lon2) { + @Deprecated + public static double haversinKilometers(double lat1, double lon1, double lat2, double lon2) { + double h = haversinSortKey(lat1, lon1, lat2, lon2); + return TO_KILOMETERS * 2 * asin(Math.min(1, Math.sqrt(h * 0.5))); + } + + /** + * Returns a sort key for distance. This is less expensive to compute than + * {@link #haversinMeters(double, double, double, double)}, but it always compares the same. + * This can be converted into an actual distance with {@link #haversinMeters(double)}, which + * effectively does the second half of the computation. + */ + public static double haversinSortKey(double lat1, double lon1, double lat2, double lon2) { double x1 = lat1 * TO_RADIANS; double x2 = lat2 * TO_RADIANS; double h1 = 1 - cos(x1 - x2); double h2 = 1 - cos((lon1 - lon2) * TO_RADIANS); - double h = (h1 + cos(x1) * cos(x2) * h2) / 2; - - double avgLat = (x1 + x2) / 2d; - double diameter = earthDiameter(avgLat); - - return diameter * asin(Math.min(1, Math.sqrt(h))); - + return h1 + cos(x1) * cos(x2) * h2; } /** @@ -88,40 +121,6 @@ public static double cos(double a) { return indexCos + delta * (-indexSin + delta * (-indexCos * ONE_DIV_F2 + delta * (indexSin * ONE_DIV_F3 + delta * indexCos * ONE_DIV_F4))); } - /** - * Returns the trigonometric sine of an angle converted as a cos operation. - *

    - * Error is around 1E-15. - *

    - * Special cases: - *

      - *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. - *
    - * @param a an angle, in radians. - * @return the sine of the argument. - * @see Math#cos(double) - */ - public static double sin(double a) { - return cos(a - PIO2); - } - - /** - * Returns the trigonometric tangent of an angle converted in terms of a sin/cos operation. - *

    - * Error is around 1E-15. - *

    - * Special cases: - *

      - *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. - *
    - * @param a an angle, in radians. - * @return the tangent of the argument. - * @see Math#sin(double) aand Math#cos(double) - */ - public static double tan(double a) { - return sin(a) / cos(a); - } - /** * Returns the arc sine of a value. *

    @@ -173,22 +172,18 @@ public static double asin(double a) { } } - /** Return an approximate value of the diameter of the earth at the given latitude, in kilometers. */ - public static double earthDiameter(double latitude) { - final int index = (int)(Math.abs(latitude) * RADIUS_INDEXER + 0.5) % earthDiameterPerLatitude.length; - return earthDiameterPerLatitude[index]; - } - // haversin + // TODO: remove these for java 9, they fixed Math.toDegrees()/toRadians() to work just like this. public static final double TO_RADIANS = Math.PI / 180D; public static final double TO_DEGREES = 180D / Math.PI; + private static final double TO_METERS = 6_378_137D; // equatorial radius + private static final double TO_KILOMETERS = 6_378.137D; // equatorial radius // cos/asin private static final double ONE_DIV_F2 = 1/2.0; private static final double ONE_DIV_F3 = 1/6.0; private static final double ONE_DIV_F4 = 1/24.0; - public static final double PIO2 = Math.PI / 2D; private static final double PIO2_HI = Double.longBitsToDouble(0x3FF921FB54400000L); // 1.57079632673412561417e+00 first 33 bits of pi/2 private static final double PIO2_LO = Double.longBitsToDouble(0x3DD0B4611A626331L); // 6.07710050650619224932e-11 pi/2 - PIO2_HI private static final double TWOPI_HI = 4*PIO2_HI; @@ -232,11 +227,6 @@ public static double earthDiameter(double latitude) { private static final double ASIN_QS3 = Double.longBitsToDouble(0xbfe6066c1b8d0159L); // -6.88283971605453293030e-01 private static final double ASIN_QS4 = Double.longBitsToDouble(0x3fb3b8c5b12e9282L); // 7.70381505559019352791e-02 - private static final int RADIUS_TABS_SIZE = (1<<10) + 1; - private static final double RADIUS_DELTA = (StrictMath.PI/2d) / (RADIUS_TABS_SIZE - 1); - private static final double RADIUS_INDEXER = 1d/RADIUS_DELTA; - private static final double[] earthDiameterPerLatitude = new double[RADIUS_TABS_SIZE]; - /** Initializes look-up tables. */ static { // sin and cos @@ -279,27 +269,5 @@ public static double earthDiameter(double latitude) { asinDer3DivF3Tab[i] = ((1+2*x*x)*oneMinusXSqInv2_5) * ONE_DIV_F3; asinDer4DivF4Tab[i] = ((5+2*x*(2+x*(5-2*x)))*oneMinusXSqInv3_5) * ONE_DIV_F4; } - - - // WGS84 earth-ellipsoid major (a) and minor (b) radius - final double a = 6_378_137; // [m] - final double b = 6_356_752.31420; // [m] - - final double a2 = a*a; - final double b2 = b*b; - - earthDiameterPerLatitude[0] = 2 * a / 1000d; - earthDiameterPerLatitude[RADIUS_TABS_SIZE-1] = 2 * b / 1000d; - // earth radius - for (int i=1;i 0) { return cmp; @@ -182,32 +181,17 @@ public int compareTop(int doc) throws IOException { long encoded = currentDocs.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - minValue = Math.min(minValue, haversin1(latitude, longitude, docLatitude, docLongitude)); + minValue = Math.min(minValue, SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude)); } return minValue; } - // sort by first part of the haversin computation. note that this value is meaningless to the user. - // invoke haversin2() to "complete" the calculation and get a distance in meters. - static double haversin1(double lat1, double lon1, double lat2, double lon2) { - double dLat = SloppyMath.TO_RADIANS * (lat2 - lat1); - double dLon = SloppyMath.TO_RADIANS * (lon2 - lon1); - lat1 = SloppyMath.TO_RADIANS * (lat1); - lat2 = SloppyMath.TO_RADIANS * (lat2); - - final double sinDLatO2 = SloppyMath.sin(dLat / 2); - final double sinDLonO2 = SloppyMath.sin(dLon / 2); - - return sinDLatO2*sinDLatO2 + sinDLonO2 * sinDLonO2 * SloppyMath.cos(lat1) * SloppyMath.cos(lat2); - } - // second half of the haversin calculation, used to convert results from haversin1 (used internally // for sorting) for display purposes. static double haversin2(double partial) { if (Double.isInfinite(partial)) { return partial; } - double c = 2 * SloppyMath.asin(Math.sqrt(partial)); - return (GeoProjectionUtils.SEMIMAJOR_AXIS * c); + return SloppyMath.haversinMeters(partial); } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index e8c7c085c1e7..8f4b3475d728 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -35,13 +35,13 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.SparseFixedBitSet; import org.apache.lucene.util.StringHelper; @@ -106,9 +106,8 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // to reject edge cases faster in matches() final double minPartialDistance; if (box.maxLon - longitude < 90 && longitude - box.minLon < 90) { - minPartialDistance = Math.max(LatLonPointDistanceComparator.haversin1(latitude, longitude, latitude, box.maxLon), - LatLonPointDistanceComparator.haversin1(latitude, longitude, box.maxLat, longitude)); - assert LatLonPointDistanceComparator.haversin2(minPartialDistance) >= radiusMeters; + minPartialDistance = Math.max(SloppyMath.haversinSortKey(latitude, longitude, latitude, box.maxLon), + SloppyMath.haversinSortKey(latitude, longitude, box.maxLat, longitude)); } else { minPartialDistance = Double.POSITIVE_INFINITY; } @@ -199,10 +198,10 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); if (lonMax - longitude < 90 && longitude - lonMin < 90 && - GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMin) <= radiusMeters && - GeoDistanceUtils.haversin(latitude, longitude, latMin, lonMax) <= radiusMeters && - GeoDistanceUtils.haversin(latitude, longitude, latMax, lonMin) <= radiusMeters && - GeoDistanceUtils.haversin(latitude, longitude, latMax, lonMax) <= radiusMeters) { + SloppyMath.haversinMeters(latitude, longitude, latMin, lonMin) <= radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMin, lonMax) <= radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMax, lonMin) <= radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMax, lonMax) <= radiusMeters) { // we are fully enclosed, collect everything within this subtree return Relation.CELL_INSIDE_QUERY; } else { @@ -235,13 +234,13 @@ public boolean matches() throws IOException { double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); // first check the partial distance, if its more than that, it can't be <= radiusMeters - double h1 = LatLonPointDistanceComparator.haversin1(latitude, longitude, docLatitude, docLongitude); + double h1 = SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude); if (h1 > minPartialDistance) { continue; } // fully confirm with part 2: - if (LatLonPointDistanceComparator.haversin2(h1) <= radiusMeters) { + if (SloppyMath.haversinMeters(h1) <= radiusMeters) { return true; } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java index c69791ca55d5..193c2a330276 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java @@ -34,9 +34,11 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.util.GeoDistanceUtils; +import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.bkd.BKDWriter; @@ -159,7 +161,7 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException for (int doc = 0; doc < reader.maxDoc(); doc++) { double docLatitude = reader.document(doc).getField("lat").numericValue().doubleValue(); double docLongitude = reader.document(doc).getField("lon").numericValue().doubleValue(); - double distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + double distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude); if (distance <= radius) { expected.set(doc); } @@ -177,7 +179,7 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException for (int doc = 0; doc < reader.maxDoc(); doc++) { double docLatitude = reader.document(doc).getField("lat").numericValue().doubleValue(); double docLongitude = reader.document(doc).getField("lon").numericValue().doubleValue(); - double distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + double distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude); System.out.println("" + doc + ": (" + docLatitude + "," + docLongitude + "), distance=" + distance); } throw e; @@ -187,4 +189,55 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException writer.close(); dir.close(); } + + public void testBoundingBoxOpto() { + for (int i = 0; i < 1000; i++) { + double lat = -90 + 180.0 * random().nextDouble(); + double lon = -180 + 360.0 * random().nextDouble(); + double radius = 50000000 * random().nextDouble(); + GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); + final GeoRect box1; + final GeoRect box2; + if (box.crossesDateline()) { + box1 = new GeoRect(-180, box.maxLon, box.minLat, box.maxLat); + box2 = new GeoRect(box.minLon, 180, box.minLat, box.maxLat); + } else { + box1 = box; + box2 = null; + } + + for (int j = 0; j < 10000; j++) { + double lat2 = -90 + 180.0 * random().nextDouble(); + double lon2 = -180 + 360.0 * random().nextDouble(); + // if the point is within radius, then it should be in our bounding box + if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { + assertTrue(lat >= box.minLat && lat <= box.maxLat); + assertTrue(lon >= box1.minLon && lon <= box1.maxLon || (box2 != null && lon >= box2.minLon && lon <= box2.maxLon)); + } + } + } + } + + public void testHaversinOpto() { + for (int i = 0; i < 1000; i++) { + double lat = -90 + 180.0 * random().nextDouble(); + double lon = -180 + 360.0 * random().nextDouble(); + double radius = 50000000 * random().nextDouble(); + GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); + + if (box.maxLon - lon < 90 && lon - box.minLon < 90) { + double minPartialDistance = Math.max(SloppyMath.haversinSortKey(lat, lon, lat, box.maxLon), + SloppyMath.haversinSortKey(lat, lon, box.maxLat, lon)); + + for (int j = 0; j < 10000; j++) { + double lat2 = -90 + 180.0 * random().nextDouble(); + double lon2 = -180 + 360.0 * random().nextDouble(); + // if the point is within radius, then it should be <= our sort key + if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { + assertTrue(SloppyMath.haversinSortKey(lat, lon, lat2, lon2) <= minPartialDistance); + } + } + } + } + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 437631310ad3..8e8b03de688c 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -27,11 +27,11 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; /** Simple tests for {@link LatLonPoint#newDistanceSort} */ @@ -63,13 +63,13 @@ public void testDistanceSort() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); + assertEquals(462.6174876948475D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; - assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); + assertEquals(1056.163041670945D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[2]; - assertEquals(5291.798081404466D, (Double)d.fields[0], 0.0D); + assertEquals(5291.798081190281D, (Double)d.fields[0], 0.0D); reader.close(); dir.close(); @@ -100,10 +100,10 @@ public void testMissingLast() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.61748421408186D, (Double)d.fields[0], 0.0D); + assertEquals(462.6174876948475D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; - assertEquals(1056.1630445911035D, (Double)d.fields[0], 0.0D); + assertEquals(1056.163041670945D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[2]; assertEquals(Double.POSITIVE_INFINITY, (Double)d.fields[0], 0.0D); @@ -214,7 +214,7 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { } else { double docLatitude = targetDoc.getField("lat").numericValue().doubleValue(); double docLongitude = targetDoc.getField("lon").numericValue().doubleValue(); - distance = GeoDistanceUtils.haversin(lat, lon, docLatitude, docLongitude); + distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude); } int id = targetDoc.getField("id").numericValue().intValue(); expected[doc] = new Result(id, distance); @@ -252,26 +252,6 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { writer.close(); dir.close(); } - - /** Test this method sorts the same way as real haversin */ - public void testPartialHaversin() { - for (int i = 0; i < 100000; i++) { - double centerLat = -90 + 180.0 * random().nextDouble(); - double centerLon = -180 + 360.0 * random().nextDouble(); - - double lat1 = -90 + 180.0 * random().nextDouble(); - double lon1 = -180 + 360.0 * random().nextDouble(); - - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); - - int expected = Integer.signum(Double.compare(GeoDistanceUtils.haversin(centerLat, centerLon, lat1, lon1), - GeoDistanceUtils.haversin(centerLat, centerLon, lat2, lon2))); - int actual = Integer.signum(Double.compare(LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat1, lon1), - LatLonPointDistanceComparator.haversin1(centerLat, centerLon, lat2, lon2))); - assertEquals(expected, actual); - } - } /** Test infinite radius covers whole earth */ public void testInfiniteRect() { diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 028d6b844767..90502f9fd2bc 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -19,9 +19,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; -import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.util.SloppyMath; public class TestLatLonPointQueries extends BaseGeoPointTestCase { // TODO: remove this! @@ -136,7 +136,7 @@ protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double po @Override protected Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { - double distanceMeters = GeoDistanceUtils.haversin(centerLat, centerLon, pointLat, pointLon); + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); boolean result = distanceMeters <= radiusMeters; //System.out.println(" shouldMatch? centerLon=" + centerLon + " centerLat=" + centerLat + " pointLon=" + pointLon + " pointLat=" + pointLat + " result=" + result + " distanceMeters=" + (distanceKM * 1000)); return result; @@ -144,7 +144,7 @@ protected Boolean circleContainsPoint(double centerLat, double centerLon, double @Override protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { - final double d = GeoDistanceUtils.haversin(centerLat, centerLon, pointLat, pointLon); + final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); return d >= minRadiusMeters && d <= radiusMeters; } @@ -199,7 +199,7 @@ public void testRandomCircleToBBox() throws Exception { } } - double distanceMeters = GeoDistanceUtils.haversin(centerLat, centerLon, lat, lon); + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, lat, lon); // Haversin says it's within the circle: boolean haversinSays = distanceMeters <= radiusMeters; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java index 396d88ede126..a513b3d27c02 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java @@ -31,7 +31,7 @@ * the morton codes of the min and max lat/lon pairs that intersect the boundary of the point-radius * circle. Terms * passing this initial filter are then passed to a secondary {@code postFilter} method that verifies whether the - * decoded lat/lon point fall within the specified query distance (see {@link org.apache.lucene.util.SloppyMath#haversin}. + * decoded lat/lon point fall within the specified query distance (see {@link org.apache.lucene.util.SloppyMath#haversinMeters(double, double, double, double)}. * All morton value comparisons are subject to the same precision tolerance defined in * {@value org.apache.lucene.spatial.util.GeoEncodingUtils#TOLERANCE} and distance comparisons are subject to the accuracy of the * haversine formula (from R.W. Sinnott, "Virtues of the Haversine", Sky and Telescope, vol. 68, no. 2, 1984, p. 159) diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index f091bcb85ec4..6a54e23688dc 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -73,11 +73,11 @@ protected boolean cellIntersectsShape(final double minLon, final double minLat, * The two-phase query approach. The parent {@link GeoPointTermsEnum} class matches * encoded terms that fall within the minimum bounding box of the point-radius circle. Those documents that pass * the initial bounding box filter are then post filter compared to the provided distance using the - * {@link org.apache.lucene.util.SloppyMath#haversin} method. + * {@link org.apache.lucene.util.SloppyMath#haversinMeters(double, double, double, double)} method. */ @Override protected boolean postFilter(final double lon, final double lat) { - return (SloppyMath.haversin(distanceQuery.centerLat, centerLon, lat, lon) * 1000.0 <= distanceQuery.radiusMeters); + return SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, lat, lon) <= distanceQuery.radiusMeters; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java index 675b2a164cd2..107b1364f39f 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java @@ -53,7 +53,7 @@ protected short computeMaxShift() { double midLon = (minLon + maxLon) * 0.5; double midLat = (minLat + maxLat) * 0.5; - if (SloppyMath.haversin(minLat, minLon, midLat, midLon)*1000 > 1000000) { + if (SloppyMath.haversinMeters(minLat, minLon, midLat, midLon) > 1000000) { shiftFactor = 5; } else { shiftFactor = 4; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java index b83d68059ca1..edbd53d28084 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java @@ -105,7 +105,7 @@ protected short computeMaxShift() { double midLon = (minLon + maxLon) * 0.5; double midLat = (minLat + maxLat) * 0.5; - if (SloppyMath.haversin(minLat, minLon, midLat, midLon)*1000 > 1000000) { + if (SloppyMath.haversinMeters(minLat, minLon, midLat, midLon) > 1000000) { shiftFactor = 5; } else { shiftFactor = 4; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java index e845c9e0c3ba..482ed0f8cd78 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java @@ -33,28 +33,6 @@ public class GeoDistanceUtils { private GeoDistanceUtils() { } - /** - * Compute the great-circle distance using original haversine implementation published by Sinnot in: - * R.W. Sinnott, "Virtues of the Haversine", Sky and Telescope, vol. 68, no. 2, 1984, p. 159 - * - * NOTE: this differs from {@link org.apache.lucene.util.SloppyMath#haversin} in that it uses the semi-major axis - * of the earth instead of an approximation based on the average latitude of the two points (which can introduce an - * additional error up to .337%, or ~67.6 km at the equator) - */ - public static double haversin(double lat1, double lon1, double lat2, double lon2) { - double dLat = TO_RADIANS * (lat2 - lat1); - double dLon = TO_RADIANS * (lon2 - lon1); - lat1 = TO_RADIANS * (lat1); - lat2 = TO_RADIANS * (lat2); - - final double sinDLatO2 = SloppyMath.sin(dLat / 2); - final double sinDLonO2 = SloppyMath.sin(dLon / 2); - - double a = sinDLatO2*sinDLatO2 + sinDLonO2 * sinDLonO2 * SloppyMath.cos(lat1) * SloppyMath.cos(lat2); - double c = 2 * SloppyMath.asin(Math.sqrt(a)); - return (GeoProjectionUtils.SEMIMAJOR_AXIS * c); - } - /** * Compute the distance between two geo-points using vincenty distance formula * Vincenty uses the oblate spheroid whereas haversine uses unit sphere, this will give roughly @@ -138,12 +116,11 @@ public static double linearDistance(double[] pt1, double[] pt2) { * @return Sloppy distance in degrees longitude for provided distance in meters */ public static double distanceToDegreesLon(double lat, double distance) { - distance /= 1000.0; // convert latitude to radians lat = StrictMath.toRadians(lat); // get the diameter at the latitude - final double diameter = SloppyMath.earthDiameter(StrictMath.toRadians(lat)); + final double diameter = 2 * GeoProjectionUtils.SEMIMAJOR_AXIS; // compute inverse haversine double a = StrictMath.sin(distance/diameter); @@ -197,9 +174,9 @@ public static void closestPointOnBBox(final double rMinX, final double rMinY, fi /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */ public static double maxRadialDistanceMeters(final double centerLon, final double centerLat) { if (Math.abs(centerLat) == GeoUtils.MAX_LAT_INCL) { - return GeoDistanceUtils.haversin(centerLat, centerLon, 0, centerLon); + return SloppyMath.haversinMeters(centerLat, centerLon, 0, centerLon); } - return GeoDistanceUtils.haversin(centerLat, centerLon, centerLat, (GeoUtils.MAX_LON_INCL + centerLon) % 360); + return SloppyMath.haversinMeters(centerLat, centerLon, centerLat, (GeoUtils.MAX_LON_INCL + centerLon) % 360); } /** @@ -210,8 +187,7 @@ public static double maxRadialDistanceMeters(final double centerLon, final doubl */ public static double distanceToDegreesLat(double lat, double distance) { // get the diameter at the latitude - final double diameter = SloppyMath.earthDiameter(StrictMath.toRadians(lat)); - distance /= 1000.0; + final double diameter = 2 * GeoProjectionUtils.SEMIMAJOR_AXIS; // compute inverse haversine double a = StrictMath.sin(distance/diameter); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java index 5a81adc1b615..12d135045ac6 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java @@ -20,9 +20,6 @@ import static org.apache.lucene.util.SloppyMath.asin; import static org.apache.lucene.util.SloppyMath.cos; -import static org.apache.lucene.util.SloppyMath.sin; -import static org.apache.lucene.util.SloppyMath.tan; -import static org.apache.lucene.util.SloppyMath.PIO2; import static org.apache.lucene.util.SloppyMath.TO_DEGREES; import static org.apache.lucene.util.SloppyMath.TO_RADIANS; @@ -30,8 +27,11 @@ import static org.apache.lucene.spatial.util.GeoUtils.MAX_LON_INCL; import static org.apache.lucene.spatial.util.GeoUtils.MIN_LAT_INCL; import static org.apache.lucene.spatial.util.GeoUtils.MIN_LON_INCL; +import static org.apache.lucene.spatial.util.GeoUtils.PIO2; import static org.apache.lucene.spatial.util.GeoUtils.normalizeLat; import static org.apache.lucene.spatial.util.GeoUtils.normalizeLon; +import static org.apache.lucene.spatial.util.GeoUtils.sloppySin; +import static org.apache.lucene.spatial.util.GeoUtils.sloppyTan; /** * Reusable geo-spatial projection utility methods. @@ -149,7 +149,7 @@ public static final double[] llaToECF(double lon, double lat, double alt, double lon = TO_RADIANS * lon; lat = TO_RADIANS * lat; - final double sl = sin(lat); + final double sl = sloppySin(lat); final double s2 = sl*sl; final double cl = cos(lat); @@ -170,7 +170,7 @@ public static final double[] llaToECF(double lon, double lat, double alt, double final double rn = SEMIMAJOR_AXIS / StrictMath.sqrt(1.0D - E2 * s2); ecf[0] = (rn+alt) * cl * cos(lon); - ecf[1] = (rn+alt) * cl * sin(lon); + ecf[1] = (rn+alt) * cl * sloppySin(lon); ecf[2] = ((rn*(1.0-E2))+alt)*sl; return ecf; @@ -304,9 +304,9 @@ private static double[][] createPhiTransform(double originLon, double originLat, originLon = TO_RADIANS * originLon; originLat = TO_RADIANS * originLat; - final double sLon = sin(originLon); + final double sLon = sloppySin(originLon); final double cLon = cos(originLon); - final double sLat = sin(originLat); + final double sLat = sloppySin(originLat); final double cLat = cos(originLat); phiMatrix[0][0] = -sLon; @@ -338,9 +338,9 @@ private static double[][] createTransposedPhiTransform(double originLon, double originLon = TO_RADIANS * originLon; originLat = TO_RADIANS * originLat; - final double sLat = sin(originLat); + final double sLat = sloppySin(originLat); final double cLat = cos(originLat); - final double sLon = sin(originLon); + final double sLon = sloppySin(originLon); final double cLon = cos(originLon); phiMatrix[0][0] = -sLon; @@ -374,8 +374,8 @@ public static final double[] pointFromLonLatBearingVincenty(double lon, double l final double alpha1 = TO_RADIANS * bearing; final double cosA1 = cos(alpha1); - final double sinA1 = sin(alpha1); - final double tanU1 = (1-FLATTENING) * tan(TO_RADIANS * lat); + final double sinA1 = sloppySin(alpha1); + final double tanU1 = (1-FLATTENING) * sloppyTan(TO_RADIANS * lat); final double cosU1 = 1 / StrictMath.sqrt((1+tanU1*tanU1)); final double sinU1 = tanU1*cosU1; final double sig1 = StrictMath.atan2(tanU1, cosA1); @@ -391,7 +391,7 @@ public static final double[] pointFromLonLatBearingVincenty(double lon, double l do { cos2SigmaM = cos(2*sig1 + sigma); - sinSigma = sin(sigma); + sinSigma = sloppySin(sigma); cosSigma = cos(sigma); deltaSigma = B * sinSigma * (cos2SigmaM + (B/4D) * (cosSigma*(-1+2*cos2SigmaM*cos2SigmaM)- @@ -435,12 +435,12 @@ public static final double[] pointFromLonLatBearingGreatCircle(double lon, doubl bearing *= TO_RADIANS; final double cLat = cos(lat); - final double sLat = sin(lat); - final double sinDoR = sin(dist / GeoProjectionUtils.SEMIMAJOR_AXIS); + final double sLat = sloppySin(lat); + final double sinDoR = sloppySin(dist / GeoProjectionUtils.SEMIMAJOR_AXIS); final double cosDoR = cos(dist / GeoProjectionUtils.SEMIMAJOR_AXIS); pt[1] = asin(sLat*cosDoR + cLat * sinDoR * cos(bearing)); - pt[0] = TO_DEGREES * (lon + Math.atan2(sin(bearing) * sinDoR * cLat, cosDoR - sLat * sin(pt[1]))); + pt[0] = TO_DEGREES * (lon + Math.atan2(sloppySin(bearing) * sinDoR * cLat, cosDoR - sLat * sloppySin(pt[1]))); pt[1] *= TO_DEGREES; return pt; @@ -458,8 +458,8 @@ public static double bearingGreatCircle(double lon1, double lat1, double lon2, d double dLon = (lon2 - lon1) * TO_RADIANS; lat2 *= TO_RADIANS; lat1 *= TO_RADIANS; - double y = sin(dLon) * cos(lat2); - double x = cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(dLon); + double y = sloppySin(dLon) * cos(lat2); + double x = cos(lat1) * sloppySin(lat2) - sloppySin(lat1) * cos(lat2) * cos(dLon); return Math.atan2(y, x) * TO_DEGREES; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java index d9d6991b8c0a..e40b91a4b535 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java @@ -289,10 +289,10 @@ private static boolean rectAnyCornersInCircle(final double rMinX, final double r } double w = Math.abs(rMaxX - rMinX); if (w <= 90.0) { - return GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMinX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; } // partition w /= 4; @@ -300,24 +300,24 @@ private static boolean rectAnyCornersInCircle(final double rMinX, final double r final double p2 = p1 + w; final double p3 = p2 + w; - return GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMinX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, p1) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, p1) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, p2) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, p2) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, p3) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, p3) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p1) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p1) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p2) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p2) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p3) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p3) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; } private static boolean rectAnyCornersInCircleSloppy(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, final double centerLon, final double centerLat, final double radiusMeters) { - return SloppyMath.haversin(centerLat, centerLon, rMinY, rMinX)*1000.0 <= radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMinX)*1000.0 <= radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMaxX)*1000.0 <= radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMinY, rMaxX)*1000.0 <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; } /** @@ -336,10 +336,10 @@ private static boolean rectAnyCornersOutsideCircle(final double rMinX, final dou } // if span is less than 70 degrees we can approximate using distance alone if (Math.abs(rMaxX - rMinX) <= 70.0) { - return GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMinX) > radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMinX) > radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxY, rMaxX) > radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMinY, rMaxX) > radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) > radiusMeters; } return rectCrossesOblateCircle(centerLon, centerLat, radiusMeters, rMinX, rMinY, rMaxX, rMaxY); } @@ -363,10 +363,10 @@ public static boolean rectCrossesOblateCircle(double centerLon, double centerLat final double d1, d2; // short-circuit if we find a corner outside the circle - if ( (d1 = GeoDistanceUtils.haversin(centerLat, centerLon, rMinLat, p1)) > radiusMeters - || (d2 = GeoDistanceUtils.haversin(centerLat, centerLon, rMinLat, maxLon)) > radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxLat, p1) > radiusMeters - || GeoDistanceUtils.haversin(centerLat, centerLon, rMaxLat, maxLon) > radiusMeters) { + if ( (d1 = SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p1)) > radiusMeters + || (d2 = SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, maxLon)) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p1) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, maxLon) > radiusMeters) { return true; } @@ -379,7 +379,7 @@ public static boolean rectCrossesOblateCircle(double centerLon, double centerLat GeoProjectionUtils.bearingGreatCircle(maxLon, rMinLat, maxLon, rMaxLat), radiusMeters - d2, pt))[1] < rMinLat || pt[1] < rMaxLat || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(maxLon, rMinLat, GeoProjectionUtils.bearingGreatCircle(maxLon, rMinLat, (midLon = p1 + 0.5*(maxLon - p1)), rMaxLat), - radiusMeters - GeoDistanceUtils.haversin(centerLat, centerLon, rMinLat, midLon), pt))[1] < rMinLat + radiusMeters - SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, midLon), pt))[1] < rMinLat || pt[1] < rMaxLat == false ) { return true; } @@ -390,10 +390,10 @@ public static boolean rectCrossesOblateCircle(double centerLon, double centerLat private static boolean rectAnyCornersOutsideCircleSloppy(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, final double centerLon, final double centerLat, final double radiusMeters) { - return SloppyMath.haversin(centerLat, centerLon, rMinY, rMinX)*1000.0 > radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMinX)*1000.0 > radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMaxY, rMaxX)*1000.0 > radiusMeters - || SloppyMath.haversin(centerLat, centerLon, rMinY, rMaxX)*1000.0 > radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) > radiusMeters; } /** @@ -446,7 +446,7 @@ private static boolean isClosestPointOnRectWithinRange(final double rMinX, final final boolean approx) { double[] closestPt = {0, 0}; GeoDistanceUtils.closestPointOnBBox(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, closestPt); - boolean haverShortCut = GeoDistanceUtils.haversin(centerLat, centerLon, closestPt[1], closestPt[0]) <= radiusMeters; + boolean haverShortCut = SloppyMath.haversinMeters(centerLat, centerLon, closestPt[1], closestPt[0]) <= radiusMeters; if (approx == true || haverShortCut == true) { return haverShortCut; } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index 5becef70368f..6d755ce52612 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -18,6 +18,8 @@ import java.util.ArrayList; +import org.apache.lucene.util.SloppyMath; + import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.PI; @@ -25,7 +27,6 @@ import static org.apache.lucene.util.SloppyMath.asin; import static org.apache.lucene.util.SloppyMath.cos; -import static org.apache.lucene.util.SloppyMath.sin; import static org.apache.lucene.util.SloppyMath.TO_DEGREES; import static org.apache.lucene.util.SloppyMath.TO_RADIANS; import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE; @@ -137,7 +138,7 @@ public static GeoRect circleToBBox(final double centerLon, final double centerLa double maxLon; if (minLat > MIN_LAT_RADIANS && maxLat < MAX_LAT_RADIANS) { - double deltaLon = asin(sin(radDistance) / cos(radLat)); + double deltaLon = asin(sloppySin(radDistance) / cos(radLat)); minLon = radLon - deltaLon; if (minLon < MIN_LON_RADIANS) { minLon += 2d * PI; @@ -184,4 +185,45 @@ public static GeoRect polyToBBox(double[] polyLons, double[] polyLats) { return new GeoRect(max(minLon - TOLERANCE, MIN_LON_INCL), min(maxLon + TOLERANCE, MAX_LON_INCL), max(minLat - TOLERANCE, MIN_LAT_INCL), min(maxLat + TOLERANCE, MAX_LAT_INCL)); } + + + // some sloppyish stuff, do we really need this to be done in a sloppy way? + // unless it is performance sensitive, we should try to remove. + static final double PIO2 = Math.PI / 2D; + /** + * Returns the trigonometric sine of an angle converted as a cos operation. + *

    + * Note that this is not quite right... e.g. sin(0) != 0 + *

    + * Special cases: + *

      + *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. + *
    + * @param a an angle, in radians. + * @return the sine of the argument. + * @see Math#sin(double) + */ + // TODO: deprecate/remove this? at least its no longer public. + static double sloppySin(double a) { + return cos(a - PIO2); + } + + + /** + * Returns the trigonometric tangent of an angle converted in terms of a sin/cos operation. + *

    + * Note that this is probably not quite right (untested) + *

    + * Special cases: + *

      + *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. + *
    + * @param a an angle, in radians. + * @return the tangent of the argument. + * @see Math#sin(double) aand Math#cos(double) + */ + // TODO: deprecate/remove this? at least its no longer public. + static double sloppyTan(double a) { + return sloppySin(a) / cos(a); + } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 13ce23520cda..2fd64bd3d839 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -205,7 +205,7 @@ protected Boolean circleContainsPoint(double centerLat, double centerLon, double if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { return null; } else { - return SloppyMath.haversin(centerLat, centerLon, pointLat, pointLon)*1000.0 <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon) <= radiusMeters; } } @@ -215,7 +215,7 @@ protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, || radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { return null; } else { - final double d = SloppyMath.haversin(centerLat, centerLon, pointLat, pointLon)*1000.0; + final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); return d >= minRadiusMeters && d <= radiusMeters; } } @@ -229,7 +229,7 @@ private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); - double ptDistance = SloppyMath.haversin(centerLat, centerLon, ptLat, ptLon)*1000.0; + double ptDistance = SloppyMath.haversinMeters(centerLat, centerLon, ptLat, ptLon); double delta = StrictMath.abs(ptDistance - radius); // if its within the distance error then it can be wrong diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index e7f9913f6733..e14ffbde2e08 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; import org.junit.BeforeClass; @@ -724,7 +725,7 @@ protected Boolean shouldMatch(double pointLat, double pointLon) { @Override protected void describe(int docID, double pointLat, double pointLon) { - double distanceMeters = GeoDistanceUtils.haversin(centerLat, centerLon, pointLat, pointLon); + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); System.out.println(" docID=" + docID + " centerLon=" + centerLon + " centerLat=" + centerLat + " pointLon=" + pointLon + " pointLat=" + pointLat + " distanceMeters=" + distanceMeters + " vs" + ((rangeQuery == true) ? " minRadiusMeters=" + minRadiusMeters : "") + " radiusMeters=" + radiusMeters); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index d58e4802ebab..bc76852c1e15 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SloppyMath; import org.junit.BeforeClass; import com.carrotsearch.randomizedtesting.generators.RandomInts; @@ -315,7 +316,7 @@ private void findMatches(Set hits, PrintWriter log, Cell root, // Leaf cell: brute force check all docs that fall within this cell: for(int docID=0;docID Date: Sun, 20 Mar 2016 22:04:14 -0400 Subject: [PATCH 0179/1113] SOLR-8878: Allow the DaemonStream run rate be controlled by the internal stream --- .../client/solrj/io/stream/DaemonStream.java | 35 +++++++----- .../client/solrj/io/stream/TopicStream.java | 16 +++++- .../client/solrj/io/stream/StreamingTest.java | 56 ++++++++++++++++++- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java index edafd7e92a7d..a0aabe3b61bf 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java @@ -73,7 +73,7 @@ public DaemonStream(StreamExpression expression, StreamFactory factory) throws I } if(runExpression == null) { - throw new IOException("Invalid expression runInterval parameter expected"); + runInterval = 2000; } else { runInterval = Long.parseLong(((StreamExpressionValue) runExpression.getParameter()).getValue()); } @@ -243,7 +243,7 @@ public void run() { OUTER: while (!getShutdown()) { long now = new Date().getTime(); - if((now-lastRun) > this.runInterval) { + if ((now - lastRun) > this.runInterval) { lastRun = now; try { tupleStream.open(); @@ -252,25 +252,31 @@ public void run() { Tuple tuple = tupleStream.read(); if (tuple.EOF) { errors = 0; // Reset errors on successful run. + if (tuple.fields.containsKey("sleepMillis")) { + this.sleepMillis = tuple.getLong("sleepMillis"); + this.runInterval = -1; + } break INNER; } else if (!eatTuples) { try { queue.put(tuple); - } catch(InterruptedException e) { + } catch (InterruptedException e) { break OUTER; } } } } catch (IOException e) { + e.printStackTrace(); exception = e; - logger.error("Error in DaemonStream:"+id, e); + logger.error("Error in DaemonStream:" + id, e); ++errors; - if(errors > 100) { - logger.error("Too many consectutive errors. Stopping DaemonStream:"+id); + if (errors > 100) { + logger.error("Too many consectutive errors. Stopping DaemonStream:" + id); break OUTER; } } catch (Throwable t) { - logger.error("Fatal Error in DaemonStream:"+id, t); + t.printStackTrace(); + logger.error("Fatal Error in DaemonStream:" + id, t); //For anything other then IOException break out of the loop and shutdown the thread. break OUTER; } finally { @@ -279,18 +285,21 @@ public void run() { } catch (IOException e1) { if (exception == null) { exception = e1; - logger.error("Error in DaemonStream:"+id, e1); + logger.error("Error in DaemonStream:" + id, e1); break OUTER; } } } } incrementIterations(); - try { - Thread.sleep(sleepMillis); - } catch (InterruptedException e) { - logger.error("Error in DaemonStream:"+id, e); - break OUTER; + + if (sleepMillis > 0) { + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException e) { + logger.error("Error in DaemonStream:" + id, e); + break OUTER; + } } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java index 3b7aa90e16e3..a52e8f9550bc 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java @@ -65,6 +65,7 @@ public class TopicStream extends CloudSolrStream implements Expressible { private static final long serialVersionUID = 1; private long count; + private int runCount; private String id; protected long checkpointEvery; @@ -98,6 +99,9 @@ private void init(String zkHost, this.checkpointEvery = checkpointEvery; this.id = id; this.comp = new FieldComparator("_version_", ComparatorOrder.ASCENDING); + if(!params.containsKey("rows")) { + params.put("rows", "500"); + } } public TopicStream(StreamExpression expression, StreamFactory factory) throws IOException{ @@ -257,6 +261,7 @@ private void openStreams() throws IOException { } public void close() throws IOException { + runCount = 0; try { persistCheckpoints(); } finally { @@ -277,10 +282,17 @@ public Tuple read() throws IOException { Tuple tuple = _read(); if(tuple.EOF) { + if(runCount > 0) { + tuple.put("sleepMillis", 0); + } else { + tuple.put("sleepMillis", 1000); + } + return tuple; } ++count; + ++runCount; if(checkpointEvery > -1 && (count % checkpointEvery) == 0) { persistCheckpoints(); } @@ -427,7 +439,9 @@ protected void constructStreams() throws IOException { params.put("distrib", "false"); // We are the aggregator. String fl = params.get("fl"); params.put("sort", "_version_ asc"); - fl += ",_version_"; + if(!fl.contains("_version_")) { + fl += ",_version_"; + } params.put("fl", fl); Random random = new Random(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java index cec2c525f61e..165029ffe42f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java @@ -27,6 +27,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; import org.apache.solr.client.solrj.io.comp.MultipleFieldComparator; import org.apache.solr.client.solrj.io.comp.FieldComparator; @@ -226,7 +227,7 @@ private void testParallelUniqueStream() throws Exception { attachStreamFactory(pstream); List tuples = getTuples(pstream); assert(tuples.size() == 5); - assertOrder(tuples, 0,1,3,4,6); + assertOrder(tuples, 0, 1, 3, 4, 6); //Test the eofTuples @@ -1369,7 +1370,59 @@ private void testRollupStream() throws Exception { } + private void testDaemonTopicStream() throws Exception { + String zkHost = zkServer.getZkAddress(); + + StreamContext context = new StreamContext(); + SolrClientCache cache = new SolrClientCache(); + context.setSolrClientCache(cache); + + Map params = new HashMap(); + params.put("q","a_s:hello0"); + params.put("rows", "500"); + params.put("fl", "id"); + + TopicStream topicStream = new TopicStream(zkHost, "collection1", "collection1", "50000000", 1000000, params); + + DaemonStream daemonStream = new DaemonStream(topicStream, "daemon1", 1000, 500); + daemonStream.setStreamContext(context); + + daemonStream.open(); + + indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1"); + indexr(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2"); + indexr(id, "3", "a_s", "hello0", "a_i", "3", "a_f", "3"); + indexr(id, "4", "a_s", "hello0", "a_i", "4", "a_f", "4"); + indexr(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5"); + + commit(); + + + for(int i=0; i<5; i++) { + daemonStream.read(); + } + + + indexr(id, "5", "a_s", "hello0", "a_i", "4", "a_f", "4"); + indexr(id, "6", "a_s", "hello0", "a_i", "4", "a_f", "4"); + + commit(); + + for(int i=0; i<2; i++) { + daemonStream.read(); + } + + daemonStream.shutdown(); + + Tuple tuple = daemonStream.read(); + + assertTrue(tuple.EOF); + daemonStream.close(); + cache.close(); + del("*:*"); + commit(); + } private void testParallelRollupStream() throws Exception { @@ -1799,6 +1852,7 @@ public void streamTests() throws Exception { testSubFacetStream(); testStatsStream(); //testExceptionStream(); + testDaemonTopicStream(); testParallelEOF(); testParallelUniqueStream(); testParallelRankStream(); From faa077224cad8d0ae9f06748cfe72fdc7f208399 Mon Sep 17 00:00:00 2001 From: jbernste Date: Sun, 20 Mar 2016 22:19:38 -0400 Subject: [PATCH 0180/1113] SOLR-8878: Remove debugging --- .../org/apache/solr/client/solrj/io/stream/DaemonStream.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java index a0aabe3b61bf..752ea7ce1d1a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DaemonStream.java @@ -266,7 +266,6 @@ public void run() { } } } catch (IOException e) { - e.printStackTrace(); exception = e; logger.error("Error in DaemonStream:" + id, e); ++errors; @@ -275,7 +274,6 @@ public void run() { break OUTER; } } catch (Throwable t) { - t.printStackTrace(); logger.error("Fatal Error in DaemonStream:" + id, t); //For anything other then IOException break out of the loop and shutdown the thread. break OUTER; From 451ff488afc4707cb8555c5629bcf0c9ebef210f Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 21 Mar 2016 17:39:37 +0000 Subject: [PATCH 0181/1113] SOLR-8837: Fix duplicate election node detection --- solr/CHANGES.txt | 12 +++++++----- .../java/org/apache/solr/cloud/LeaderElector.java | 5 +---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b386f6a8e391..23e8e89ba66e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -17,6 +17,7 @@ servlet container in the directory named "example". See the Quick Start guide at http://lucene.apache.org/solr/quickstart.html ================== 6.1.0 ================== + Detailed Change List ---------------------- * SOLR-8782: Add asynchronous sugar methods to the SolrJ Collections API. You @@ -27,13 +28,13 @@ Detailed Change List * SOLR-8765: Enforce required parameters at query construction time in the SolrJ Collections API, add static factory methods, and deprecate old setter methods. (Alan Woodward, Jason Gerlowski) -* SOLR-8842: authorization APIs do not use name as an identifier for a permission - for update, delete
 commands and 'before' attribute (noble) + +* SOLR-8842: authorization APIs do not use name as an identifier for a permission + for update, delete commands and 'before' attribute (noble) New Features ---------------------- - * SOLR-8814: Support GeoJSON response writer and general spatial formatting. Adding &wt=geojson&geojson.field= Will return a FeatureCollection for each SolrDocumentList and a Feature with the @@ -276,8 +277,9 @@ Bug Fixes * SOLR-8656: PeerSync should use same nUpdates everywhere. (Ramsey Haddad via Mark Miller) -* SOLR-8697: Scope ZK election nodes by session to prevent elections from interfering with each other - and other small LeaderElector improvements. (Scott Blum via Mark Miller) +* SOLR-8697, SOLR-8837: Scope ZK election nodes by session to prevent elections from interfering with each other + and other small LeaderElector improvements. (Scott Blum via Mark Miller, Alan + Woodward) * SOLR-8599: After a failed connection during construction of SolrZkClient attempt to retry until a connection can be made. (Keith Laban, Dennis Gove) diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java index c0eb39583336..e71ddf09b9b6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java +++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java @@ -18,16 +18,13 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; import org.apache.solr.cloud.ZkController.ContextKey; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; @@ -116,7 +113,7 @@ private void checkIfIamLeader(final ElectionContext context, boolean replacement // If any double-registrations exist for me, remove all but this latest one! // TODO: can we even get into this state? - String prefix = zkClient.getSolrZooKeeper().getSessionId() + "-" + context.id; + String prefix = zkClient.getSolrZooKeeper().getSessionId() + "-" + context.id + "-"; Iterator it = seqs.iterator(); while (it.hasNext()) { String elec = it.next(); From b3136e5459cbedc2ec50075456a63a1dec6cb813 Mon Sep 17 00:00:00 2001 From: markrmiller Date: Mon, 21 Mar 2016 14:10:03 -0400 Subject: [PATCH 0182/1113] SOLR-8857: HdfsUpdateLog does not use configured or new default number of version buckets and is hard coded to 256. --- solr/CHANGES.txt | 3 +++ .../java/org/apache/solr/update/HdfsUpdateLog.java | 13 +++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 23e8e89ba66e..8867257714e0 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -52,6 +52,9 @@ New Features Bug Fixes ---------------------- +* SOLR-8857: HdfsUpdateLog does not use configured or new default number of version buckets and is + hard coded to 256. (Mark Miller, yonik, Gregory Chanan) + Optimizations ---------------------- * SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation. diff --git a/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java b/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java index aa66391d8c41..bdb1d82a7c1f 100644 --- a/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java @@ -97,19 +97,12 @@ public boolean dropBufferedUpdates() { @Override public void init(PluginInfo info) { - dataDir = (String) info.initArgs.get("dir"); - - defaultSyncLevel = SyncLevel.getSyncLevel((String) info.initArgs - .get("syncLevel")); - - numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100); - maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10); + super.init(info); tlogDfsReplication = (Integer) info.initArgs.get( "tlogDfsReplication"); if (tlogDfsReplication == null) tlogDfsReplication = 3; - log.info("Initializing HdfsUpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={} tlogDfsReplication={}", - dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep, tlogDfsReplication); + log.info("Initializing HdfsUpdateLog: tlogDfsReplication={}", tlogDfsReplication); } private Configuration getConf() { @@ -231,7 +224,7 @@ public void init(UpdateHandler uhandler, SolrCore core) { } try { - versionInfo = new VersionInfo(this, 256); + versionInfo = new VersionInfo(this, numVersionBuckets); } catch (SolrException e) { log.error("Unable to use updateLog: " + e.getMessage(), e); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, From dfab0a60f8c91fc8a3eb6b699ebf9530e2a8d188 Mon Sep 17 00:00:00 2001 From: Gregory Chanan Date: Fri, 18 Mar 2016 14:28:44 -0700 Subject: [PATCH 0183/1113] SOLR-8869: Optionally disable printing field cache entries in SolrFieldCacheMBean --- solr/CHANGES.txt | 2 + .../solr/search/SolrFieldCacheMBean.java | 8 ++- .../solr/search/TestSolrFieldCacheMBean.java | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8867257714e0..634f218dbb79 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -77,6 +77,8 @@ Other Changes permission name.
 The APIs are also modified to use 'index' as the unique identifier instead of name. Name is an optional attribute
 now and only to be used when specifying well-known permissions (noble) +* SOLR-8869: Optionally disable printing field cache entries in SolrFieldCacheMBean (Gregory Chanan) + ================== 6.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java index 46e35b532013..cfc68c4d6879 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java +++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java @@ -31,6 +31,8 @@ */ public class SolrFieldCacheMBean implements SolrInfoMBean { + private boolean disableEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryList"); + @Override public String getName() { return this.getClass().getName(); } @Override @@ -52,8 +54,10 @@ public NamedList getStatistics() { NamedList stats = new SimpleOrderedMap(); String[] entries = UninvertingReader.getUninvertedStats(); stats.add("entries_count", entries.length); - for (int i = 0; i < entries.length; i++) { - stats.add("entry#" + i, entries[i]); + if (!disableEntryList) { + for (int i = 0; i < entries.length; i++) { + stats.add("entry#" + i, entries[i]); + } } return stats; } diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java new file mode 100644 index 000000000000..b90430e9ef6c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; + +public class TestSolrFieldCacheMBean extends SolrTestCaseJ4 { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml","schema-minimal.xml"); + } + + @Test + public void testEntryList() throws Exception { + // ensure entries to FieldCache + assertU(adoc("id", "id0")); + assertU(commit()); + assertQ(req("q", "*:*", "sort", "id asc"), "//*[@numFound='1']"); + + SolrFieldCacheMBean mbean = new SolrFieldCacheMBean(); + NamedList stats = mbean.getStatistics(); + assert(new Integer(stats.get("entries_count").toString()) > 0); + assertNotNull(stats.get("entry#0")); + + // Test again with entry list disabled + System.setProperty("disableSolrFieldCacheMBeanEntryList", "true"); + try { + mbean = new SolrFieldCacheMBean(); + stats = mbean.getStatistics(); + assert(new Integer(stats.get("entries_count").toString()) > 0); + assertNull(stats.get("entry#0")); + } finally { + System.clearProperty("disableSolrFieldCacheMBeanEntryList"); + } + } +} From 0926cecae647862760068d5eb0c8e8bc5afc6a79 Mon Sep 17 00:00:00 2001 From: Gregory Chanan Date: Fri, 18 Mar 2016 14:36:15 -0700 Subject: [PATCH 0184/1113] LUCENE-7115: Speed up FieldCache.CacheEntry toString by setting initial StringBuilder capacity --- lucene/CHANGES.txt | 3 +++ .../src/java/org/apache/lucene/uninverting/FieldCache.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 78d7aa3c0e58..138f6bd069fd 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -26,6 +26,9 @@ Optimizations * LUCENE-7097: IntroSorter now recurses to 2 * log_2(count) quicksort stack depth before switching to heapsort (Adrien Grand, Mike McCandless) +* LUCENE-7115: Speed up FieldCache.CacheEntry toString by setting initial + StringBuilder capacity (Gregory Chanan) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java index 27d68e04abb0..11185a903e91 100644 --- a/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java +++ b/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java @@ -404,7 +404,7 @@ public String getEstimatedSize() { @Override public String toString() { - StringBuilder b = new StringBuilder(); + StringBuilder b = new StringBuilder(250); b.append("'").append(getReaderKey()).append("'=>"); b.append("'").append(getFieldName()).append("',"); b.append(getCacheType()).append(",").append(getCustom()); From e4359aadf278ce88b12d3d0bb522c26fcef1c597 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Mar 2016 22:01:52 -0400 Subject: [PATCH 0185/1113] LUCENE-7125: remove BKD_TOLERANCE from LatLonPoint polygon tests --- .../lucene/search/TestLatLonPointQueries.java | 50 ++++--------------- 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 90502f9fd2bc..3b48eee9234a 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -20,12 +20,11 @@ import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoRelationUtils; import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.SloppyMath; public class TestLatLonPointQueries extends BaseGeoPointTestCase { - // TODO: remove this! - public static final double BKD_TOLERANCE = 1e-7; @Override protected void addPointToDoc(String field, Document doc, double lat, double lon) { @@ -89,49 +88,20 @@ protected double quantizeLon(double lonRaw) { return LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); } - // todo reconcile with GeoUtils (see LUCENE-6996) - public static double compare(final double v1, final double v2) { - final double delta = v1-v2; - return Math.abs(delta) <= BKD_TOLERANCE ? 0 : delta; - } - @Override protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { // TODO write better random polygon tests - - assert Double.isNaN(pointLat) == false; - - // TODO: this comment is wrong! we have fixed the quantization error (we now pre-quantize all randomly generated test points) yet the test - // still fails if we remove this evil "return null": - // false positive/negatives due to quantization error exist for both rectangles and polygons - if (compare(pointLat, rect.minLat) == 0 - || compare(pointLat, rect.maxLat) == 0 - || compare(pointLon, rect.minLon) == 0 - || compare(pointLon, rect.maxLon) == 0) { - return null; - } - - int rectLatMinEnc = LatLonPoint.encodeLatitude(rect.minLat); - int rectLatMaxEnc = LatLonPoint.encodeLatitude(rect.maxLat); - int rectLonMinEnc = LatLonPoint.encodeLongitude(rect.minLon); - int rectLonMaxEnc = LatLonPoint.encodeLongitude(rect.maxLon); - - int pointLatEnc = LatLonPoint.encodeLatitude(pointLat); - int pointLonEnc = LatLonPoint.encodeLongitude(pointLon); + // note: logic must be slightly different than rectContainsPoint, to satisfy + // insideness for cases exactly on boundaries. + + assert Double.isNaN(pointLat) == false; + assert rect.crossesDateline() == false; + double y[] = new double[] { rect.minLat, rect.maxLat, rect.maxLat, rect.minLat, rect.minLat }; + double x[] = new double[] { rect.minLon, rect.minLon, rect.maxLon, rect.maxLon, rect.minLon }; - if (rect.minLon < rect.maxLon) { - return pointLatEnc >= rectLatMinEnc && - pointLatEnc <= rectLatMaxEnc && - pointLonEnc >= rectLonMinEnc && - pointLonEnc <= rectLonMaxEnc; - } else { - // Rect crosses dateline: - return pointLatEnc >= rectLatMinEnc && - pointLatEnc <= rectLatMaxEnc && - (pointLonEnc >= rectLonMinEnc || - pointLonEnc <= rectLonMaxEnc); - } + // TODO: separately test this method is 100% correct, here treat it like a black box (like haversin) + return GeoRelationUtils.pointInPolygon(x, y, pointLat, pointLon); } @Override From 5da1ec91d069b83c5759aca4692c81a445e186b6 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Mar 2016 22:47:31 -0400 Subject: [PATCH 0186/1113] LUCENE-7125: remove additional confusion (we already quantize once, don't do it again). --- .../lucene/search/TestLatLonPointQueries.java | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 3b48eee9234a..5831095738d5 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -53,28 +53,14 @@ protected Query newPolygonQuery(String field, double[] lats, double[] lons) { @Override protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - assert Double.isNaN(pointLat) == false; - int rectLatMinEnc = LatLonPoint.encodeLatitude(rect.minLat); - int rectLatMaxEnc = LatLonPoint.encodeLatitude(rect.maxLat); - int rectLonMinEnc = LatLonPoint.encodeLongitude(rect.minLon); - int rectLonMaxEnc = LatLonPoint.encodeLongitude(rect.maxLon); - - int pointLatEnc = LatLonPoint.encodeLatitude(pointLat); - int pointLonEnc = LatLonPoint.encodeLongitude(pointLon); - if (rect.minLon < rect.maxLon) { - return pointLatEnc >= rectLatMinEnc && - pointLatEnc <= rectLatMaxEnc && - pointLonEnc >= rectLonMinEnc && - pointLonEnc <= rectLonMaxEnc; + return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); } else { // Rect crosses dateline: - return pointLatEnc >= rectLatMinEnc && - pointLatEnc <= rectLatMaxEnc && - (pointLonEnc >= rectLonMinEnc || - pointLonEnc <= rectLonMaxEnc); + return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, -180.0, rect.minLat, rect.maxLon, rect.maxLat) + || GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, 180.0, rect.maxLat); } } From c9a197490d446c80a252962c3899da140de520a2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Mar 2016 23:09:49 -0400 Subject: [PATCH 0187/1113] move tests from sandbox/ to spatial/ since they test GeoUtils methods (now with no unwanted quantization) --- .../TestLatLonPointDistanceQuery.java | 53 ------- .../document/TestLatLonPointDistanceSort.java | 16 -- .../lucene/search/TestLatLonPointQueries.java | 84 ---------- .../lucene/spatial/util/TestGeoUtils.java | 150 ++++++++++++++++++ 4 files changed, 150 insertions(+), 153 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java index 193c2a330276..983b1a7afa94 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java @@ -34,8 +34,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -189,55 +187,4 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException writer.close(); dir.close(); } - - public void testBoundingBoxOpto() { - for (int i = 0; i < 1000; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); - double radius = 50000000 * random().nextDouble(); - GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); - final GeoRect box1; - final GeoRect box2; - if (box.crossesDateline()) { - box1 = new GeoRect(-180, box.maxLon, box.minLat, box.maxLat); - box2 = new GeoRect(box.minLon, 180, box.minLat, box.maxLat); - } else { - box1 = box; - box2 = null; - } - - for (int j = 0; j < 10000; j++) { - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); - // if the point is within radius, then it should be in our bounding box - if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { - assertTrue(lat >= box.minLat && lat <= box.maxLat); - assertTrue(lon >= box1.minLon && lon <= box1.maxLon || (box2 != null && lon >= box2.minLon && lon <= box2.maxLon)); - } - } - } - } - - public void testHaversinOpto() { - for (int i = 0; i < 1000; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); - double radius = 50000000 * random().nextDouble(); - GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); - - if (box.maxLon - lon < 90 && lon - box.minLon < 90) { - double minPartialDistance = Math.max(SloppyMath.haversinSortKey(lat, lon, lat, box.maxLon), - SloppyMath.haversinSortKey(lat, lon, box.maxLat, lon)); - - for (int j = 0; j < 10000; j++) { - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); - // if the point is within radius, then it should be <= our sort key - if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { - assertTrue(SloppyMath.haversinSortKey(lat, lon, lat2, lon2) <= minPartialDistance); - } - } - } - } - } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 8e8b03de688c..0e5d898e41b9 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -27,8 +27,6 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -252,18 +250,4 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { writer.close(); dir.close(); } - - /** Test infinite radius covers whole earth */ - public void testInfiniteRect() { - for (int i = 0; i < 100000; i++) { - double centerLat = -90 + 180.0 * random().nextDouble(); - double centerLon = -180 + 360.0 * random().nextDouble(); - GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY); - assertEquals(-180.0, rect.minLon, 0.0D); - assertEquals(180.0, rect.maxLon, 0.0D); - assertEquals(-90.0, rect.minLat, 0.0D); - assertEquals(90.0, rect.maxLat, 0.0D); - assertFalse(rect.crossesDateline()); - } - } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 5831095738d5..e12923fffdfb 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -21,7 +21,6 @@ import org.apache.lucene.spatial.util.BaseGeoPointTestCase; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.SloppyMath; public class TestLatLonPointQueries extends BaseGeoPointTestCase { @@ -103,87 +102,4 @@ protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); return d >= minRadiusMeters && d <= radiusMeters; } - - /** Returns random double min to max or up to 1% outside of that range */ - private double randomRangeMaybeSlightlyOutside(double min, double max) { - return min + (random().nextDouble() + (0.5 - random().nextDouble()) * .02) * (max - min); - } - - // We rely heavily on GeoUtils.circleToBBox so we test it here: - public void testRandomCircleToBBox() throws Exception { - int iters = atLeast(1000); - for(int iter=0;iter= bbox.minLat && lat <= bbox.maxLat) { - bboxSays = lon <= bbox.maxLon || lon >= bbox.minLon; - } else { - bboxSays = false; - } - } else { - bboxSays = lat >= bbox.minLat && lat <= bbox.maxLat && lon >= bbox.minLon && lon <= bbox.maxLon; - } - - if (haversinSays) { - if (bboxSays == false) { - System.out.println("small=" + useSmallRanges + " centerLat=" + centerLat + " cetnerLon=" + centerLon + " radiusMeters=" + radiusMeters); - System.out.println(" bbox: lat=" + bbox.minLat + " to " + bbox.maxLat + " lon=" + bbox.minLon + " to " + bbox.maxLon); - System.out.println(" point: lat=" + lat + " lon=" + lon); - System.out.println(" haversin: " + distanceMeters); - fail("point was within the distance according to haversin, but the bbox doesn't contain it"); - } - } else { - // it's fine if haversin said it was outside the radius and bbox said it was inside the box - } - } - } - } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index bc76852c1e15..5529cdf2e90d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -563,4 +563,154 @@ public void testGeoPrefixCoding() throws Exception { } } } + + /** Returns random double min to max or up to 1% outside of that range */ + private double randomRangeMaybeSlightlyOutside(double min, double max) { + return min + (random().nextDouble() + (0.5 - random().nextDouble()) * .02) * (max - min); + } + + // We rely heavily on GeoUtils.circleToBBox so we test it here: + public void testRandomCircleToBBox() throws Exception { + int iters = atLeast(1000); + for(int iter=0;iter= bbox.minLat && lat <= bbox.maxLat) { + bboxSays = lon <= bbox.maxLon || lon >= bbox.minLon; + } else { + bboxSays = false; + } + } else { + bboxSays = lat >= bbox.minLat && lat <= bbox.maxLat && lon >= bbox.minLon && lon <= bbox.maxLon; + } + + if (haversinSays) { + if (bboxSays == false) { + System.out.println("small=" + useSmallRanges + " centerLat=" + centerLat + " cetnerLon=" + centerLon + " radiusMeters=" + radiusMeters); + System.out.println(" bbox: lat=" + bbox.minLat + " to " + bbox.maxLat + " lon=" + bbox.minLon + " to " + bbox.maxLon); + System.out.println(" point: lat=" + lat + " lon=" + lon); + System.out.println(" haversin: " + distanceMeters); + fail("point was within the distance according to haversin, but the bbox doesn't contain it"); + } + } else { + // it's fine if haversin said it was outside the radius and bbox said it was inside the box + } + } + } + } + + // similar to testRandomCircleToBBox, but different, less evil, maybe simpler + public void testBoundingBoxOpto() { + for (int i = 0; i < 1000; i++) { + double lat = -90 + 180.0 * random().nextDouble(); + double lon = -180 + 360.0 * random().nextDouble(); + double radius = 50000000 * random().nextDouble(); + GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); + final GeoRect box1; + final GeoRect box2; + if (box.crossesDateline()) { + box1 = new GeoRect(-180, box.maxLon, box.minLat, box.maxLat); + box2 = new GeoRect(box.minLon, 180, box.minLat, box.maxLat); + } else { + box1 = box; + box2 = null; + } + + for (int j = 0; j < 10000; j++) { + double lat2 = -90 + 180.0 * random().nextDouble(); + double lon2 = -180 + 360.0 * random().nextDouble(); + // if the point is within radius, then it should be in our bounding box + if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { + assertTrue(lat >= box.minLat && lat <= box.maxLat); + assertTrue(lon >= box1.minLon && lon <= box1.maxLon || (box2 != null && lon >= box2.minLon && lon <= box2.maxLon)); + } + } + } + } + + // test we can use haversinSortKey() for distance queries. + public void testHaversinOpto() { + for (int i = 0; i < 1000; i++) { + double lat = -90 + 180.0 * random().nextDouble(); + double lon = -180 + 360.0 * random().nextDouble(); + double radius = 50000000 * random().nextDouble(); + GeoRect box = GeoUtils.circleToBBox(lon, lat, radius); + + if (box.maxLon - lon < 90 && lon - box.minLon < 90) { + double minPartialDistance = Math.max(SloppyMath.haversinSortKey(lat, lon, lat, box.maxLon), + SloppyMath.haversinSortKey(lat, lon, box.maxLat, lon)); + + for (int j = 0; j < 10000; j++) { + double lat2 = -90 + 180.0 * random().nextDouble(); + double lon2 = -180 + 360.0 * random().nextDouble(); + // if the point is within radius, then it should be <= our sort key + if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { + assertTrue(SloppyMath.haversinSortKey(lat, lon, lat2, lon2) <= minPartialDistance); + } + } + } + } + } + + /** Test infinite radius covers whole earth */ + public void testInfiniteRect() { + for (int i = 0; i < 1000; i++) { + double centerLat = -90 + 180.0 * random().nextDouble(); + double centerLon = -180 + 360.0 * random().nextDouble(); + GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY); + assertEquals(-180.0, rect.minLon, 0.0D); + assertEquals(180.0, rect.maxLon, 0.0D); + assertEquals(-90.0, rect.minLat, 0.0D); + assertEquals(90.0, rect.maxLat, 0.0D); + assertFalse(rect.crossesDateline()); + } + } } From c7b7f1d027c6d887ccd7d7dd98d7dcee33021fd7 Mon Sep 17 00:00:00 2001 From: jbernste Date: Mon, 21 Mar 2016 23:37:51 -0400 Subject: [PATCH 0188/1113] SOLR-8882: Wait for checkpoint in DaemonStream, TopicStream testcase --- .../client/solrj/io/stream/StreamingTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java index 165029ffe42f..3091a19083fb 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java @@ -1390,6 +1390,26 @@ private void testDaemonTopicStream() throws Exception { daemonStream.open(); + // Wait for the checkpoint + CloudJettyRunner jetty = this.cloudJettys.get(0); + + Map params1 = new HashMap(); + params1.put("qt","/get"); + params1.put("ids","50000000"); + params1.put("fl","id"); + int count = 0; + while(count == 0) { + SolrStream solrStream = new SolrStream(jetty.url, params1); + List tuples = getTuples(solrStream); + count = tuples.size(); + if(count > 0) { + Tuple t = tuples.get(0); + assertTrue(t.getLong("id") == 50000000); + } else { + System.out.println("###### Waiting for checkpoint #######:" + count); + } + } + indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1"); indexr(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2"); indexr(id, "3", "a_s", "hello0", "a_i", "3", "a_f", "3"); From ee6e0acf0fe40f185a1bed77d99320c2f36ee7a0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Mar 2016 23:41:51 -0400 Subject: [PATCH 0189/1113] Separate simple tests from random tests, separate tests for different encodings. We can simplify the random tests to be easier to debug (like sandbox latlonpoint), but first we need to ensure that no coverage is lost: otherwise we might weaken tests with quantization. --- .../geopoint/search/TestGeoPointField.java | 238 +++++++++++++++ .../geopoint/search/TestGeoPointQuery.java | 283 +----------------- .../search/TestLegacyGeoPointField.java | 240 +++++++++++++++ .../search/TestLegacyGeoPointQuery.java | 127 ++++++++ .../lucene/spatial/util/TestGeoUtils.java | 45 +++ 5 files changed, 656 insertions(+), 277 deletions(-) create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java new file mode 100644 index 000000000000..fbbf643a9faa --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.geopoint.search; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; +import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; +import org.apache.lucene.spatial.util.GeoRelationUtils; +import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** Simple tests for GeoPoint */ +public class TestGeoPointField extends LuceneTestCase { + + private static Directory directory = null; + private static IndexReader reader = null; + private static IndexSearcher searcher = null; + private static final String FIELD_NAME = "point"; + + @BeforeClass + public static void beforeClass() throws Exception { + directory = newDirectory(); + + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, + newIndexWriterConfig(new MockAnalyzer(random())) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setMergePolicy(newLogMergePolicy())); + + // this is a simple systematic test + GeoPointField[] pts = new GeoPointField[] { + new GeoPointField(FIELD_NAME, -96.774, 32.763420, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.7759895324707, 32.7559529921407, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.77701950073242, 32.77866942010977, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.7706036567688, 32.7756745755423, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -139.73458170890808, 27.703618681345585, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.4538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.65084838867188, 33.06047141970814, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -96.7772, 32.778650, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -177.23537676036358, -88.56029371730983, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -26.779373834241003, 33.541429799076354, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -77.35379276106497, 26.774024500421728, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -14.796283808944777, -90.0, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -178.8538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 178.8538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -73.998776, 40.720611, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -179.5, -44.5, GeoPointField.PREFIX_TYPE_NOT_STORED)}; + + for (GeoPointField p : pts) { + Document doc = new Document(); + doc.add(p); + writer.addDocument(doc); + } + + // add explicit multi-valued docs + for (int i=0; i { + bboxQuery(179.0, -92.0, 181.0, -91.0, 20); + }); + } + + public void testGeoDistanceQuery() throws Exception { + TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000, 20); + assertEquals("GeoDistanceQuery failed", 2, td.totalHits); + } + + /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ + public void testNonEmptyTermsEnum() throws Exception { + TopDocs td = geoDistanceQuery(-177.23537676036358, -88.56029371730983, 7757.999232959935, 20); + assertEquals("GeoDistanceQuery failed", 2, td.totalHits); + } + + public void testMultiValuedQuery() throws Exception { + TopDocs td = bboxQuery(-96.4538113027811, 32.7559529921407, -96.7706036567688, 32.7756745755423, 20); + // 3 single valued docs + 2 multi-valued docs + assertEquals("testMultiValuedQuery failed", 5, td.totalHits); + } + + public void testTooBigRadius() throws Exception { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + geoDistanceQuery(0.0, 85.0, 4000000, 20); + }); + assertTrue(expected.getMessage().contains("exceeds maxRadius")); + } + + /** + * Explicitly large + */ + public void testGeoDistanceQueryHuge() throws Exception { + TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000000, 20); + assertEquals("GeoDistanceQuery failed", 16, td.totalHits); + } + + public void testGeoDistanceQueryCrossDateline() throws Exception { + TopDocs td = geoDistanceQuery(-179.9538113027811, 32.94823588839368, 120000, 20); + assertEquals("GeoDistanceQuery failed", 3, td.totalHits); + } + + // GeoDistanceQuery should not accept invalid lat/lon as origin + public void testInvalidGeoDistanceQuery() throws Exception { + expectThrows(Exception.class, () -> { + geoDistanceQuery(181.0, 92.0, 120000, 20); + }); + } + + public void testMaxDistanceRangeQuery() throws Exception { + TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); + assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 2fd64bd3d839..3bf22d335fb1 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -16,45 +16,26 @@ */ package org.apache.lucene.spatial.geopoint.search; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.SloppyMath; -import org.apache.lucene.util.TestUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; import static org.apache.lucene.spatial.util.GeoDistanceUtils.DISTANCE_PCT_ERR; /** - * Unit testing for basic GeoPoint query logic + * random testing for GeoPoint query logic * * @lucene.experimental */ public class TestGeoPointQuery extends BaseGeoPointTestCase { - private static Directory directory = null; - private static IndexReader reader = null; - private static IndexSearcher searcher = null; - private static TermEncoding termEncoding = null; - private static FieldType fieldType = null; - @Override protected boolean forceSmall() { return false; @@ -62,118 +43,27 @@ protected boolean forceSmall() { @Override protected void addPointToDoc(String field, Document doc, double lat, double lon) { - doc.add(new GeoPointField(field, lon, lat, fieldType)); + doc.add(new GeoPointField(field, lon, lat, GeoPointField.PREFIX_TYPE_NOT_STORED)); } @Override protected Query newRectQuery(String field, GeoRect rect) { - return new GeoPointInBBoxQuery(field, termEncoding, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + return new GeoPointInBBoxQuery(field, TermEncoding.PREFIX, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); } @Override protected Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters) { - return new GeoPointDistanceQuery(field, termEncoding, centerLon, centerLat, radiusMeters); + return new GeoPointDistanceQuery(field, TermEncoding.PREFIX, centerLon, centerLat, radiusMeters); } @Override protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return new GeoPointDistanceRangeQuery(field, termEncoding, centerLon, centerLat, minRadiusMeters, radiusMeters); + return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLon, centerLat, minRadiusMeters, radiusMeters); } @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return new GeoPointInPolygonQuery(field, termEncoding, lons, lats); - } - - @BeforeClass - public static void beforeClass() throws Exception { - directory = newDirectory(); - termEncoding = randomTermEncoding(); - fieldType = randomFieldType(); - - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random())) - .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) - .setMergePolicy(newLogMergePolicy())); - - // this is a simple systematic test - GeoPointField[] pts = new GeoPointField[] { - new GeoPointField(FIELD_NAME, -96.774, 32.763420, fieldType), - new GeoPointField(FIELD_NAME, -96.7759895324707, 32.7559529921407, fieldType), - new GeoPointField(FIELD_NAME, -96.77701950073242, 32.77866942010977, fieldType), - new GeoPointField(FIELD_NAME, -96.7706036567688, 32.7756745755423, fieldType), - new GeoPointField(FIELD_NAME, -139.73458170890808, 27.703618681345585, fieldType), - new GeoPointField(FIELD_NAME, -96.4538113027811, 32.94823588839368, fieldType), - new GeoPointField(FIELD_NAME, -96.65084838867188, 33.06047141970814, fieldType), - new GeoPointField(FIELD_NAME, -96.7772, 32.778650, fieldType), - new GeoPointField(FIELD_NAME, -177.23537676036358, -88.56029371730983, fieldType), - new GeoPointField(FIELD_NAME, -26.779373834241003, 33.541429799076354, fieldType), - new GeoPointField(FIELD_NAME, -77.35379276106497, 26.774024500421728, fieldType), - new GeoPointField(FIELD_NAME, -14.796283808944777, -90.0, fieldType), - new GeoPointField(FIELD_NAME, -178.8538113027811, 32.94823588839368, fieldType), - new GeoPointField(FIELD_NAME, 178.8538113027811, 32.94823588839368, fieldType), - new GeoPointField(FIELD_NAME, -73.998776, 40.720611, fieldType), - new GeoPointField(FIELD_NAME, -179.5, -44.5, fieldType)}; - - for (GeoPointField p : pts) { - Document doc = new Document(); - doc.add(p); - writer.addDocument(doc); - } - - // add explicit multi-valued docs - for (int i=0; i { - bboxQuery(179.0, -92.0, 181.0, -91.0, 20); - }); - } - - public void testGeoDistanceQuery() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ - public void testNonEmptyTermsEnum() throws Exception { - TopDocs td = geoDistanceQuery(-177.23537676036358, -88.56029371730983, 7757.999232959935, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - public void testMultiValuedQuery() throws Exception { - TopDocs td = bboxQuery(-96.4538113027811, 32.7559529921407, -96.7706036567688, 32.7756745755423, 20); - // 3 single valued docs + 2 multi-valued docs - assertEquals("testMultiValuedQuery failed", 5, td.totalHits); - } - - public void testTooBigRadius() throws Exception { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - geoDistanceQuery(0.0, 85.0, 4000000, 20); - }); - assertTrue(expected.getMessage().contains("exceeds maxRadius")); - } - - /** - * Explicitly large - */ - public void testGeoDistanceQueryHuge() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000000, 20); - assertEquals("GeoDistanceQuery failed", 16, td.totalHits); - } - - public void testGeoDistanceQueryCrossDateline() throws Exception { - TopDocs td = geoDistanceQuery(-179.9538113027811, 32.94823588839368, 120000, 20); - assertEquals("GeoDistanceQuery failed", 3, td.totalHits); - } - - // GeoDistanceQuery should not accept invalid lat/lon as origin - public void testInvalidGeoDistanceQuery() throws Exception { - expectThrows(Exception.class, () -> { - geoDistanceQuery(181.0, 92.0, 120000, 20); - }); - } - - public void testMaxDistanceRangeQuery() throws Exception { - TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); - assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); - } - - public void testMortonEncoding() throws Exception { - long hash = GeoEncodingUtils.mortonHash(180, 90); - assertEquals(180.0, GeoEncodingUtils.mortonUnhashLon(hash), 0); - assertEquals(90.0, GeoEncodingUtils.mortonUnhashLat(hash), 0); - } - - public void testEncodeDecode() throws Exception { - int iters = atLeast(10000); - boolean small = random().nextBoolean(); - for(int iter=0;iter { + bboxQuery(179.0, -92.0, 181.0, -91.0, 20); + }); + } + + public void testGeoDistanceQuery() throws Exception { + TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000, 20); + assertEquals("GeoDistanceQuery failed", 2, td.totalHits); + } + + /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ + public void testNonEmptyTermsEnum() throws Exception { + TopDocs td = geoDistanceQuery(-177.23537676036358, -88.56029371730983, 7757.999232959935, 20); + assertEquals("GeoDistanceQuery failed", 2, td.totalHits); + } + + public void testMultiValuedQuery() throws Exception { + TopDocs td = bboxQuery(-96.4538113027811, 32.7559529921407, -96.7706036567688, 32.7756745755423, 20); + // 3 single valued docs + 2 multi-valued docs + assertEquals("testMultiValuedQuery failed", 5, td.totalHits); + } + + public void testTooBigRadius() throws Exception { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + geoDistanceQuery(0.0, 85.0, 4000000, 20); + }); + assertTrue(expected.getMessage().contains("exceeds maxRadius")); + } + + /** + * Explicitly large + */ + public void testGeoDistanceQueryHuge() throws Exception { + TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000000, 20); + assertEquals("GeoDistanceQuery failed", 16, td.totalHits); + } + + public void testGeoDistanceQueryCrossDateline() throws Exception { + TopDocs td = geoDistanceQuery(-179.9538113027811, 32.94823588839368, 120000, 20); + assertEquals("GeoDistanceQuery failed", 3, td.totalHits); + } + + // GeoDistanceQuery should not accept invalid lat/lon as origin + public void testInvalidGeoDistanceQuery() throws Exception { + expectThrows(Exception.class, () -> { + geoDistanceQuery(181.0, 92.0, 120000, 20); + }); + } + + public void testMaxDistanceRangeQuery() throws Exception { + TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); + assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java new file mode 100644 index 000000000000..c20faffd0b28 --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.geopoint.search; + +import org.apache.lucene.document.Document; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; +import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; +import org.apache.lucene.spatial.util.BaseGeoPointTestCase; +import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoRelationUtils; +import org.apache.lucene.util.SloppyMath; + +import static org.apache.lucene.spatial.util.GeoDistanceUtils.DISTANCE_PCT_ERR; + +/** + * random testing for GeoPoint query logic (with deprecated numeric encoding) + * @deprecated remove this when TermEncoding.NUMERIC is removed + */ +@Deprecated +public class TestLegacyGeoPointQuery extends BaseGeoPointTestCase { + + @Override + protected boolean forceSmall() { + return false; + } + + @Override + protected void addPointToDoc(String field, Document doc, double lat, double lon) { + doc.add(new GeoPointField(field, lon, lat, GeoPointField.NUMERIC_TYPE_NOT_STORED)); + } + + @Override + protected Query newRectQuery(String field, GeoRect rect) { + return new GeoPointInBBoxQuery(field, TermEncoding.NUMERIC, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + } + + @Override + protected Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters) { + return new GeoPointDistanceQuery(field, TermEncoding.NUMERIC, centerLon, centerLat, radiusMeters); + } + + @Override + protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { + return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLon, centerLat, minRadiusMeters, radiusMeters); + } + + @Override + protected Query newPolygonQuery(String field, double[] lats, double[] lons) { + return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lons, lats); + } + + @Override + protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { + if (GeoEncodingUtils.compare(pointLon, rect.minLon) == 0.0 || + GeoEncodingUtils.compare(pointLon, rect.maxLon) == 0.0 || + GeoEncodingUtils.compare(pointLat, rect.minLat) == 0.0 || + GeoEncodingUtils.compare(pointLat, rect.maxLat) == 0.0) { + // Point is very close to rect boundary + return null; + } + + if (rect.minLon < rect.maxLon) { + return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + } else { + // Rect crosses dateline: + return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, -180.0, rect.minLat, rect.maxLon, rect.maxLat) + || GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, 180.0, rect.maxLat); + } + } + + @Override + protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { + return rectContainsPoint(rect, pointLat, pointLon); + } + + @Override + protected Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { + if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { + return null; + } else { + return SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon) <= radiusMeters; + } + } + + @Override + protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { + if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, minRadiusMeters) + || radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { + return null; + } else { + final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); + return d >= minRadiusMeters && d <= radiusMeters; + } + } + + private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, double ptLon, double ptLat, + final double radius) { + final long hashedCntr = GeoEncodingUtils.mortonHash(centerLon, centerLat); + centerLon = GeoEncodingUtils.mortonUnhashLon(hashedCntr); + centerLat = GeoEncodingUtils.mortonUnhashLat(hashedCntr); + final long hashedPt = GeoEncodingUtils.mortonHash(ptLon, ptLat); + ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); + ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); + + double ptDistance = SloppyMath.haversinMeters(centerLat, centerLon, ptLat, ptLon); + double delta = StrictMath.abs(ptDistance - radius); + + // if its within the distance error then it can be wrong + return delta < (ptDistance*DISTANCE_PCT_ERR); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 5529cdf2e90d..37938328725d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -564,6 +564,47 @@ public void testGeoPrefixCoding() throws Exception { } } + public void testMortonEncoding() throws Exception { + long hash = GeoEncodingUtils.mortonHash(180, 90); + assertEquals(180.0, GeoEncodingUtils.mortonUnhashLon(hash), 0); + assertEquals(90.0, GeoEncodingUtils.mortonUnhashLat(hash), 0); + } + + public void testEncodeDecode() throws Exception { + int iters = atLeast(10000); + boolean small = random().nextBoolean(); + for(int iter=0;iter Date: Tue, 22 Mar 2016 11:52:26 -0400 Subject: [PATCH 0191/1113] LUCENE-7128: clean up new geo APIs to consistently take lat before lon, make methods private when possible, use lat/lon instead of y/x naming, remove unused code --- .../LatLonPointDistanceComparator.java | 2 +- .../document/LatLonPointDistanceQuery.java | 2 +- .../document/LatLonPointInPolygonQuery.java | 18 +- .../lucene/search/TestLatLonPointQueries.java | 12 +- .../geopoint/document/GeoPointField.java | 32 +- .../document/GeoPointTokenStream.java | 8 +- .../search/GeoPointDistanceQuery.java | 44 +- .../search/GeoPointDistanceQueryImpl.java | 21 +- .../search/GeoPointDistanceRangeQuery.java | 22 +- .../geopoint/search/GeoPointInBBoxQuery.java | 68 +-- .../search/GeoPointInBBoxQueryImpl.java | 49 +-- .../search/GeoPointInPolygonQuery.java | 44 +- .../search/GeoPointInPolygonQueryImpl.java | 26 +- .../search/GeoPointMultiTermQuery.java | 42 +- .../search/GeoPointNumericTermsEnum.java | 6 +- .../search/GeoPointPrefixTermsEnum.java | 14 +- ...GeoPointTermQueryConstantScoreWrapper.java | 2 +- .../geopoint/search/GeoPointTermsEnum.java | 4 +- .../lucene/spatial/util/GeoDistanceUtils.java | 86 +--- .../lucene/spatial/util/GeoEncodingUtils.java | 7 +- .../lucene/spatial/util/GeoHashUtils.java | 283 ------------- .../spatial/util/GeoProjectionUtils.java | 327 +-------------- .../apache/lucene/spatial/util/GeoRect.java | 10 +- .../lucene/spatial/util/GeoRelationUtils.java | 388 +++++++++--------- .../apache/lucene/spatial/util/GeoUtils.java | 47 +-- .../geopoint/search/TestGeoPointField.java | 94 ++--- .../geopoint/search/TestGeoPointQuery.java | 20 +- .../search/TestLegacyGeoPointField.java | 92 +++-- .../search/TestLegacyGeoPointQuery.java | 20 +- .../spatial/util/BaseGeoPointTestCase.java | 2 +- .../lucene/spatial/util/TestGeoUtils.java | 183 +-------- 31 files changed, 580 insertions(+), 1395 deletions(-) delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoHashUtils.java diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index 2f142baa6afd..e818e1d2baf3 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -83,7 +83,7 @@ public void setBottom(int slot) { // sampling if we get called way too much: don't make gobs of bounding // boxes if comparator hits a worst case order (e.g. backwards distance order) if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { - GeoRect box = GeoUtils.circleToBBox(longitude, latitude, haversin2(bottom)); + GeoRect box = GeoUtils.circleToBBox(latitude, longitude, haversin2(bottom)); // pre-encode our box to our integer encoding, so we don't have to decode // to double values for uncompetitive hits. This has some cost! minLat = LatLonPoint.encodeLatitude(box.minLat); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 8f4b3475d728..5e7a04e76572 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -75,7 +75,7 @@ public LatLonPointDistanceQuery(String field, double latitude, double longitude, @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - GeoRect box = GeoUtils.circleToBBox(longitude, latitude, radiusMeters); + GeoRect box = GeoUtils.circleToBBox(latitude, longitude, radiusMeters); // create bounding box(es) for the distance range // these are pre-encoded with LatLonPoint's encoding final byte minLat[] = new byte[Integer.BYTES]; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index cb988951b94f..68c734e9a492 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -169,13 +169,13 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { if (cellMinLat <= minLat && cellMaxLat >= maxLat && cellMinLon <= minLon && cellMaxLon >= maxLon) { // Cell fully encloses the query return Relation.CELL_CROSSES_QUERY; - } else if (GeoRelationUtils.rectWithinPolyPrecise(cellMinLon, cellMinLat, cellMaxLon, cellMaxLat, - polyLons, polyLats, - minLon, minLat, maxLon, maxLat)) { + } else if (GeoRelationUtils.rectWithinPolyPrecise(cellMinLat, cellMaxLat, cellMinLon, cellMaxLon, + polyLats, polyLons, + minLat, maxLat, minLon, maxLon)) { return Relation.CELL_INSIDE_QUERY; - } else if (GeoRelationUtils.rectCrossesPolyPrecise(cellMinLon, cellMinLat, cellMaxLon, cellMaxLat, - polyLons, polyLats, - minLon, minLat, maxLon, maxLat)) { + } else if (GeoRelationUtils.rectCrossesPolyPrecise(cellMinLat, cellMaxLat, cellMinLon, cellMaxLon, + polyLats, polyLons, + minLat, maxLat, minLon, maxLon)) { return Relation.CELL_CROSSES_QUERY; } else { return Relation.CELL_OUTSIDE_QUERY; @@ -204,7 +204,7 @@ public boolean matches() throws IOException { long encoded = docValues.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - if (GeoRelationUtils.pointInPolygon(polyLons, polyLats, docLatitude, docLongitude)) { + if (GeoRelationUtils.pointInPolygon(polyLats, polyLons, docLatitude, docLongitude)) { return true; } } @@ -289,9 +289,9 @@ public String toString(String field) { sb.append(" Points: "); for (int i=0; i * // using prefix terms - * GeoPointField geoPointField = new GeoPointField(fieldName1, lon, lat, GeoPointField.PREFIX_TYPE_NOT_STORED); + * GeoPointField geoPointField = new GeoPointField(fieldName1, lat, lon, GeoPointField.PREFIX_TYPE_NOT_STORED); * document.add(geoPointField); * * // query by bounding box (default uses TermEncoding.PREFIX) * Query q = new GeoPointInBBoxQuery(fieldName1, minLon, minLat, maxLon, maxLat); * * // using numeric terms - * geoPointField = new GeoPointField(fieldName2, lon, lat, GeoPointField.NUMERIC_TYPE_NOT_STORED); + * geoPointField = new GeoPointField(fieldName2, lat, lon, GeoPointField.NUMERIC_TYPE_NOT_STORED); * document.add(geoPointField); * * // query by distance (requires TermEncoding.NUMERIC) - * q = new GeoPointDistanceQuery(fieldName2, TermEncoding.NUMERIC, centerLon, centerLat, radiusMeters); + * q = new GeoPointDistanceQuery(fieldName2, TermEncoding.NUMERIC, centerLat, centerLon, radiusMeters); * * * @lucene.experimental @@ -230,4 +230,4 @@ public boolean equals(Object obj) { public String toString() { return getClass().getSimpleName() + "(precisionStep=" + PRECISION_STEP + " shift=" + geoPointTermAtt.getShift() + ")"; } -} \ No newline at end of file +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java index a513b3d27c02..2486852f87d7 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java @@ -42,19 +42,19 @@ * * @lucene.experimental */ public class GeoPointDistanceQuery extends GeoPointInBBoxQuery { - /** longitude value (in degrees) for query location */ - protected final double centerLon; /** latitude value (in degrees) for query location */ protected final double centerLat; - /** distance (in meters) from lon, lat center location */ + /** longitude value (in degrees) for query location */ + protected final double centerLon; + /** distance (in meters) from lat, lon center location */ protected final double radiusMeters; /** * Constructs a Query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types within a * distance (in meters) from a given point **/ - public GeoPointDistanceQuery(final String field, final double centerLon, final double centerLat, final double radiusMeters) { - this(field, TermEncoding.PREFIX, centerLon, centerLat, radiusMeters); + public GeoPointDistanceQuery(final String field, final double centerLat, final double centerLon, final double radiusMeters) { + this(field, TermEncoding.PREFIX, centerLat, centerLon, radiusMeters); } /** @@ -62,38 +62,38 @@ public GeoPointDistanceQuery(final String field, final double centerLon, final d * distance (in meters) from a given point. Accepts optional * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter **/ - public GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final double centerLon, final double centerLat, final double radiusMeters) { - this(field, termEncoding, GeoUtils.circleToBBox(centerLon, centerLat, radiusMeters), centerLon, centerLat, radiusMeters); + public GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, final double radiusMeters) { + this(field, termEncoding, GeoUtils.circleToBBox(centerLat, centerLon, radiusMeters), centerLat, centerLon, radiusMeters); } - private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final GeoRect bbox, final double centerLon, - final double centerLat, final double radiusMeters) { - super(field, termEncoding, bbox.minLon, bbox.minLat, bbox.maxLon, bbox.maxLat); + private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final GeoRect bbox, + final double centerLat, final double centerLon, final double radiusMeters) { + super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); { // check longitudinal overlap (restrict distance to maximum longitudinal radius) // todo this restriction technically shouldn't be needed, // its only purpose is to ensure the bounding box doesn't self overlap. - final double maxRadius = GeoDistanceUtils.maxRadialDistanceMeters(centerLon, centerLat); + final double maxRadius = GeoDistanceUtils.maxRadialDistanceMeters(centerLat, centerLon); if (radiusMeters > maxRadius) { throw new IllegalArgumentException("radiusMeters " + radiusMeters + " exceeds maxRadius [" + maxRadius - + "] at location [" + centerLon + " " + centerLat + "]"); + + "] at location [" + centerLat + " " + centerLon + "]"); } } - if (GeoUtils.isValidLon(centerLon) == false) { - throw new IllegalArgumentException("invalid centerLon " + centerLon); - } - if (GeoUtils.isValidLat(centerLat) == false) { throw new IllegalArgumentException("invalid centerLat " + centerLat); } + if (GeoUtils.isValidLon(centerLon) == false) { + throw new IllegalArgumentException("invalid centerLon " + centerLon); + } + if (radiusMeters <= 0.0) { throw new IllegalArgumentException("invalid radiusMeters " + radiusMeters); } - this.centerLon = centerLon; this.centerLat = centerLat; + this.centerLon = centerLon; this.radiusMeters = radiusMeters; } @@ -110,7 +110,7 @@ public Query rewrite(IndexReader reader) { unwrappedLon += -360.0D; } GeoPointDistanceQueryImpl left = new GeoPointDistanceQueryImpl(field, termEncoding, this, unwrappedLon, - new GeoRect(GeoUtils.MIN_LON_INCL, maxLon, minLat, maxLat)); + new GeoRect(minLat, maxLat, GeoUtils.MIN_LON_INCL, maxLon)); bqb.add(new BooleanClause(left, BooleanClause.Occur.SHOULD)); if (unwrappedLon < maxLon) { @@ -118,13 +118,13 @@ public Query rewrite(IndexReader reader) { unwrappedLon += 360.0D; } GeoPointDistanceQueryImpl right = new GeoPointDistanceQueryImpl(field, termEncoding, this, unwrappedLon, - new GeoRect(minLon, GeoUtils.MAX_LON_INCL, minLat, maxLat)); + new GeoRect(minLat, maxLat, minLon, GeoUtils.MAX_LON_INCL)); bqb.add(new BooleanClause(right, BooleanClause.Occur.SHOULD)); return bqb.build(); } return new GeoPointDistanceQueryImpl(field, termEncoding, this, centerLon, - new GeoRect(this.minLon, this.maxLon, this.minLat, this.maxLat)); + new GeoRect(this.minLat, this.maxLat, this.minLon, this.maxLon)); } @Override @@ -166,9 +166,9 @@ public String toString(String field) { sb.append(':'); } return sb.append( " Center: [") - .append(centerLon) - .append(',') .append(centerLat) + .append(',') + .append(centerLon) .append(']') .append(" Distance: ") .append(radiusMeters) diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index 6a54e23688dc..8f50db5bed6c 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -32,7 +32,7 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { GeoPointDistanceQueryImpl(final String field, final TermEncoding termEncoding, final GeoPointDistanceQuery q, final double centerLonUnwrapped, final GeoRect bbox) { - super(field, termEncoding, bbox.minLon, bbox.minLat, bbox.maxLon, bbox.maxLat); + super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); distanceQuery = q; centerLon = centerLonUnwrapped; } @@ -53,20 +53,21 @@ private final class GeoPointRadiusCellComparator extends CellComparator { } @Override - protected boolean cellCrosses(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return GeoRelationUtils.rectCrossesCircle(minLon, minLat, maxLon, maxLat, - centerLon, distanceQuery.centerLat, distanceQuery.radiusMeters, true); + protected boolean cellCrosses(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return GeoRelationUtils.rectCrossesCircle(minLat, maxLat, minLon, maxLon, + distanceQuery.centerLat, centerLon, distanceQuery.radiusMeters, true); } @Override - protected boolean cellWithin(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return GeoRelationUtils.rectWithinCircle(minLon, minLat, maxLon, maxLat, - centerLon, distanceQuery.centerLat, distanceQuery.radiusMeters, true); + protected boolean cellWithin(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return GeoRelationUtils.rectWithinCircle(minLat, maxLat, minLon, maxLon, + distanceQuery.centerLat, centerLon, + distanceQuery.radiusMeters, true); } @Override - protected boolean cellIntersectsShape(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return cellCrosses(minLon, minLat, maxLon, maxLat); + protected boolean cellIntersectsShape(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return cellCrosses(minLat, maxLat, minLon, maxLon); } /** @@ -76,7 +77,7 @@ protected boolean cellIntersectsShape(final double minLon, final double minLat, * {@link org.apache.lucene.util.SloppyMath#haversinMeters(double, double, double, double)} method. */ @Override - protected boolean postFilter(final double lon, final double lat) { + protected boolean postFilter(final double lat, final double lon) { return SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, lat, lon) <= distanceQuery.radiusMeters; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java index f24aa6af2e05..bdfbb882fc49 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java @@ -37,9 +37,9 @@ public final class GeoPointDistanceRangeQuery extends GeoPointDistanceQuery { * Constructs a query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types within a minimum / maximum * distance (in meters) range from a given point */ - public GeoPointDistanceRangeQuery(final String field, final double centerLon, final double centerLat, + public GeoPointDistanceRangeQuery(final String field, final double centerLat, final double centerLon, final double minRadiusMeters, final double maxRadiusMeters) { - this(field, TermEncoding.PREFIX, centerLon, centerLat, minRadiusMeters, maxRadiusMeters); + this(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, maxRadiusMeters); } /** @@ -47,9 +47,9 @@ public GeoPointDistanceRangeQuery(final String field, final double centerLon, fi * distance (in meters) range from a given point. Accepts an optional * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} */ - public GeoPointDistanceRangeQuery(final String field, final TermEncoding termEncoding, final double centerLon, final double centerLat, + public GeoPointDistanceRangeQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, final double minRadiusMeters, final double maxRadius) { - super(field, termEncoding, centerLon, centerLat, maxRadius); + super(field, termEncoding, centerLat, centerLon, maxRadius); this.minRadiusMeters = minRadiusMeters; } @@ -64,7 +64,7 @@ public Query rewrite(IndexReader reader) { BooleanQuery.Builder bqb = new BooleanQuery.Builder(); // create a new exclusion query - GeoPointDistanceQuery exclude = new GeoPointDistanceQuery(field, termEncoding, centerLon, centerLat, minRadiusMeters); + GeoPointDistanceQuery exclude = new GeoPointDistanceQuery(field, termEncoding, centerLat, centerLon, minRadiusMeters); // full map search // if (radiusMeters >= GeoProjectionUtils.SEMIMINOR_AXIS) { // bqb.add(new BooleanClause(new GeoPointInBBoxQuery(this.field, -180.0, -90.0, 180.0, 90.0), BooleanClause.Occur.MUST)); @@ -87,9 +87,9 @@ public String toString(String field) { sb.append(':'); } return sb.append( " Center: [") - .append(centerLon) - .append(',') .append(centerLat) + .append(',') + .append(centerLon) .append(']') .append(" From Distance: ") .append(minRadiusMeters) @@ -98,14 +98,14 @@ public String toString(String field) { .append(radiusMeters) .append(" m") .append(" Lower Left: [") - .append(minLon) - .append(',') .append(minLat) + .append(',') + .append(minLon) .append(']') .append(" Upper Right: [") - .append(maxLon) - .append(',') .append(maxLat) + .append(',') + .append(maxLon) .append("]") .toString(); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java index 32c0aecf9d23..f81f37ed9294 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java @@ -47,14 +47,14 @@ public class GeoPointInBBoxQuery extends Query { /** field name */ protected final String field; - /** minimum longitude value (in degrees) */ - protected final double minLon; /** minimum latitude value (in degrees) */ protected final double minLat; - /** maximum longitude value (in degrees) */ - protected final double maxLon; + /** minimum longitude value (in degrees) */ + protected final double minLon; /** maximum latitude value (in degrees) */ protected final double maxLat; + /** maximum longitude value (in degrees) */ + protected final double maxLon; /** term encoding enum to define how the points are encoded (PREFIX or NUMERIC) */ protected final TermEncoding termEncoding; @@ -62,28 +62,28 @@ public class GeoPointInBBoxQuery extends Query { * Constructs a query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types that fall within a * defined bounding box */ - public GeoPointInBBoxQuery(final String field, final double minLon, final double minLat, final double maxLon, final double maxLat) { - this(field, TermEncoding.PREFIX, minLon, minLat, maxLon, maxLat); + public GeoPointInBBoxQuery(final String field, final double minLat, final double maxLat, final double minLon, final double maxLon) { + this(field, TermEncoding.PREFIX, minLat, maxLat, minLon, maxLon); } /** * Constructs a query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types that fall within a * defined bounding box. Accepts optional {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter */ - public GeoPointInBBoxQuery(final String field, final TermEncoding termEncoding, final double minLon, final double minLat, final double maxLon, final double maxLat) { + public GeoPointInBBoxQuery(final String field, final TermEncoding termEncoding, final double minLat, final double maxLat, final double minLon, final double maxLon) { this.field = field; - this.minLon = minLon; this.minLat = minLat; - this.maxLon = maxLon; this.maxLat = maxLat; + this.minLon = minLon; + this.maxLon = maxLon; this.termEncoding = termEncoding; } @Override public Query rewrite(IndexReader reader) { // short-circuit to match all if specifying the whole map - if (minLon == GeoUtils.MIN_LON_INCL && maxLon == GeoUtils.MAX_LON_INCL - && minLat == GeoUtils.MIN_LAT_INCL && maxLat == GeoUtils.MAX_LAT_INCL) { + if (minLat == GeoUtils.MIN_LAT_INCL && maxLat == GeoUtils.MAX_LAT_INCL && + minLon == GeoUtils.MIN_LON_INCL && maxLon == GeoUtils.MAX_LON_INCL) { // FieldValueQuery is valid since DocValues are *required* for GeoPointField return new FieldValueQuery(field); } @@ -91,13 +91,13 @@ public Query rewrite(IndexReader reader) { if (maxLon < minLon) { BooleanQuery.Builder bqb = new BooleanQuery.Builder(); - GeoPointInBBoxQueryImpl left = new GeoPointInBBoxQueryImpl(field, termEncoding, -180.0D, minLat, maxLon, maxLat); + GeoPointInBBoxQueryImpl left = new GeoPointInBBoxQueryImpl(field, termEncoding, minLat, maxLat, -180.0D, maxLon); bqb.add(new BooleanClause(left, BooleanClause.Occur.SHOULD)); - GeoPointInBBoxQueryImpl right = new GeoPointInBBoxQueryImpl(field, termEncoding, minLon, minLat, 180.0D, maxLat); + GeoPointInBBoxQueryImpl right = new GeoPointInBBoxQueryImpl(field, termEncoding, minLat, maxLat, minLon, 180.0D); bqb.add(new BooleanClause(right, BooleanClause.Occur.SHOULD)); return bqb.build(); } - return new GeoPointInBBoxQueryImpl(field, termEncoding, minLon, minLat, maxLon, maxLat); + return new GeoPointInBBoxQueryImpl(field, termEncoding, minLat, maxLat, minLon, maxLon); } @Override @@ -111,14 +111,14 @@ public String toString(String field) { sb.append(':'); } return sb.append(" Lower Left: [") - .append(minLon) - .append(',') .append(minLat) + .append(',') + .append(minLon) .append(']') .append(" Upper Right: [") - .append(maxLon) - .append(',') .append(maxLat) + .append(',') + .append(maxLon) .append("]") .toString(); } @@ -131,10 +131,10 @@ public boolean equals(Object o) { GeoPointInBBoxQuery that = (GeoPointInBBoxQuery) o; - if (Double.compare(that.maxLat, maxLat) != 0) return false; - if (Double.compare(that.maxLon, maxLon) != 0) return false; if (Double.compare(that.minLat, minLat) != 0) return false; + if (Double.compare(that.maxLat, maxLat) != 0) return false; if (Double.compare(that.minLon, minLon) != 0) return false; + if (Double.compare(that.maxLon, maxLon) != 0) return false; if (!field.equals(that.field)) return false; return true; @@ -145,14 +145,14 @@ public int hashCode() { int result = super.hashCode(); long temp; result = 31 * result + field.hashCode(); - temp = Double.doubleToLongBits(minLon); - result = 31 * result + (int) (temp ^ (temp >>> 32)); temp = Double.doubleToLongBits(minLat); result = 31 * result + (int) (temp ^ (temp >>> 32)); - temp = Double.doubleToLongBits(maxLon); - result = 31 * result + (int) (temp ^ (temp >>> 32)); temp = Double.doubleToLongBits(maxLat); result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(minLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); return result; } @@ -161,23 +161,23 @@ public final String getField() { return this.field; } - /** getter method for retrieving the minimum longitude (in degrees) */ - public final double getMinLon() { - return this.minLon; - } - /** getter method for retrieving the minimum latitude (in degrees) */ public final double getMinLat() { return this.minLat; } - /** getter method for retrieving the maximum longitude (in degrees) */ - public final double getMaxLon() { - return this.maxLon; - } - /** getter method for retrieving the maximum latitude (in degrees) */ public final double getMaxLat() { return this.maxLat; } + + /** getter method for retrieving the minimum longitude (in degrees) */ + public final double getMinLon() { + return this.minLon; + } + + /** getter method for retrieving the maximum longitude (in degrees) */ + public final double getMaxLon() { + return this.maxLon; + } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java index 107b1364f39f..c19e6d2866a4 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQueryImpl.java @@ -36,8 +36,8 @@ class GeoPointInBBoxQueryImpl extends GeoPointMultiTermQuery { * @param maxLon upper longitude (x) value of the bounding box * @param maxLat upper latitude (y) value of the bounding box */ - GeoPointInBBoxQueryImpl(final String field, final TermEncoding termEncoding, final double minLon, final double minLat, final double maxLon, final double maxLat) { - super(field, termEncoding, minLon, minLat, maxLon, maxLat); + GeoPointInBBoxQueryImpl(final String field, final TermEncoding termEncoding, final double minLat, final double maxLat, final double minLon, final double maxLon) { + super(field, termEncoding, minLat, maxLat, minLon, maxLon); } @Override @@ -50,8 +50,8 @@ protected short computeMaxShift() { final short shiftFactor; // compute diagonal radius - double midLon = (minLon + maxLon) * 0.5; double midLat = (minLat + maxLat) * 0.5; + double midLon = (minLon + maxLon) * 0.5; if (SloppyMath.haversinMeters(minLat, minLon, midLat, midLon) > 1000000) { shiftFactor = 5; @@ -76,26 +76,29 @@ private final class GeoPointInBBoxCellComparator extends CellComparator { * Determine whether the quad-cell crosses the shape */ @Override - protected boolean cellCrosses(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return GeoRelationUtils.rectCrosses(minLon, minLat, maxLon, maxLat, GeoPointInBBoxQueryImpl.this.minLon, - GeoPointInBBoxQueryImpl.this.minLat, GeoPointInBBoxQueryImpl.this.maxLon, GeoPointInBBoxQueryImpl.this.maxLat); } + protected boolean cellCrosses(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return GeoRelationUtils.rectCrosses(minLat, maxLat, minLon, maxLon, GeoPointInBBoxQueryImpl.this.minLat, + GeoPointInBBoxQueryImpl.this.maxLat, GeoPointInBBoxQueryImpl.this.minLon, GeoPointInBBoxQueryImpl.this.maxLon); + } /** * Determine whether quad-cell is within the shape */ @Override - protected boolean cellWithin(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return GeoRelationUtils.rectWithin(minLon, minLat, maxLon, maxLat, GeoPointInBBoxQueryImpl.this.minLon, - GeoPointInBBoxQueryImpl.this.minLat, GeoPointInBBoxQueryImpl.this.maxLon, GeoPointInBBoxQueryImpl.this.maxLat); } + protected boolean cellWithin(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return GeoRelationUtils.rectWithin(minLat, maxLat, minLon, maxLon, GeoPointInBBoxQueryImpl.this.minLat, + GeoPointInBBoxQueryImpl.this.maxLat, + GeoPointInBBoxQueryImpl.this.minLon, GeoPointInBBoxQueryImpl.this.maxLon); + } @Override - protected boolean cellIntersectsShape(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return cellIntersectsMBR(minLon, minLat, maxLon, maxLat); + protected boolean cellIntersectsShape(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return cellIntersectsMBR(minLat, maxLat, minLon, maxLon); } @Override - protected boolean postFilter(final double lon, final double lat) { - return GeoRelationUtils.pointInRectPrecise(lon, lat, minLon, minLat, maxLon, maxLat); + protected boolean postFilter(final double lat, final double lon) { + return GeoRelationUtils.pointInRectPrecise(lat, lon, minLat, maxLat, minLon, maxLon); } } @@ -108,10 +111,10 @@ public boolean equals(Object o) { GeoPointInBBoxQueryImpl that = (GeoPointInBBoxQueryImpl) o; - if (Double.compare(that.maxLat, maxLat) != 0) return false; - if (Double.compare(that.maxLon, maxLon) != 0) return false; if (Double.compare(that.minLat, minLat) != 0) return false; + if (Double.compare(that.maxLat, maxLat) != 0) return false; if (Double.compare(that.minLon, minLon) != 0) return false; + if (Double.compare(that.maxLon, maxLon) != 0) return false; return true; } @@ -120,14 +123,14 @@ public boolean equals(Object o) { public int hashCode() { int result = super.hashCode(); long temp; - temp = Double.doubleToLongBits(minLon); - result = 31 * result + (int) (temp ^ (temp >>> 32)); temp = Double.doubleToLongBits(minLat); result = 31 * result + (int) (temp ^ (temp >>> 32)); - temp = Double.doubleToLongBits(maxLon); - result = 31 * result + (int) (temp ^ (temp >>> 32)); temp = Double.doubleToLongBits(maxLat); result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(minLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(maxLon); + result = 31 * result + (int) (temp ^ (temp >>> 32)); return result; } @@ -142,14 +145,14 @@ public String toString(String field) { sb.append(':'); } return sb.append(" Lower Left: [") - .append(minLon) - .append(',') .append(minLat) + .append(',') + .append(minLon) .append(']') .append(" Upper Right: [") - .append(maxLon) - .append(',') .append(maxLat) + .append(',') + .append(maxLon) .append("]") .toString(); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java index ef8c2ff0244d..f43aaf2a9065 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java @@ -47,30 +47,30 @@ public final class GeoPointInPolygonQuery extends GeoPointInBBoxQuery { // polygon position arrays - this avoids the use of any objects or // or geo library dependencies - /** array of x (longitude) values (in degrees) */ - protected final double[] x; /** array of y (latitude) values (in degrees) */ - protected final double[] y; + protected final double[] polyLats; + /** array of x (longitude) values (in degrees) */ + protected final double[] polyLons; /** * Constructs a new GeoPolygonQuery that will match encoded {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} terms * that fall within or on the boundary of the polygon defined by the input parameters. */ - public GeoPointInPolygonQuery(final String field, final double[] polyLons, final double[] polyLats) { - this(field, TermEncoding.PREFIX, GeoUtils.polyToBBox(polyLons, polyLats), polyLons, polyLats); + public GeoPointInPolygonQuery(final String field, final double[] polyLats, final double[] polyLons) { + this(field, TermEncoding.PREFIX, GeoUtils.polyToBBox(polyLats, polyLons), polyLats, polyLons); } /** * Constructs a new GeoPolygonQuery that will match encoded {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} terms * that fall within or on the boundary of the polygon defined by the input parameters. */ - public GeoPointInPolygonQuery(final String field, final TermEncoding termEncoding, final double[] polyLons, final double[] polyLats) { - this(field, termEncoding, GeoUtils.polyToBBox(polyLons, polyLats), polyLons, polyLats); + public GeoPointInPolygonQuery(final String field, final TermEncoding termEncoding, final double[] polyLats, final double[] polyLons) { + this(field, termEncoding, GeoUtils.polyToBBox(polyLats, polyLons), polyLats, polyLons); } /** Common constructor, used only internally. */ - private GeoPointInPolygonQuery(final String field, TermEncoding termEncoding, GeoRect bbox, final double[] polyLons, final double[] polyLats) { - super(field, termEncoding, bbox.minLon, bbox.minLat, bbox.maxLon, bbox.maxLat); + private GeoPointInPolygonQuery(final String field, TermEncoding termEncoding, GeoRect bbox, final double[] polyLats, final double[] polyLons) { + super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); if (polyLats.length != polyLons.length) { throw new IllegalArgumentException("polyLats and polyLons must be equal length"); } @@ -84,14 +84,14 @@ private GeoPointInPolygonQuery(final String field, TermEncoding termEncoding, Ge throw new IllegalArgumentException("first and last points of the polygon must be the same (it must close itself): polyLons[0]=" + polyLons[0] + " polyLons[" + (polyLons.length-1) + "]=" + polyLons[polyLons.length-1]); } - this.x = polyLons; - this.y = polyLats; + this.polyLons = polyLons; + this.polyLats = polyLats; } /** throw exception if trying to change rewrite method */ @Override public Query rewrite(IndexReader reader) { - return new GeoPointInPolygonQueryImpl(field, termEncoding, this, this.minLon, this.minLat, this.maxLon, this.maxLat); + return new GeoPointInPolygonQueryImpl(field, termEncoding, this, this.minLat, this.maxLat, this.minLon, this.maxLon); } @Override @@ -102,8 +102,8 @@ public boolean equals(Object o) { GeoPointInPolygonQuery that = (GeoPointInPolygonQuery) o; - if (!Arrays.equals(x, that.x)) return false; - if (!Arrays.equals(y, that.y)) return false; + if (!Arrays.equals(polyLats, that.polyLats)) return false; + if (!Arrays.equals(polyLons, that.polyLons)) return false; return true; } @@ -111,15 +111,15 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = super.hashCode(); - result = 31 * result + (x != null ? Arrays.hashCode(x) : 0); - result = 31 * result + (y != null ? Arrays.hashCode(y) : 0); + result = 31 * result + (polyLats != null ? Arrays.hashCode(polyLats) : 0); + result = 31 * result + (polyLons != null ? Arrays.hashCode(polyLons) : 0); return result; } /** print out this polygon query */ @Override public String toString(String field) { - assert x.length == y.length; + assert polyLats.length == polyLons.length; final StringBuilder sb = new StringBuilder(); sb.append(getClass().getSimpleName()); @@ -130,11 +130,11 @@ public String toString(String field) { sb.append(':'); } sb.append(" Points: "); - for (int i=0; i>>1); // if cell is within and a factor of the precision step, or it crosses the edge of the shape add the range - final boolean within = res % GeoPointField.PRECISION_STEP == 0 && relationImpl.cellWithin(minLon, minLat, maxLon, maxLat); - if (within || (level == DETAIL_LEVEL && relationImpl.cellIntersectsShape(minLon, minLat, maxLon, maxLat))) { + final boolean within = res % GeoPointField.PRECISION_STEP == 0 && relationImpl.cellWithin(minLat, maxLat, minLon, maxLon); + if (within || (level == DETAIL_LEVEL && relationImpl.cellIntersectsShape(minLat, maxLat, minLon, maxLon))) { final short nextRes = (short)(res-1); if (nextRes % GeoPointField.PRECISION_STEP == 0) { rangeBounds.add(new Range(start, nextRes, !within)); @@ -96,7 +96,7 @@ private void relateAndRecurse(final long start, final long end, final short res) } else { rangeBounds.add(new Range(start, res, !within)); } - } else if (level < DETAIL_LEVEL && relationImpl.cellIntersectsMBR(minLon, minLat, maxLon, maxLat)) { + } else if (level < DETAIL_LEVEL && relationImpl.cellIntersectsMBR(minLat, maxLat, minLon, maxLon)) { computeRange(start, (short) (res - 1)); } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java index fbaf68e462a4..eeb58b2e3480 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java @@ -58,7 +58,7 @@ final class GeoPointPrefixTermsEnum extends GeoPointTermsEnum { public GeoPointPrefixTermsEnum(final TermsEnum tenum, final GeoPointMultiTermQuery query) { super(tenum, query); - this.start = mortonHash(query.minLon, query.minLat); + this.start = mortonHash(query.minLat, query.minLon); this.currentRange = new Range(0, shift, true); // start shift at maxShift value (from computeMaxShift) this.shift = maxShift; @@ -67,12 +67,12 @@ public GeoPointPrefixTermsEnum(final TermsEnum tenum, final GeoPointMultiTermQue this.currEnd = currStart | mask; } - private boolean within(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return relationImpl.cellWithin(minLon, minLat, maxLon, maxLat); + private boolean within(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return relationImpl.cellWithin(minLat, maxLat, minLon, maxLon); } - private boolean boundary(final double minLon, final double minLat, final double maxLon, final double maxLat) { - return shift == maxShift && relationImpl.cellIntersectsShape(minLon, minLat, maxLon, maxLat); + private boolean boundary(final double minLat, final double maxLat, final double minLon, final double maxLon) { + return shift == maxShift && relationImpl.cellIntersectsShape(minLat, maxLat, minLon, maxLon); } private boolean nextWithin() { @@ -100,7 +100,7 @@ private void nextRelation() { maxLat = mortonUnhashLat(currEnd); // within or a boundary - if ((isWithin = within(minLon, minLat, maxLon, maxLat) == true) || boundary(minLon, minLat, maxLon, maxLat) == true) { + if ((isWithin = within(minLat, maxLat, minLon, maxLon) == true) || boundary(minLat, maxLat, minLon, maxLon) == true) { final int m; if (isWithin == false || (m = shift % GeoPointField.PRECISION_STEP) == 0) { setNextRange(isWithin == false); @@ -116,7 +116,7 @@ private void nextRelation() { } // within cell but not at a depth factor of PRECISION_STEP - if (isWithin == true || (relationImpl.cellIntersectsMBR(minLon, minLat, maxLon , maxLat) == true && shift != maxShift)) { + if (isWithin == true || (relationImpl.cellIntersectsMBR(minLat, maxLat, minLon, maxLon) == true && shift != maxShift)) { // descend: currStart need not change since shift handles end of range currEnd = currStart | (1L<<--shift) - 1; } else { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java index 6eea8df764eb..7e97f16652d2 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermQueryConstantScoreWrapper.java @@ -140,7 +140,7 @@ public boolean matches() throws IOException { int count = sdv.count(); for (int i = 0; i < count; i++) { long hash = sdv.valueAt(i); - if (termsEnum.postFilter(mortonUnhashLon(hash), mortonUnhashLat(hash))) { + if (termsEnum.postFilter(mortonUnhashLat(hash), mortonUnhashLon(hash))) { return true; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermsEnum.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermsEnum.java index 9f9e251bf203..d82b340e4b93 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermsEnum.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointTermsEnum.java @@ -98,8 +98,8 @@ protected AcceptStatus accept(BytesRef term) { return AcceptStatus.YES; } - protected boolean postFilter(final double lon, final double lat) { - return relationImpl.postFilter(lon, lat); + protected boolean postFilter(final double lat, final double lon) { + return relationImpl.postFilter(lat, lon); } /** diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java index 482ed0f8cd78..8956c4b3a137 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java @@ -33,68 +33,6 @@ public class GeoDistanceUtils { private GeoDistanceUtils() { } - /** - * Compute the distance between two geo-points using vincenty distance formula - * Vincenty uses the oblate spheroid whereas haversine uses unit sphere, this will give roughly - * 22m better accuracy (in worst case) than haversine - * - * @param lonA longitudinal coordinate of point A (in degrees) - * @param latA latitudinal coordinate of point A (in degrees) - * @param lonB longitudinal coordinate of point B (in degrees) - * @param latB latitudinal coordinate of point B (in degrees) - * @return distance (in meters) between point A and point B - */ - public static final double vincentyDistance(final double lonA, final double latA, final double lonB, final double latB) { - final double L = StrictMath.toRadians(lonB - lonA); - final double oF = 1 - GeoProjectionUtils.FLATTENING; - final double U1 = StrictMath.atan(oF * StrictMath.tan(StrictMath.toRadians(latA))); - final double U2 = StrictMath.atan(oF * StrictMath.tan(StrictMath.toRadians(latB))); - final double sU1 = StrictMath.sin(U1); - final double cU1 = StrictMath.cos(U1); - final double sU2 = StrictMath.sin(U2); - final double cU2 = StrictMath.cos(U2); - - double sigma, sinSigma, cosSigma; - double sinAlpha, cos2Alpha, cos2SigmaM; - double lambda = L; - double lambdaP; - double iters = 100; - double sinLambda, cosLambda, c; - - do { - sinLambda = StrictMath.sin(lambda); - cosLambda = Math.cos(lambda); - sinSigma = Math.sqrt((cU2 * sinLambda) * (cU2 * sinLambda) + (cU1 * sU2 - sU1 * cU2 * cosLambda) - * (cU1 * sU2 - sU1 * cU2 * cosLambda)); - if (sinSigma == 0) { - return 0; - } - - cosSigma = sU1 * sU2 + cU1 * cU2 * cosLambda; - sigma = Math.atan2(sinSigma, cosSigma); - sinAlpha = cU1 * cU2 * sinLambda / sinSigma; - cos2Alpha = 1 - sinAlpha * sinAlpha; - cos2SigmaM = cosSigma - 2 * sU1 * sU2 / cos2Alpha; - - c = GeoProjectionUtils.FLATTENING/16 * cos2Alpha * (4 + GeoProjectionUtils.FLATTENING * (4 - 3 * cos2Alpha)); - lambdaP = lambda; - lambda = L + (1 - c) * GeoProjectionUtils.FLATTENING * sinAlpha * (sigma + c * sinSigma * (cos2SigmaM + c * cosSigma * - (-1 + 2 * cos2SigmaM * cos2SigmaM))); - } while (StrictMath.abs(lambda - lambdaP) > 1E-12 && --iters > 0); - - if (iters == 0) { - return 0; - } - - final double uSq = cos2Alpha * (GeoProjectionUtils.SEMIMAJOR_AXIS2 - GeoProjectionUtils.SEMIMINOR_AXIS2) / (GeoProjectionUtils.SEMIMINOR_AXIS2); - final double A = 1 + uSq / 16384 * (4096 + uSq * (-768 + uSq * (320 - 175 * uSq))); - final double B = uSq / 1024 * (256 + uSq * (-128 + uSq * (74 - 47 * uSq))); - final double deltaSigma = B * sinSigma * (cos2SigmaM + B/4 * (cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM) - B/6 * cos2SigmaM - * (-3 + 4 * sinSigma * sinSigma) * (-3 + 4 * cos2SigmaM * cos2SigmaM))); - - return (GeoProjectionUtils.SEMIMINOR_AXIS * A * (sigma - deltaSigma)); - } - /** * Computes distance between two points in a cartesian (x, y, {z - optional}) coordinate system */ @@ -133,11 +71,11 @@ public static double distanceToDegreesLon(double lat, double distance) { /** * Finds the closest point within a rectangle (defined by rMinX, rMinY, rMaxX, rMaxY) to the given (lon, lat) point - * the result is provided in closestPt. When the point is outside the rectangle, the closest point is on an edge + * the result is provided in closestPt (lat, lon). When the point is outside the rectangle, the closest point is on an edge * or corner of the rectangle; else, the closest point is the point itself. */ - public static void closestPointOnBBox(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double lon, final double lat, double[] closestPt) { + public static void closestPointOnBBox(final double rMinY, final double rMaxY, final double rMinX, final double rMaxX, + final double lat, final double lon, double[] closestPt) { assert closestPt != null && closestPt.length == 2; closestPt[0] = 0; @@ -147,32 +85,32 @@ public static void closestPointOnBBox(final double rMinX, final double rMinY, fi boolean ySet = true; if (lon > rMaxX) { - closestPt[0] = rMaxX; + closestPt[1] = rMaxX; } else if (lon < rMinX) { - closestPt[0] = rMinX; + closestPt[1] = rMinX; } else { xSet = false; } if (lat > rMaxY) { - closestPt[1] = rMaxY; + closestPt[0] = rMaxY; } else if (lat < rMinY) { - closestPt[1] = rMinY; + closestPt[0] = rMinY; } else { ySet = false; } - if (closestPt[0] == 0 && xSet == false) { - closestPt[0] = lon; + if (closestPt[0] == 0 && ySet == false) { + closestPt[0] = lat; } - if (closestPt[1] == 0 && ySet == false) { - closestPt[1] = lat; + if (closestPt[1] == 0 && xSet == false) { + closestPt[1] = lon; } } /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */ - public static double maxRadialDistanceMeters(final double centerLon, final double centerLat) { + public static double maxRadialDistanceMeters(final double centerLat, final double centerLon) { if (Math.abs(centerLat) == GeoUtils.MAX_LAT_INCL) { return SloppyMath.haversinMeters(centerLat, centerLon, 0, centerLon); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java index 451688d2c7e4..38eef04b69dd 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java @@ -32,8 +32,9 @@ public final class GeoEncodingUtils { /** number of bits used for quantizing latitude and longitude values */ public static final short BITS = 31; - private static final double LON_SCALE = (0x1L<byte[] buffer size) @@ -50,10 +51,10 @@ private GeoEncodingUtils() { } /** - * encode longitude, latitude geopoint values using morton encoding method + * encode latitude, longitude geopoint values using morton encoding method * https://en.wikipedia.org/wiki/Z-order_curve */ - public static final Long mortonHash(final double lon, final double lat) { + public static final Long mortonHash(final double lat, final double lon) { return BitUtil.interleave(scaleLon(lon), scaleLat(lat)); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoHashUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoHashUtils.java deleted file mode 100644 index 49a7571aaaee..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoHashUtils.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.util; - -import java.util.ArrayList; -import java.util.Collection; - -import org.apache.lucene.util.BitUtil; - -/** - * Utilities for converting to/from the GeoHash standard - * - * The geohash long format is represented as lon/lat (x/y) interleaved with the 4 least significant bits - * representing the level (1-12) [xyxy...xyxyllll] - * - * This differs from a morton encoded value which interleaves lat/lon (y/x). - * - * @lucene.experimental - */ -public class GeoHashUtils { - private static final char[] BASE_32 = {'0', '1', '2', '3', '4', '5', '6', - '7', '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - - private static final String BASE_32_STRING = new String(BASE_32); - - /** maximum precision for geohash strings */ - public static final int PRECISION = 12; - private static final short MORTON_OFFSET = (GeoEncodingUtils.BITS<<1) - (PRECISION*5); - - // No instance: - private GeoHashUtils() { - } - - /** - * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) - */ - public static final long longEncode(final double lon, final double lat, final int level) { - // shift to appropriate level - final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET); - return ((BitUtil.flipFlop(GeoEncodingUtils.mortonHash(lon, lat)) >>> msf) << 4) | level; - } - - /** - * Encode from geohash string to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) - */ - public static final long longEncode(final String hash) { - int level = hash.length()-1; - long b; - long l = 0L; - for(char c : hash.toCharArray()) { - b = (long)(BASE_32_STRING.indexOf(c)); - l |= (b<<(level--*5)); - } - return (l<<4)|hash.length(); - } - - /** - * Encode an existing geohash long to the provided precision - */ - public static long longEncode(long geohash, int level) { - final short precision = (short)(geohash & 15); - if (precision == level) { - return geohash; - } else if (precision > level) { - return ((geohash >>> (((precision - level) * 5) + 4)) << 4) | level; - } - return ((geohash >>> 4) << (((level - precision) * 5) + 4) | level); - } - - /** - * Convert from a morton encoded long from a geohash encoded long - */ - public static long fromMorton(long morton, int level) { - long mFlipped = BitUtil.flipFlop(morton); - mFlipped >>>= (((GeoHashUtils.PRECISION - level) * 5) + MORTON_OFFSET); - return (mFlipped << 4) | level; - } - - /** - * Encode to a geohash string from the geohash based long format - */ - public static final String stringEncode(long geoHashLong) { - int level = (int)geoHashLong&15; - geoHashLong >>>= 4; - char[] chars = new char[level]; - do { - chars[--level] = BASE_32[(int) (geoHashLong&31L)]; - geoHashLong>>>=5; - } while(level > 0); - - return new String(chars); - } - - /** - * Encode to a geohash string from full resolution longitude, latitude) - */ - public static final String stringEncode(final double lon, final double lat) { - return stringEncode(lon, lat, 12); - } - - /** - * Encode to a level specific geohash string from full resolution longitude, latitude - */ - public static final String stringEncode(final double lon, final double lat, final int level) { - // convert to geohashlong - final long ghLong = fromMorton(GeoEncodingUtils.mortonHash(lon, lat), level); - return stringEncode(ghLong); - - } - - /** - * Encode to a full precision geohash string from a given morton encoded long value - */ - public static final String stringEncodeFromMortonLong(final long hashedVal) throws Exception { - return stringEncode(hashedVal, PRECISION); - } - - /** - * Encode to a geohash string at a given level from a morton long - */ - public static final String stringEncodeFromMortonLong(long hashedVal, final int level) { - // bit twiddle to geohash (since geohash is a swapped (lon/lat) encoding) - hashedVal = BitUtil.flipFlop(hashedVal); - - StringBuilder geoHash = new StringBuilder(); - short precision = 0; - final short msf = (GeoEncodingUtils.BITS<<1)-5; - long mask = 31L<>>(msf-(precision*5)))]); - // next 5 bits - mask >>>= 5; - } while (++precision < level); - return geoHash.toString(); - } - - /** - * Encode to a morton long value from a given geohash string - */ - public static final long mortonEncode(final String hash) { - int level = 11; - long b; - long l = 0L; - for(char c : hash.toCharArray()) { - b = (long)(BASE_32_STRING.indexOf(c)); - l |= (b<<((level--*5) + MORTON_OFFSET)); - } - return BitUtil.flipFlop(l); - } - - /** - * Encode to a morton long value from a given geohash long value - */ - public static final long mortonEncode(final long geoHashLong) { - final int level = (int)(geoHashLong&15); - final short odd = (short)(level & 1); - - return BitUtil.flipFlop(((geoHashLong >>> 4) << odd) << (((12 - level) * 5) + (MORTON_OFFSET - odd))); - } - - private static final char encode(int x, int y) { - return BASE_32[((x & 1) + ((y & 1) * 2) + ((x & 2) * 2) + ((y & 2) * 4) + ((x & 4) * 4)) % 32]; - } - - /** - * Calculate all neighbors of a given geohash cell. - * - * @param geohash Geohash of the defined cell - * @return geohashes of all neighbor cells - */ - public static Collection neighbors(String geohash) { - return addNeighbors(geohash, geohash.length(), new ArrayList(8)); - } - - /** - * Calculate the geohash of a neighbor of a geohash - * - * @param geohash the geohash of a cell - * @param level level of the geohash - * @param dx delta of the first grid coordinate (must be -1, 0 or +1) - * @param dy delta of the second grid coordinate (must be -1, 0 or +1) - * @return geohash of the defined cell - */ - public final static String neighbor(String geohash, int level, int dx, int dy) { - int cell = BASE_32_STRING.indexOf(geohash.charAt(level -1)); - - // Decoding the Geohash bit pattern to determine grid coordinates - int x0 = cell & 1; // first bit of x - int y0 = cell & 2; // first bit of y - int x1 = cell & 4; // second bit of x - int y1 = cell & 8; // second bit of y - int x2 = cell & 16; // third bit of x - - // combine the bitpattern to grid coordinates. - // note that the semantics of x and y are swapping - // on each level - int x = x0 + (x1 / 2) + (x2 / 4); - int y = (y0 / 2) + (y1 / 4); - - if (level == 1) { - // Root cells at north (namely "bcfguvyz") or at - // south (namely "0145hjnp") do not have neighbors - // in north/south direction - if ((dy < 0 && y == 0) || (dy > 0 && y == 3)) { - return null; - } else { - return Character.toString(encode(x + dx, y + dy)); - } - } else { - // define grid coordinates for next level - final int nx = ((level % 2) == 1) ? (x + dx) : (x + dy); - final int ny = ((level % 2) == 1) ? (y + dy) : (y + dx); - - // if the defined neighbor has the same parent a the current cell - // encode the cell directly. Otherwise find the cell next to this - // cell recursively. Since encoding wraps around within a cell - // it can be encoded here. - // xLimit and YLimit must always be respectively 7 and 3 - // since x and y semantics are swapping on each level. - if (nx >= 0 && nx <= 7 && ny >= 0 && ny <= 3) { - return geohash.substring(0, level - 1) + encode(nx, ny); - } else { - String neighbor = neighbor(geohash, level - 1, dx, dy); - return (neighbor != null) ? neighbor + encode(nx, ny) : neighbor; - } - } - } - - /** - * Add all geohashes of the cells next to a given geohash to a list. - * - * @param geohash Geohash of a specified cell - * @param neighbors list to add the neighbors to - * @return the given list - */ - public static final > E addNeighbors(String geohash, E neighbors) { - return addNeighbors(geohash, geohash.length(), neighbors); - } - - /** - * Add all geohashes of the cells next to a given geohash to a list. - * - * @param geohash Geohash of a specified cell - * @param length level of the given geohash - * @param neighbors list to add the neighbors to - * @return the given list - */ - public static final > E addNeighbors(String geohash, int length, E neighbors) { - String south = neighbor(geohash, length, 0, -1); - String north = neighbor(geohash, length, 0, +1); - if (north != null) { - neighbors.add(neighbor(north, length, -1, 0)); - neighbors.add(north); - neighbors.add(neighbor(north, length, +1, 0)); - } - - neighbors.add(neighbor(geohash, length, -1, 0)); - neighbors.add(neighbor(geohash, length, +1, 0)); - - if (south != null) { - neighbors.add(neighbor(south, length, -1, 0)); - neighbors.add(south); - neighbors.add(neighbor(south, length, +1, 0)); - } - - return neighbors; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java index 12d135045ac6..61aa09fb9c84 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java @@ -69,83 +69,14 @@ private GeoProjectionUtils() { } /** - * Converts from geocentric earth-centered earth-fixed to geodesic lat/lon/alt - * @param x Cartesian x coordinate - * @param y Cartesian y coordinate - * @param z Cartesian z coordinate - * @param lla 0: longitude 1: latitude: 2: altitude - * @return double array as 0: longitude 1: latitude 2: altitude - */ - public static final double[] ecfToLLA(final double x, final double y, final double z, double[] lla) { - boolean atPole = false; - final double ad_c = 1.0026000D; - final double cos67P5 = 0.38268343236508977D; - - if (lla == null) { - lla = new double[3]; - } - - if (x != 0.0) { - lla[0] = StrictMath.atan2(y,x); - } else { - if (y > 0) { - lla[0] = PIO2; - } else if (y < 0) { - lla[0] = -PIO2; - } else { - atPole = true; - lla[0] = 0.0D; - if (z > 0.0) { - lla[1] = PIO2; - } else if (z < 0.0) { - lla[1] = -PIO2; - } else { - lla[1] = PIO2; - lla[2] = -SEMIMINOR_AXIS; - return lla; - } - } - } - - final double w2 = x*x + y*y; - final double w = StrictMath.sqrt(w2); - final double t0 = z * ad_c; - final double s0 = StrictMath.sqrt(t0 * t0 + w2); - final double sinB0 = t0 / s0; - final double cosB0 = w / s0; - final double sin3B0 = sinB0 * sinB0 * sinB0; - final double t1 = z + SEMIMINOR_AXIS * EP2 * sin3B0; - final double sum = w - SEMIMAJOR_AXIS * E2 * cosB0 * cosB0 * cosB0; - final double s1 = StrictMath.sqrt(t1 * t1 + sum * sum); - final double sinP1 = t1 / s1; - final double cosP1 = sum / s1; - final double rn = SEMIMAJOR_AXIS / StrictMath.sqrt(1.0D - E2 * sinP1 * sinP1); - - if (cosP1 >= cos67P5) { - lla[2] = w / cosP1 - rn; - } else if (cosP1 <= -cos67P5) { - lla[2] = w / -cosP1 - rn; - } else { - lla[2] = z / sinP1 + rn * (E2 - 1.0); - } - if (!atPole) { - lla[1] = StrictMath.atan(sinP1/cosP1); - } - lla[0] = TO_DEGREES * lla[0]; - lla[1] = TO_DEGREES * lla[1]; - - return lla; - } - - /** - * Converts from geodesic lon lat alt to geocentric earth-centered earth-fixed - * @param lon geodesic longitude + * Converts from geodesic lat lon alt to geocentric earth-centered earth-fixed * @param lat geodesic latitude + * @param lon geodesic longitude * @param alt geodesic altitude * @param ecf reusable earth-centered earth-fixed result * @return either a new ecef array or the reusable ecf parameter */ - public static final double[] llaToECF(double lon, double lat, double alt, double[] ecf) { + public static final double[] llaToECF(double lat, double lon, double alt, double[] ecf) { lon = TO_RADIANS * lon; lat = TO_RADIANS * lat; @@ -177,261 +108,23 @@ public static final double[] llaToECF(double lon, double lat, double alt, double } /** - * Converts from lat lon alt (in degrees) to East North Up right-hand coordinate system - * @param lon longitude in degrees - * @param lat latitude in degrees - * @param alt altitude in meters - * @param centerLon reference point longitude in degrees - * @param centerLat reference point latitude in degrees - * @param centerAlt reference point altitude in meters - * @param enu result east, north, up coordinate - * @return east, north, up coordinate - */ - public static double[] llaToENU(final double lon, final double lat, final double alt, double centerLon, - double centerLat, final double centerAlt, double[] enu) { - if (enu == null) { - enu = new double[3]; - } - - // convert point to ecf coordinates - final double[] ecf = llaToECF(lon, lat, alt, null); - - // convert from ecf to enu - return ecfToENU(ecf[0], ecf[1], ecf[2], centerLon, centerLat, centerAlt, enu); - } - - /** - * Converts from East North Up right-hand rule to lat lon alt in degrees - * @param x easting (in meters) - * @param y northing (in meters) - * @param z up (in meters) - * @param centerLon reference point longitude (in degrees) - * @param centerLat reference point latitude (in degrees) - * @param centerAlt reference point altitude (in meters) - * @param lla resulting lat, lon, alt point (in degrees) - * @return lat, lon, alt point (in degrees) - */ - public static double[] enuToLLA(final double x, final double y, final double z, final double centerLon, - final double centerLat, final double centerAlt, double[] lla) { - // convert enuToECF - if (lla == null) { - lla = new double[3]; - } - - // convert enuToECF, storing intermediate result in lla - lla = enuToECF(x, y, z, centerLon, centerLat, centerAlt, lla); - - // convert ecf to LLA - return ecfToLLA(lla[0], lla[1], lla[2], lla); - } - - /** - * Convert from Earth-Centered-Fixed to Easting, Northing, Up Right Hand System - * @param x ECF X coordinate (in meters) - * @param y ECF Y coordinate (in meters) - * @param z ECF Z coordinate (in meters) - * @param centerLon ENU origin longitude (in degrees) - * @param centerLat ENU origin latitude (in degrees) - * @param centerAlt ENU altitude (in meters) - * @param enu reusable enu result - * @return Easting, Northing, Up coordinate - */ - public static double[] ecfToENU(double x, double y, double z, final double centerLon, - final double centerLat, final double centerAlt, double[] enu) { - if (enu == null) { - enu = new double[3]; - } - - // create rotation matrix and rotate to enu orientation - final double[][] phi = createPhiTransform(centerLon, centerLat, null); - - // convert origin to ENU - final double[] originECF = llaToECF(centerLon, centerLat, centerAlt, null); - final double[] originENU = new double[3]; - originENU[0] = ((phi[0][0] * originECF[0]) + (phi[0][1] * originECF[1]) + (phi[0][2] * originECF[2])); - originENU[1] = ((phi[1][0] * originECF[0]) + (phi[1][1] * originECF[1]) + (phi[1][2] * originECF[2])); - originENU[2] = ((phi[2][0] * originECF[0]) + (phi[2][1] * originECF[1]) + (phi[2][2] * originECF[2])); - - // rotate then translate - enu[0] = ((phi[0][0] * x) + (phi[0][1] * y) + (phi[0][2] * z)) - originENU[0]; - enu[1] = ((phi[1][0] * x) + (phi[1][1] * y) + (phi[1][2] * z)) - originENU[1]; - enu[2] = ((phi[2][0] * x) + (phi[2][1] * y) + (phi[2][2] * z)) - originENU[2]; - - return enu; - } - - /** - * Convert from Easting, Northing, Up Right-Handed system to Earth Centered Fixed system - * @param x ENU x coordinate (in meters) - * @param y ENU y coordinate (in meters) - * @param z ENU z coordinate (in meters) - * @param centerLon ENU origin longitude (in degrees) - * @param centerLat ENU origin latitude (in degrees) - * @param centerAlt ENU origin altitude (in meters) - * @param ecf reusable ecf result - * @return ecf result coordinate - */ - public static double[] enuToECF(final double x, final double y, final double z, double centerLon, - double centerLat, final double centerAlt, double[] ecf) { - if (ecf == null) { - ecf = new double[3]; - } - - double[][] phi = createTransposedPhiTransform(centerLon, centerLat, null); - double[] ecfOrigin = llaToECF(centerLon, centerLat, centerAlt, null); - - // rotate and translate - ecf[0] = (phi[0][0]*x + phi[0][1]*y + phi[0][2]*z) + ecfOrigin[0]; - ecf[1] = (phi[1][0]*x + phi[1][1]*y + phi[1][2]*z) + ecfOrigin[1]; - ecf[2] = (phi[2][0]*x + phi[2][1]*y + phi[2][2]*z) + ecfOrigin[2]; - - return ecf; - } - - /** - * Create the rotation matrix for converting Earth Centered Fixed to Easting Northing Up - * @param originLon ENU origin longitude (in degrees) - * @param originLat ENU origin latitude (in degrees) - * @param phiMatrix reusable phi matrix result - * @return phi rotation matrix - */ - private static double[][] createPhiTransform(double originLon, double originLat, double[][] phiMatrix) { - - if (phiMatrix == null) { - phiMatrix = new double[3][3]; - } - - originLon = TO_RADIANS * originLon; - originLat = TO_RADIANS * originLat; - - final double sLon = sloppySin(originLon); - final double cLon = cos(originLon); - final double sLat = sloppySin(originLat); - final double cLat = cos(originLat); - - phiMatrix[0][0] = -sLon; - phiMatrix[0][1] = cLon; - phiMatrix[0][2] = 0.0D; - phiMatrix[1][0] = -sLat * cLon; - phiMatrix[1][1] = -sLat * sLon; - phiMatrix[1][2] = cLat; - phiMatrix[2][0] = cLat * cLon; - phiMatrix[2][1] = cLat * sLon; - phiMatrix[2][2] = sLat; - - return phiMatrix; - } - - /** - * Create the transposed rotation matrix for converting Easting Northing Up coordinates to Earth Centered Fixed - * @param originLon ENU origin longitude (in degrees) - * @param originLat ENU origin latitude (in degrees) - * @param phiMatrix reusable phi rotation matrix result - * @return transposed phi rotation matrix - */ - private static double[][] createTransposedPhiTransform(double originLon, double originLat, double[][] phiMatrix) { - - if (phiMatrix == null) { - phiMatrix = new double[3][3]; - } - - originLon = TO_RADIANS * originLon; - originLat = TO_RADIANS * originLat; - - final double sLat = sloppySin(originLat); - final double cLat = cos(originLat); - final double sLon = sloppySin(originLon); - final double cLon = cos(originLon); - - phiMatrix[0][0] = -sLon; - phiMatrix[1][0] = cLon; - phiMatrix[2][0] = 0.0D; - phiMatrix[0][1] = -sLat * cLon; - phiMatrix[1][1] = -sLat * sLon; - phiMatrix[2][1] = cLat; - phiMatrix[0][2] = cLat * cLon; - phiMatrix[1][2] = cLat * sLon; - phiMatrix[2][2] = sLat; - - return phiMatrix; - } - - /** - * Finds a point along a bearing from a given lon,lat geolocation using vincenty's distance formula + * Finds a point along a bearing from a given lat,lon geolocation using great circle arc * - * @param lon origin longitude in degrees * @param lat origin latitude in degrees - * @param bearing azimuthal bearing in degrees - * @param dist distance in meters - * @param pt resulting point - * @return the point along a bearing at a given distance in meters - */ - public static final double[] pointFromLonLatBearingVincenty(double lon, double lat, double bearing, double dist, double[] pt) { - - if (pt == null) { - pt = new double[2]; - } - - final double alpha1 = TO_RADIANS * bearing; - final double cosA1 = cos(alpha1); - final double sinA1 = sloppySin(alpha1); - final double tanU1 = (1-FLATTENING) * sloppyTan(TO_RADIANS * lat); - final double cosU1 = 1 / StrictMath.sqrt((1+tanU1*tanU1)); - final double sinU1 = tanU1*cosU1; - final double sig1 = StrictMath.atan2(tanU1, cosA1); - final double sinAlpha = cosU1 * sinA1; - final double cosSqAlpha = 1 - sinAlpha*sinAlpha; - final double uSq = cosSqAlpha * EP2; - final double A = 1 + uSq/16384D*(4096D + uSq * (-768D + uSq * (320D - 175D*uSq))); - final double B = uSq/1024D * (256D + uSq * (-128D + uSq * (74D - 47D * uSq))); - - double sigma = dist / (SEMIMINOR_AXIS*A); - double sigmaP; - double sinSigma, cosSigma, cos2SigmaM, deltaSigma; - - do { - cos2SigmaM = cos(2*sig1 + sigma); - sinSigma = sloppySin(sigma); - cosSigma = cos(sigma); - - deltaSigma = B * sinSigma * (cos2SigmaM + (B/4D) * (cosSigma*(-1+2*cos2SigmaM*cos2SigmaM)- - (B/6) * cos2SigmaM*(-3+4*sinSigma*sinSigma)*(-3+4*cos2SigmaM*cos2SigmaM))); - sigmaP = sigma; - sigma = dist / (SEMIMINOR_AXIS*A) + deltaSigma; - } while (StrictMath.abs(sigma-sigmaP) > 1E-12); - - final double tmp = sinU1*sinSigma - cosU1*cosSigma*cosA1; - final double lat2 = StrictMath.atan2(sinU1*cosSigma + cosU1*sinSigma*cosA1, - (1-FLATTENING) * StrictMath.sqrt(sinAlpha*sinAlpha + tmp*tmp)); - final double lambda = StrictMath.atan2(sinSigma*sinA1, cosU1*cosSigma - sinU1*sinSigma*cosA1); - final double c = FLATTENING/16 * cosSqAlpha * (4 + FLATTENING * (4 - 3 * cosSqAlpha)); - - final double lam = lambda - (1-c) * FLATTENING * sinAlpha * - (sigma + c * sinSigma * (cos2SigmaM + c * cosSigma * (-1 + 2* cos2SigmaM*cos2SigmaM))); - pt[0] = normalizeLon(lon + TO_DEGREES * lam); - pt[1] = normalizeLat(TO_DEGREES * lat2); - - return pt; - } - - /** - * Finds a point along a bearing from a given lon,lat geolocation using great circle arc - * * @param lon origin longitude in degrees - * @param lat origin latitude in degrees * @param bearing azimuthal bearing in degrees * @param dist distance in meters * @param pt resulting point * @return the point along a bearing at a given distance in meters */ - public static final double[] pointFromLonLatBearingGreatCircle(double lon, double lat, double bearing, double dist, double[] pt) { + public static final double[] pointFromLonLatBearingGreatCircle(double lat, double lon, double bearing, double dist, double[] pt) { if (pt == null) { pt = new double[2]; } - lon *= TO_RADIANS; lat *= TO_RADIANS; + lon *= TO_RADIANS; bearing *= TO_RADIANS; final double cLat = cos(lat); @@ -447,14 +140,14 @@ public static final double[] pointFromLonLatBearingGreatCircle(double lon, doubl } /** - * Finds the bearing (in degrees) between 2 geo points (lon, lat) using great circle arc - * @param lon1 first point longitude in degrees + * Finds the bearing (in degrees) between 2 geo points (lat, lon) using great circle arc * @param lat1 first point latitude in degrees - * @param lon2 second point longitude in degrees + * @param lon1 first point longitude in degrees * @param lat2 second point latitude in degrees + * @param lon2 second point longitude in degrees * @return the bearing (in degrees) between the two provided points */ - public static double bearingGreatCircle(double lon1, double lat1, double lon2, double lat2) { + public static double bearingGreatCircle(double lat1, double lon1, double lat2, double lon2) { double dLon = (lon2 - lon1) * TO_RADIANS; lat2 *= TO_RADIANS; lat1 *= TO_RADIANS; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java index e147351b2ca4..8d947b4d8670 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java @@ -18,19 +18,19 @@ /** Represents a lat/lon rectangle. */ public class GeoRect { - /** minimum longitude value (in degrees) */ - public final double minLon; - /** minimum latitude value (in degrees) */ - public final double maxLon; /** maximum longitude value (in degrees) */ public final double minLat; + /** minimum longitude value (in degrees) */ + public final double minLon; /** maximum latitude value (in degrees) */ public final double maxLat; + /** minimum latitude value (in degrees) */ + public final double maxLon; /** * Constructs a bounding box by first validating the provided latitude and longitude coordinates */ - public GeoRect(double minLon, double maxLon, double minLat, double maxLat) { + public GeoRect(double minLat, double maxLat, double minLon, double maxLon) { if (GeoUtils.isValidLon(minLon) == false) { throw new IllegalArgumentException("invalid minLon " + minLon); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java index e40b91a4b535..4a31202931d4 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java @@ -28,13 +28,14 @@ private GeoRelationUtils() { } /** - * Determine if a bbox (defined by minLon, minLat, maxLon, maxLat) contains the provided point (defined by lon, lat) + * Determine if a bbox (defined by minLat, maxLat, minLon, maxLon) contains the provided point (defined by lat, lon) * NOTE: this is a basic method that does not handle dateline or pole crossing. Unwrapping must be done before * calling this method. */ - public static boolean pointInRectPrecise(final double lon, final double lat, final double minLon, - final double minLat, final double maxLon, final double maxLat) { - return lon >= minLon && lon <= maxLon && lat >= minLat && lat <= maxLat; + public static boolean pointInRectPrecise(final double lat, final double lon, + final double minLat, final double maxLat, + final double minLon, final double maxLon) { + return lat >= minLat && lat <= maxLat && lon >= minLon && lon <= maxLon; } /** @@ -46,17 +47,17 @@ public static boolean pointInRectPrecise(final double lon, final double lat, fin * * NOTE: Requires polygon point (x,y) order either clockwise or counter-clockwise */ - public static boolean pointInPolygon(double[] x, double[] y, double lat, double lon) { - assert x.length == y.length; + public static boolean pointInPolygon(double[] polyLats, double[] polyLons, double lat, double lon) { + assert polyLats.length == polyLons.length; boolean inPoly = false; /** * Note: This is using a euclidean coordinate system which could result in * upwards of 110KM error at the equator. * TODO convert coordinates to cylindrical projection (e.g. mercator) */ - for (int i = 1; i < x.length; i++) { - if (x[i] <= lon && x[i-1] >= lon || x[i-1] <= lon && x[i] >= lon) { - if (y[i] + (lon - x[i]) / (x[i-1] - x[i]) * (y[i-1] - y[i]) <= lat) { + for (int i = 1; i < polyLats.length; i++) { + if (polyLons[i] <= lon && polyLons[i-1] >= lon || polyLons[i-1] <= lon && polyLons[i] >= lon) { + if (polyLats[i] + (lon - polyLons[i]) / (polyLons[i-1] - polyLons[i]) * (polyLats[i-1] - polyLats[i]) <= lat) { inPoly = !inPoly; } } @@ -71,42 +72,34 @@ public static boolean pointInPolygon(double[] x, double[] y, double lat, double /** * Computes whether two rectangles are disjoint */ - public static boolean rectDisjoint(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return (aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY); + private static boolean rectDisjoint(final double aMinLat, final double aMaxLat, final double aMinLon, final double aMaxLon, + final double bMinLat, final double bMaxLat, final double bMinLon, final double bMaxLon) { + return (aMaxLon < bMinLon || aMinLon > bMaxLon || aMaxLat < bMinLat || aMinLat > bMaxLat); } /** * Computes whether the first (a) rectangle is wholly within another (b) rectangle (shared boundaries allowed) */ - public static boolean rectWithin(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY); + public static boolean rectWithin(final double aMinLat, final double aMaxLat, final double aMinLon, final double aMaxLon, + final double bMinLat, final double bMaxLat, final double bMinLon, final double bMaxLon) { + return !(aMinLon < bMinLon || aMinLat < bMinLat || aMaxLon > bMaxLon || aMaxLat > bMaxLat); } /** * Computes whether two rectangles cross */ - public static boolean rectCrosses(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(rectDisjoint(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY) || - rectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY)); - } - - /** - * Computes whether rectangle a contains rectangle b (touching allowed) - */ - public static boolean rectContains(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !(bMinX < aMinX || bMinY < aMinY || bMaxX > aMaxX || bMaxY > aMaxY); + public static boolean rectCrosses(final double aMinLat, final double aMaxLat, final double aMinLon, final double aMaxLon, + final double bMinLat, final double bMaxLat, final double bMinLon, final double bMaxLon) { + return !(rectDisjoint(aMinLat, aMaxLat, aMinLon, aMaxLon, bMinLat, bMaxLat, bMinLon, bMaxLon) || + rectWithin(aMinLat, aMaxLat, aMinLon, aMaxLon, bMinLat, bMaxLat, bMinLon, bMaxLon)); } /** * Computes whether a rectangle intersects another rectangle (crosses, within, touching, etc) */ - public static boolean rectIntersects(final double aMinX, final double aMinY, final double aMaxX, final double aMaxY, - final double bMinX, final double bMinY, final double bMaxX, final double bMaxY) { - return !((aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY) ); + public static boolean rectIntersects(final double aMinLat, final double aMaxLat, final double aMinLon, final double aMaxLon, + final double bMinLat, final double bMaxLat, final double bMinLon, final double bMaxLon) { + return !((aMaxLon < bMinLon || aMinLon > bMaxLon || aMaxLat < bMinLat || aMinLat > bMaxLat)); } ///////////////////////// @@ -116,33 +109,35 @@ public static boolean rectIntersects(final double aMinX, final double aMinY, fin /** * Convenience method for accurately computing whether a rectangle crosses a poly */ - public static boolean rectCrossesPolyPrecise(final double rMinX, final double rMinY, final double rMaxX, - final double rMaxY, final double[] shapeX, final double[] shapeY, - final double sMinX, final double sMinY, final double sMaxX, - final double sMaxY) { + public static boolean rectCrossesPolyPrecise(final double rMinLat, final double rMaxLat, + final double rMinLon, final double rMaxLon, + final double[] shapeLat, final double[] shapeLon, + final double sMinLat, final double sMaxLat, + final double sMinLon, final double sMaxLon) { // short-circuit: if the bounding boxes are disjoint then the shape does not cross - if (rectDisjoint(rMinX, rMinY, rMaxX, rMaxY, sMinX, sMinY, sMaxX, sMaxY)) { + if (rectDisjoint(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon)) { return false; } - return rectCrossesPoly(rMinX, rMinY, rMaxX, rMaxY, shapeX, shapeY); + return rectCrossesPoly(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLat, shapeLon); } /** * Compute whether a rectangle crosses a shape. (touching not allowed) Includes a flag for approximating the * relation. */ - public static boolean rectCrossesPolyApprox(final double rMinX, final double rMinY, final double rMaxX, - final double rMaxY, final double[] shapeX, final double[] shapeY, - final double sMinX, final double sMinY, final double sMaxX, - final double sMaxY) { + public static boolean rectCrossesPolyApprox(final double rMinLat, final double rMaxLat, + final double rMinLon, final double rMaxLon, + final double[] shapeLat, final double[] shapeLon, + final double sMinLat, final double sMaxLat, + final double sMinLon, final double sMaxLon) { // short-circuit: if the bounding boxes are disjoint then the shape does not cross - if (rectDisjoint(rMinX, rMinY, rMaxX, rMaxY, sMinX, sMinY, sMaxX, sMaxY)) { + if (rectDisjoint(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon)) { return false; } - final int polyLength = shapeX.length-1; + final int polyLength = shapeLon.length-1; for (short p=0; p= maxAy || maxBy <= minAy); } // horizontally collinear - final double minAx = Math.min(aX1, aX2); - final double maxAx = Math.max(aX1, aX2); - final double minBx = Math.min(bX1, bX2); - final double maxBx = Math.max(bX1, bX2); + final double minAx = Math.min(aLon1, aLon2); + final double maxAx = Math.max(aLon1, aLon2); + final double minBx = Math.min(bLon1, bLon2); + final double maxBx = Math.max(bLon1, bLon2); return !(minBx >= maxAx || maxBx <= minAx); } @@ -249,107 +250,113 @@ private static boolean lineCrossesLine(final double aX1, final double aY1, final * Computes whether a rectangle is within a polygon (shared boundaries not allowed) with more rigor than the * {@link GeoRelationUtils#rectWithinPolyApprox} counterpart */ - public static boolean rectWithinPolyPrecise(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double[] shapeX, final double[] shapeY, final double sMinX, - final double sMinY, final double sMaxX, final double sMaxY) { + public static boolean rectWithinPolyPrecise(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double[] shapeLats, final double[] shapeLons, final double sMinLat, + final double sMaxLat, final double sMinLon, final double sMaxLon) { // check if rectangle crosses poly (to handle concave/pacman polys), then check that all 4 corners // are contained - return !(rectCrossesPolyPrecise(rMinX, rMinY, rMaxX, rMaxY, shapeX, shapeY, sMinX, sMinY, sMaxX, sMaxY) || - !pointInPolygon(shapeX, shapeY, rMinY, rMinX) || !pointInPolygon(shapeX, shapeY, rMinY, rMaxX) || - !pointInPolygon(shapeX, shapeY, rMaxY, rMaxX) || !pointInPolygon(shapeX, shapeY, rMaxY, rMinX)); + return !(rectCrossesPolyPrecise(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLats, shapeLons, sMinLat, sMaxLat, sMinLon, sMaxLon) || + !pointInPolygon(shapeLats, shapeLons, rMinLat, rMinLon) || !pointInPolygon(shapeLats, shapeLons, rMinLat, rMaxLon) || + !pointInPolygon(shapeLats, shapeLons, rMaxLat, rMaxLon) || !pointInPolygon(shapeLats, shapeLons, rMaxLat, rMinLon)); } /** * Computes whether a rectangle is within a given polygon (shared boundaries allowed) */ - public static boolean rectWithinPolyApprox(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double[] shapeX, final double[] shapeY, final double sMinX, - final double sMinY, final double sMaxX, final double sMaxY) { + public static boolean rectWithinPolyApprox(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double[] shapeLats, final double[] shapeLons, final double sMinLat, + final double sMaxLat, final double sMinLon, final double sMaxLon) { // approximation: check if rectangle crosses poly (to handle concave/pacman polys), then check one of the corners // are contained // short-cut: if bounding boxes cross, rect is not within - if (rectCrosses(rMinX, rMinY, rMaxX, rMaxY, sMinX, sMinY, sMaxX, sMaxY) == true) { + if (rectCrosses(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon) == true) { return false; } - return !(rectCrossesPolyApprox(rMinX, rMinY, rMaxX, rMaxY, shapeX, shapeY, sMinX, sMinY, sMaxX, sMaxY) - || !pointInPolygon(shapeX, shapeY, rMinY, rMinX)); + return !(rectCrossesPolyApprox(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLats, shapeLons, sMinLat, sMaxLat, sMinLon, sMaxLon) + || !pointInPolygon(shapeLats, shapeLons, rMinLat, rMinLon)); } ///////////////////////// // Circle relations ///////////////////////// - private static boolean rectAnyCornersInCircle(final double rMinX, final double rMinY, final double rMaxX, - final double rMaxY, final double centerLon, final double centerLat, + private static boolean rectAnyCornersInCircle(final double rMinLat, final double rMaxLat, final double rMinLon, + final double rMaxLon, final double centerLat, final double centerLon, final double radiusMeters, final boolean approx) { if (approx == true) { - return rectAnyCornersInCircleSloppy(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters); + return rectAnyCornersInCircleSloppy(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters); } - double w = Math.abs(rMaxX - rMinX); + double w = Math.abs(rMaxLon - rMinLon); if (w <= 90.0) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; } // partition w /= 4; - final double p1 = rMinX + w; + final double p1 = rMinLon + w; final double p2 = p1 + w; final double p3 = p2 + w; - return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p1) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p1) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p2) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p2) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, p3) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, p3) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; + return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p1) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p1) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p2) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p2) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p3) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p3) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; } - private static boolean rectAnyCornersInCircleSloppy(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) <= radiusMeters; + private static boolean rectAnyCornersInCircleSloppy(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double centerLat, final double centerLon, final double radiusMeters) { + return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; } /** * Compute whether any of the 4 corners of the rectangle (defined by min/max X/Y) are outside the circle (defined * by centerLon, centerLat, radiusMeters) * - * Note: exotic rectangles at the poles (e.g., those whose lon/lat distance ratios greatly deviate from 1) can not + * Note: exotic rectangles at the poles (e.g., those whose lat/lon distance ratios greatly deviate from 1) can not * be determined by using distance alone. For this reason the approx flag may be set to false, in which case the * space will be further divided to more accurately compute whether the rectangle crosses the circle */ - private static boolean rectAnyCornersOutsideCircle(final double rMinX, final double rMinY, final double rMaxX, - final double rMaxY, final double centerLon, final double centerLat, + private static boolean rectAnyCornersOutsideCircle(final double rMinLat, final double rMaxLat, final double rMinLon, + final double rMaxLon, final double centerLat, final double centerLon, final double radiusMeters, final boolean approx) { if (approx == true) { - return rectAnyCornersOutsideCircleSloppy(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters); + return rectAnyCornersOutsideCircleSloppy(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters); } // if span is less than 70 degrees we can approximate using distance alone - if (Math.abs(rMaxX - rMinX) <= 70.0) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) > radiusMeters; + if (Math.abs(rMaxLon - rMinLon) <= 70.0) { + return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) > radiusMeters; } - return rectCrossesOblateCircle(centerLon, centerLat, radiusMeters, rMinX, rMinY, rMaxX, rMaxY); + return rectCrossesOblateCircle(centerLat, centerLon, + radiusMeters, + rMinLat, rMaxLat, + rMinLon, rMaxLon); } /** - * Compute whether the rectangle (defined by min/max Lon/Lat) crosses a potentially oblate circle + * Compute whether the rectangle (defined by min/max Lat/Lon) crosses a potentially oblate circle * * TODO benchmark for replacing existing rectCrossesCircle. */ - public static boolean rectCrossesOblateCircle(double centerLon, double centerLat, double radiusMeters, double rMinLon, double rMinLat, double rMaxLon, double rMaxLat) { + private static boolean rectCrossesOblateCircle(double centerLat, double centerLon, + double radiusMeters, + double rMinLat, double rMaxLat, + double rMinLon, double rMaxLon) { double w = Math.abs(rMaxLon - rMinLon); final int segs = (int)Math.ceil(w / 45.0); w /= segs; @@ -373,14 +380,14 @@ public static boolean rectCrossesOblateCircle(double centerLon, double centerLat // else we treat as an oblate circle by slicing the longitude space and checking the azimuthal range // OPTIMIZATION: this is only executed for latitude values "closeTo" the poles (e.g., 88.0 > lat < -88.0) if ( (rMaxLat > 88.0 || rMinLat < -88.0) - && (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(p1, rMinLat, - GeoProjectionUtils.bearingGreatCircle(p1, rMinLat, p1, rMaxLat), radiusMeters - d1, pt))[1] < rMinLat || pt[1] < rMaxLat - || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(maxLon, rMinLat, - GeoProjectionUtils.bearingGreatCircle(maxLon, rMinLat, maxLon, rMaxLat), radiusMeters - d2, pt))[1] < rMinLat || pt[1] < rMaxLat - || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(maxLon, rMinLat, - GeoProjectionUtils.bearingGreatCircle(maxLon, rMinLat, (midLon = p1 + 0.5*(maxLon - p1)), rMaxLat), - radiusMeters - SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, midLon), pt))[1] < rMinLat - || pt[1] < rMaxLat == false ) { + && (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, p1, + GeoProjectionUtils.bearingGreatCircle(rMinLat, p1, rMaxLat, p1), radiusMeters - d1, pt))[1] < rMinLat || pt[1] < rMaxLat + || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, maxLon, + GeoProjectionUtils.bearingGreatCircle(rMinLat, maxLon, rMaxLat, maxLon), radiusMeters - d2, pt))[1] < rMinLat || pt[1] < rMaxLat + || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, maxLon, + GeoProjectionUtils.bearingGreatCircle(rMinLat, maxLon, rMaxLat, (midLon = p1 + 0.5*(maxLon - p1))), + radiusMeters - SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, midLon), pt))[1] < rMinLat + || pt[1] < rMaxLat == false ) { return true; } p1 += w; @@ -388,81 +395,77 @@ public static boolean rectCrossesOblateCircle(double centerLon, double centerLat return false; } - private static boolean rectAnyCornersOutsideCircleSloppy(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMinX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMinX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxY, rMaxX) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinY, rMaxX) > radiusMeters; - } - - /** - * Convenience method for computing whether a rectangle is within a circle using additional precision checks - */ - public static boolean rectWithinCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters) { - return rectWithinCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, false); + private static boolean rectAnyCornersOutsideCircleSloppy(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double centerLat, final double centerLon, final double radiusMeters) { + return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) > radiusMeters + || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) > radiusMeters; } /** * Computes whether a rectangle is within a circle. Note: approx == true will be faster but less precise and may * fail on large rectangles */ - public static boolean rectWithinCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters, + public static boolean rectWithinCircle(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double centerLat, final double centerLon, final double radiusMeters, final boolean approx) { - return rectAnyCornersOutsideCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx) == false; - } - - /** - * Determine if a bbox (defined by minLon, minLat, maxLon, maxLat) contains the provided point (defined by lon, lat) - * NOTE: this is basic method that does not handle dateline or pole crossing. Unwrapping must be done before - * calling this method. - */ - public static boolean rectCrossesCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters) { - return rectCrossesCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, false); + return rectAnyCornersOutsideCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx) == false; } /** * Computes whether a rectangle crosses a circle. Note: approx == true will be faster but less precise and may * fail on large rectangles + * + *

    NOTE: this is basic method that does not handle dateline or pole crossing. Unwrapping must be done before + * calling this method. */ - public static boolean rectCrossesCircle(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters, + public static boolean rectCrossesCircle(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, + final double centerLat, final double centerLon, final double radiusMeters, final boolean approx) { if (approx == true) { - return rectAnyCornersInCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx) - || isClosestPointOnRectWithinRange(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx); + if (rectAnyCornersInCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { + return true; + } + } else { + if (rectAnyCornersInCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx) && + rectAnyCornersOutsideCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { + return true; + } } - - return (rectAnyCornersInCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx) && - rectAnyCornersOutsideCircle(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx)) - || isClosestPointOnRectWithinRange(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, radiusMeters, approx); + if (isClosestPointOnRectWithinRange(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { + return true; + } + return false; } - private static boolean isClosestPointOnRectWithinRange(final double rMinX, final double rMinY, final double rMaxX, final double rMaxY, - final double centerLon, final double centerLat, final double radiusMeters, + private static boolean isClosestPointOnRectWithinRange(final double rMinLat, final double rMaxLat, + final double rMinLon, final double rMaxLon, + final double centerLat, final double centerLon, + final double radiusMeters, final boolean approx) { double[] closestPt = {0, 0}; - GeoDistanceUtils.closestPointOnBBox(rMinX, rMinY, rMaxX, rMaxY, centerLon, centerLat, closestPt); - boolean haverShortCut = SloppyMath.haversinMeters(centerLat, centerLon, closestPt[1], closestPt[0]) <= radiusMeters; + GeoDistanceUtils.closestPointOnBBox(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, closestPt); + boolean haverShortCut = SloppyMath.haversinMeters(centerLat, centerLon, closestPt[0], closestPt[1]) <= radiusMeters; if (approx == true || haverShortCut == true) { return haverShortCut; } - double lon1 = rMinX; - double lon2 = rMaxX; - double lat1 = rMinY; - double lat2 = rMaxY; - if (closestPt[0] == rMinX || closestPt[0] == rMaxX) { - lon1 = closestPt[0]; + double lon1 = rMinLon; + double lon2 = rMaxLon; + double lat1 = rMinLat; + double lat2 = rMaxLat; + if (closestPt[1] == rMinLon || closestPt[1] == rMaxLon) { + lon1 = closestPt[1]; lon2 = lon1; - } else if (closestPt[1] == rMinY || closestPt[1] == rMaxY) { - lat1 = closestPt[1]; + } else if (closestPt[0] == rMinLat || closestPt[0] == rMaxLat) { + lat1 = closestPt[0]; lat2 = lat1; } - return lineCrossesSphere(lon1, lat1, 0, lon2, lat2, 0, centerLon, centerLat, 0, radiusMeters); + return lineCrossesSphere(lat1, lon1, 0, + lat2, lon2, 0, + centerLat, centerLon, 0, + radiusMeters); } /** @@ -480,17 +483,18 @@ private static boolean isClosestPointOnRectWithinRange(final double rMinX, final * @param radiusMeters search sphere radius (in meters) * @return whether the provided line segment is a secant of the */ - private static boolean lineCrossesSphere(double lon1, double lat1, double alt1, double lon2, - double lat2, double alt2, double centerLon, double centerLat, - double centerAlt, double radiusMeters) { + private static boolean lineCrossesSphere(double lat1, double lon1, double alt1, + double lat2, double lon2, double alt2, + double centerLat, double centerLon, double centerAlt, + double radiusMeters) { // convert to cartesian 3d (in meters) - double[] ecf1 = GeoProjectionUtils.llaToECF(lon1, lat1, alt1, null); - double[] ecf2 = GeoProjectionUtils.llaToECF(lon2, lat2, alt2, null); - double[] cntr = GeoProjectionUtils.llaToECF(centerLon, centerLat, centerAlt, null); + double[] ecf1 = GeoProjectionUtils.llaToECF(lat1, lon1, alt1, null); + double[] ecf2 = GeoProjectionUtils.llaToECF(lat2, lon2, alt2, null); + double[] cntr = GeoProjectionUtils.llaToECF(centerLat, centerLon, centerAlt, null); // convert radius from arc radius to cartesian radius - double[] oneEighty = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(centerLon, centerLat, 180.0d, radiusMeters, new double[3]); - GeoProjectionUtils.llaToECF(oneEighty[0], oneEighty[1], 0, oneEighty); + double[] oneEighty = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(centerLat, centerLon, 180.0d, radiusMeters, new double[3]); + GeoProjectionUtils.llaToECF(oneEighty[1], oneEighty[0], 0, oneEighty); radiusMeters = GeoDistanceUtils.linearDistance(oneEighty, cntr);// Math.sqrt(oneEighty[0]*cntr[0] + oneEighty[1]*cntr[1] + oneEighty[2]*cntr[2]); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index 6d755ce52612..935da603c685 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -34,7 +34,6 @@ import static org.apache.lucene.spatial.util.GeoProjectionUtils.MAX_LON_RADIANS; import static org.apache.lucene.spatial.util.GeoProjectionUtils.MIN_LAT_RADIANS; import static org.apache.lucene.spatial.util.GeoProjectionUtils.MIN_LON_RADIANS; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.pointFromLonLatBearingGreatCircle; import static org.apache.lucene.spatial.util.GeoProjectionUtils.SEMIMAJOR_AXIS; /** @@ -93,42 +92,8 @@ public static double normalizeLat(double lat_deg) { return (off <= 180 ? off : 360-off) - 90; } - /** - * Converts a given circle (defined as a point/radius) to an approximated line-segment polygon - * - * @param lon longitudinal center of circle (in degrees) - * @param lat latitudinal center of circle (in degrees) - * @param radiusMeters distance radius of circle (in meters) - * @return a list of lon/lat points representing the circle - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public static ArrayList circleToPoly(final double lon, final double lat, final double radiusMeters) { - double angle; - // a little under-sampling (to limit the number of polygonal points): using archimedes estimation of pi - final int sides = 25; - ArrayList geometry = new ArrayList(); - double[] lons = new double[sides]; - double[] lats = new double[sides]; - - double[] pt = new double[2]; - final int sidesLen = sides - 1; - for (int i = 0; i < sidesLen; ++i) { - angle = (i * 360 / sides); - pt = pointFromLonLatBearingGreatCircle(lon, lat, angle, radiusMeters, pt); - lons[i] = pt[0]; - lats[i] = pt[1]; - } - // close the poly - lons[sidesLen] = lons[0]; - lats[sidesLen] = lats[0]; - geometry.add(lons); - geometry.add(lats); - - return geometry; - } - /** Compute Bounding Box for a circle using WGS-84 parameters */ - public static GeoRect circleToBBox(final double centerLon, final double centerLat, final double radiusMeters) { + public static GeoRect circleToBBox(final double centerLat, final double centerLon, final double radiusMeters) { final double radLat = TO_RADIANS * centerLat; final double radLon = TO_RADIANS * centerLon; double radDistance = radiusMeters / SEMIMAJOR_AXIS; @@ -155,11 +120,11 @@ public static GeoRect circleToBBox(final double centerLon, final double centerLa maxLon = MAX_LON_RADIANS; } - return new GeoRect(TO_DEGREES * minLon, TO_DEGREES * maxLon, TO_DEGREES * minLat, TO_DEGREES * maxLat); + return new GeoRect(TO_DEGREES * minLat, TO_DEGREES * maxLat, TO_DEGREES * minLon, TO_DEGREES * maxLon); } /** Compute Bounding Box for a polygon using WGS-84 parameters */ - public static GeoRect polyToBBox(double[] polyLons, double[] polyLats) { + public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { if (polyLons.length != polyLats.length) { throw new IllegalArgumentException("polyLons and polyLats must be equal length"); } @@ -182,14 +147,14 @@ public static GeoRect polyToBBox(double[] polyLons, double[] polyLats) { maxLat = max(polyLats[i], maxLat); } // expand bounding box by TOLERANCE factor to handle round-off error - return new GeoRect(max(minLon - TOLERANCE, MIN_LON_INCL), min(maxLon + TOLERANCE, MAX_LON_INCL), - max(minLat - TOLERANCE, MIN_LAT_INCL), min(maxLat + TOLERANCE, MAX_LAT_INCL)); + return new GeoRect(max(minLat - TOLERANCE, MIN_LAT_INCL), min(maxLat + TOLERANCE, MAX_LAT_INCL), + max(minLon - TOLERANCE, MIN_LON_INCL), min(maxLon + TOLERANCE, MAX_LON_INCL)); } - // some sloppyish stuff, do we really need this to be done in a sloppy way? // unless it is performance sensitive, we should try to remove. static final double PIO2 = Math.PI / 2D; + /** * Returns the trigonometric sine of an angle converted as a cos operation. *

    diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java index fbbf643a9faa..41136b954133 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java @@ -53,22 +53,22 @@ public static void beforeClass() throws Exception { // this is a simple systematic test GeoPointField[] pts = new GeoPointField[] { - new GeoPointField(FIELD_NAME, -96.774, 32.763420, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7759895324707, 32.7559529921407, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.77701950073242, 32.77866942010977, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7706036567688, 32.7756745755423, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -139.73458170890808, 27.703618681345585, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.4538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.65084838867188, 33.06047141970814, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7772, 32.778650, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -177.23537676036358, -88.56029371730983, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -26.779373834241003, 33.541429799076354, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -77.35379276106497, 26.774024500421728, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -14.796283808944777, -90.0, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -178.8538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 178.8538113027811, 32.94823588839368, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -73.998776, 40.720611, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -179.5, -44.5, GeoPointField.PREFIX_TYPE_NOT_STORED)}; + new GeoPointField(FIELD_NAME, 32.763420, -96.774, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.7559529921407, -96.7759895324707, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.77866942010977, -96.77701950073242, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.7756745755423, -96.7706036567688, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 27.703618681345585, -139.73458170890808, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, -96.4538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 33.06047141970814, -96.65084838867188, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.778650, -96.7772, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -88.56029371730983, -177.23537676036358, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 33.541429799076354, -26.779373834241003, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 26.774024500421728, -77.35379276106497, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -90.0, -14.796283808944777, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, -178.8538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, 178.8538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 40.720611, -73.998776, GeoPointField.PREFIX_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -44.5, -179.5, GeoPointField.PREFIX_TYPE_NOT_STORED)}; for (GeoPointField p : pts) { Document doc = new Document(); @@ -105,37 +105,39 @@ public static void afterClass() throws Exception { directory = null; } - private TopDocs bboxQuery(double minLon, double minLat, double maxLon, double maxLat, int limit) throws Exception { - GeoPointInBBoxQuery q = new GeoPointInBBoxQuery(FIELD_NAME, TermEncoding.PREFIX, minLon, minLat, maxLon, maxLat); + private TopDocs bboxQuery(double minLat, double maxLat, double minLon, double maxLon, int limit) throws Exception { + GeoPointInBBoxQuery q = new GeoPointInBBoxQuery(FIELD_NAME, TermEncoding.PREFIX, minLat, maxLat, minLon, maxLon); return searcher.search(q, limit); } - private TopDocs polygonQuery(double[] lon, double[] lat, int limit) throws Exception { - GeoPointInPolygonQuery q = new GeoPointInPolygonQuery(FIELD_NAME, TermEncoding.PREFIX, lon, lat); + private TopDocs polygonQuery(double[] polyLats, double[] polyLons, int limit) throws Exception { + GeoPointInPolygonQuery q = new GeoPointInPolygonQuery(FIELD_NAME, TermEncoding.PREFIX, polyLats, polyLons); return searcher.search(q, limit); } - private TopDocs geoDistanceQuery(double lon, double lat, double radius, int limit) throws Exception { - GeoPointDistanceQuery q = new GeoPointDistanceQuery(FIELD_NAME, TermEncoding.PREFIX, lon, lat, radius); + private TopDocs geoDistanceQuery(double lat, double lon, double radius, int limit) throws Exception { + GeoPointDistanceQuery q = new GeoPointDistanceQuery(FIELD_NAME, TermEncoding.PREFIX, lat, lon, radius); return searcher.search(q, limit); } - private TopDocs geoDistanceRangeQuery(double lon, double lat, double minRadius, double maxRadius, int limit) + private TopDocs geoDistanceRangeQuery(double lat, double lon, double minRadius, double maxRadius, int limit) throws Exception { - GeoPointDistanceRangeQuery q = new GeoPointDistanceRangeQuery(FIELD_NAME, TermEncoding.PREFIX, lon, lat, minRadius, maxRadius); + GeoPointDistanceRangeQuery q = new GeoPointDistanceRangeQuery(FIELD_NAME, TermEncoding.PREFIX, lat, lon, minRadius, maxRadius); return searcher.search(q, limit); } public void testBBoxQuery() throws Exception { - TopDocs td = bboxQuery(-96.7772, 32.778650, -96.77690000, 32.778950, 5); + TopDocs td = bboxQuery(32.778650, 32.778950, -96.7772, -96.77690000, 5); assertEquals("GeoBoundingBoxQuery failed", 4, td.totalHits); } public void testPolyQuery() throws Exception { - TopDocs td = polygonQuery(new double[]{-96.7682647, -96.8280029, -96.6288757, -96.4929199, - -96.6041564, -96.7449188, -96.76826477, -96.7682647}, - new double[]{33.073130, 32.9942669, 32.938386, 33.0374494, - 33.1369762, 33.1162747, 33.073130, 33.073130}, 5); + TopDocs td = polygonQuery( + new double[]{33.073130, 32.9942669, 32.938386, 33.0374494, + 33.1369762, 33.1162747, 33.073130, 33.073130}, + new double[]{-96.7682647, -96.8280029, -96.6288757, -96.4929199, + -96.6041564, -96.7449188, -96.76826477, -96.7682647}, + 5); assertEquals("GeoPolygonQuery failed", 2, td.totalHits); } @@ -157,56 +159,56 @@ public void testPacManPolyQuery() throws Exception { double yMax = 1;//5; // test cell crossing poly - assertTrue(GeoRelationUtils.rectCrossesPolyApprox(xMin, yMin, xMax, yMax, px, py, xMinA, yMinA, xMaxA, yMaxA)); - assertFalse(GeoRelationUtils.rectCrossesPolyApprox(-5, 0, 0.000001, 5, px, py, xMin, yMin, xMax, yMax)); - assertTrue(GeoRelationUtils.rectWithinPolyApprox(-5, 0, -2, 5, px, py, xMin, yMin, xMax, yMax)); + assertTrue(GeoRelationUtils.rectCrossesPolyApprox(yMin, yMax, xMin, yMax, py, px, yMinA, yMaxA, xMinA, xMaxA)); + assertFalse(GeoRelationUtils.rectCrossesPolyApprox(0, 5, -5, 0.000001, py, px, yMin, yMax, xMin, xMax)); + assertTrue(GeoRelationUtils.rectWithinPolyApprox(0, 5, -5, -2, py, px, yMin, yMax, xMin, xMax)); } public void testBBoxCrossDateline() throws Exception { - TopDocs td = bboxQuery(179.0, -45.0, -179.0, -44.0, 20); + TopDocs td = bboxQuery(-45.0, -44.0, 179.0, -179.0, 20); assertEquals("BBoxCrossDateline query failed", 2, td.totalHits); } public void testWholeMap() throws Exception { - TopDocs td = bboxQuery(GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LAT_INCL, 20); - assertEquals("testWholeMap failed", 24, td.totalHits); - td = polygonQuery(new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL}, - new double[] {GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MIN_LAT_INCL}, 20); + TopDocs td = bboxQuery(GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, 20); assertEquals("testWholeMap failed", 24, td.totalHits); + td = polygonQuery(new double[] {GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MIN_LAT_INCL}, + new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL}, + 20); assertEquals("testWholeMap failed", 24, td.totalHits); } public void smallTest() throws Exception { - TopDocs td = geoDistanceQuery(-73.998776, 40.720611, 1, 20); + TopDocs td = geoDistanceQuery(40.720611, -73.998776, 1, 20); assertEquals("smallTest failed", 2, td.totalHits); } // GeoBoundingBox should not accept invalid lat/lon public void testInvalidBBox() throws Exception { expectThrows(Exception.class, () -> { - bboxQuery(179.0, -92.0, 181.0, -91.0, 20); + bboxQuery(-92.0, -91.0, 179.0, 181.0, 20); }); } public void testGeoDistanceQuery() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000, 20); assertEquals("GeoDistanceQuery failed", 2, td.totalHits); } /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ public void testNonEmptyTermsEnum() throws Exception { - TopDocs td = geoDistanceQuery(-177.23537676036358, -88.56029371730983, 7757.999232959935, 20); + TopDocs td = geoDistanceQuery(-88.56029371730983, -177.23537676036358, 7757.999232959935, 20); assertEquals("GeoDistanceQuery failed", 2, td.totalHits); } public void testMultiValuedQuery() throws Exception { - TopDocs td = bboxQuery(-96.4538113027811, 32.7559529921407, -96.7706036567688, 32.7756745755423, 20); + TopDocs td = bboxQuery(32.7559529921407, 32.7756745755423, -96.4538113027811, -96.7706036567688, 20); // 3 single valued docs + 2 multi-valued docs assertEquals("testMultiValuedQuery failed", 5, td.totalHits); } public void testTooBigRadius() throws Exception { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - geoDistanceQuery(0.0, 85.0, 4000000, 20); + geoDistanceQuery(85.0, 0.0, 4000000, 20); }); assertTrue(expected.getMessage().contains("exceeds maxRadius")); } @@ -215,19 +217,19 @@ public void testTooBigRadius() throws Exception { * Explicitly large */ public void testGeoDistanceQueryHuge() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000000, 20); assertEquals("GeoDistanceQuery failed", 16, td.totalHits); } public void testGeoDistanceQueryCrossDateline() throws Exception { - TopDocs td = geoDistanceQuery(-179.9538113027811, 32.94823588839368, 120000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -179.9538113027811, 120000, 20); assertEquals("GeoDistanceQuery failed", 3, td.totalHits); } // GeoDistanceQuery should not accept invalid lat/lon as origin public void testInvalidGeoDistanceQuery() throws Exception { expectThrows(Exception.class, () -> { - geoDistanceQuery(181.0, 92.0, 120000, 20); + geoDistanceQuery(92.0, 181.0, 120000, 20); }); } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 3bf22d335fb1..53ff25c544c2 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -43,27 +43,27 @@ protected boolean forceSmall() { @Override protected void addPointToDoc(String field, Document doc, double lat, double lon) { - doc.add(new GeoPointField(field, lon, lat, GeoPointField.PREFIX_TYPE_NOT_STORED)); + doc.add(new GeoPointField(field, lat, lon, GeoPointField.PREFIX_TYPE_NOT_STORED)); } @Override protected Query newRectQuery(String field, GeoRect rect) { - return new GeoPointInBBoxQuery(field, TermEncoding.PREFIX, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + return new GeoPointInBBoxQuery(field, TermEncoding.PREFIX, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); } @Override protected Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters) { - return new GeoPointDistanceQuery(field, TermEncoding.PREFIX, centerLon, centerLat, radiusMeters); + return new GeoPointDistanceQuery(field, TermEncoding.PREFIX, centerLat, centerLon, radiusMeters); } @Override protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLon, centerLat, minRadiusMeters, radiusMeters); + return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, radiusMeters); } @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, lons, lats); + return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, lats, lons); } @Override @@ -77,11 +77,11 @@ protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointL } if (rect.minLon < rect.maxLon) { - return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); } else { // Rect crosses dateline: - return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, -180.0, rect.minLat, rect.maxLon, rect.maxLat) - || GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, 180.0, rect.maxLat); + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) + || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); } } @@ -112,10 +112,10 @@ protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, double ptLon, double ptLat, final double radius) { - final long hashedCntr = GeoEncodingUtils.mortonHash(centerLon, centerLat); + final long hashedCntr = GeoEncodingUtils.mortonHash(centerLat, centerLon); centerLon = GeoEncodingUtils.mortonUnhashLon(hashedCntr); centerLat = GeoEncodingUtils.mortonUnhashLat(hashedCntr); - final long hashedPt = GeoEncodingUtils.mortonHash(ptLon, ptLat); + final long hashedPt = GeoEncodingUtils.mortonHash(ptLat, ptLon); ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java index 69a3c3f6d23e..beab5b974774 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java @@ -55,22 +55,22 @@ public static void beforeClass() throws Exception { // this is a simple systematic test GeoPointField[] pts = new GeoPointField[] { - new GeoPointField(FIELD_NAME, -96.774, 32.763420, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7759895324707, 32.7559529921407, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.77701950073242, 32.77866942010977, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7706036567688, 32.7756745755423, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -139.73458170890808, 27.703618681345585, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.4538113027811, 32.94823588839368, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.65084838867188, 33.06047141970814, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -96.7772, 32.778650, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -177.23537676036358, -88.56029371730983, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -26.779373834241003, 33.541429799076354, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -77.35379276106497, 26.774024500421728, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -14.796283808944777, -90.0, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -178.8538113027811, 32.94823588839368, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 178.8538113027811, 32.94823588839368, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -73.998776, 40.720611, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -179.5, -44.5, GeoPointField.NUMERIC_TYPE_NOT_STORED)}; + new GeoPointField(FIELD_NAME, 32.763420, -96.774, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.7559529921407, -96.7759895324707, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.77866942010977, -96.77701950073242, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.7756745755423, -96.7706036567688, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 27.703618681345585, -139.73458170890808, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, -96.4538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 33.06047141970814, -96.65084838867188, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.778650, -96.7772, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -88.56029371730983, -177.23537676036358, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 33.541429799076354, -26.779373834241003, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 26.774024500421728, -77.35379276106497, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -90.0, -14.796283808944777, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, -178.8538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 32.94823588839368, 178.8538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, 40.720611, -73.998776, GeoPointField.NUMERIC_TYPE_NOT_STORED), + new GeoPointField(FIELD_NAME, -44.5, -179.5, GeoPointField.NUMERIC_TYPE_NOT_STORED)}; for (GeoPointField p : pts) { Document doc = new Document(); @@ -107,37 +107,39 @@ public static void afterClass() throws Exception { directory = null; } - private TopDocs bboxQuery(double minLon, double minLat, double maxLon, double maxLat, int limit) throws Exception { - GeoPointInBBoxQuery q = new GeoPointInBBoxQuery(FIELD_NAME, TermEncoding.NUMERIC, minLon, minLat, maxLon, maxLat); + private TopDocs bboxQuery(double minLat, double maxLat, double minLon, double maxLon, int limit) throws Exception { + GeoPointInBBoxQuery q = new GeoPointInBBoxQuery(FIELD_NAME, TermEncoding.NUMERIC, minLat, maxLat, minLon, maxLon); return searcher.search(q, limit); } - private TopDocs polygonQuery(double[] lon, double[] lat, int limit) throws Exception { - GeoPointInPolygonQuery q = new GeoPointInPolygonQuery(FIELD_NAME, TermEncoding.NUMERIC, lon, lat); + private TopDocs polygonQuery(double[] polyLats, double[] polyLons, int limit) throws Exception { + GeoPointInPolygonQuery q = new GeoPointInPolygonQuery(FIELD_NAME, TermEncoding.NUMERIC, polyLats, polyLons); return searcher.search(q, limit); } - private TopDocs geoDistanceQuery(double lon, double lat, double radius, int limit) throws Exception { - GeoPointDistanceQuery q = new GeoPointDistanceQuery(FIELD_NAME, TermEncoding.NUMERIC, lon, lat, radius); + private TopDocs geoDistanceQuery(double lat, double lon, double radius, int limit) throws Exception { + GeoPointDistanceQuery q = new GeoPointDistanceQuery(FIELD_NAME, TermEncoding.NUMERIC, lat, lon, radius); return searcher.search(q, limit); } - private TopDocs geoDistanceRangeQuery(double lon, double lat, double minRadius, double maxRadius, int limit) + private TopDocs geoDistanceRangeQuery(double lat, double lon, double minRadius, double maxRadius, int limit) throws Exception { - GeoPointDistanceRangeQuery q = new GeoPointDistanceRangeQuery(FIELD_NAME, TermEncoding.NUMERIC, lon, lat, minRadius, maxRadius); + GeoPointDistanceRangeQuery q = new GeoPointDistanceRangeQuery(FIELD_NAME, TermEncoding.NUMERIC, lat, lon, minRadius, maxRadius); return searcher.search(q, limit); } public void testBBoxQuery() throws Exception { - TopDocs td = bboxQuery(-96.7772, 32.778650, -96.77690000, 32.778950, 5); + TopDocs td = bboxQuery(32.778650, 32.778950, -96.7772, -96.77690000, 5); assertEquals("GeoBoundingBoxQuery failed", 4, td.totalHits); } public void testPolyQuery() throws Exception { - TopDocs td = polygonQuery(new double[]{-96.7682647, -96.8280029, -96.6288757, -96.4929199, - -96.6041564, -96.7449188, -96.76826477, -96.7682647}, + TopDocs td = polygonQuery( new double[]{33.073130, 32.9942669, 32.938386, 33.0374494, - 33.1369762, 33.1162747, 33.073130, 33.073130}, 5); + 33.1369762, 33.1162747, 33.073130, 33.073130}, + new double[]{-96.7682647, -96.8280029, -96.6288757, -96.4929199, + -96.6041564, -96.7449188, -96.76826477, -96.7682647}, + 5); assertEquals("GeoPolygonQuery failed", 2, td.totalHits); } @@ -159,56 +161,56 @@ public void testPacManPolyQuery() throws Exception { double yMax = 1;//5; // test cell crossing poly - assertTrue(GeoRelationUtils.rectCrossesPolyApprox(xMin, yMin, xMax, yMax, px, py, xMinA, yMinA, xMaxA, yMaxA)); - assertFalse(GeoRelationUtils.rectCrossesPolyApprox(-5, 0, 0.000001, 5, px, py, xMin, yMin, xMax, yMax)); - assertTrue(GeoRelationUtils.rectWithinPolyApprox(-5, 0, -2, 5, px, py, xMin, yMin, xMax, yMax)); + assertTrue(GeoRelationUtils.rectCrossesPolyApprox(yMin, yMax, xMin, yMax, py, px, yMinA, yMaxA, xMinA, xMaxA)); + assertFalse(GeoRelationUtils.rectCrossesPolyApprox(0, 5, -5, 0.000001, py, px, yMin, yMax, xMin, xMax)); + assertTrue(GeoRelationUtils.rectWithinPolyApprox(0, 5, -5, -2, py, px, yMin, yMax, xMin, xMax)); } public void testBBoxCrossDateline() throws Exception { - TopDocs td = bboxQuery(179.0, -45.0, -179.0, -44.0, 20); + TopDocs td = bboxQuery(-45.0, -44.0, 179.0, -179.0, 20); assertEquals("BBoxCrossDateline query failed", 2, td.totalHits); } public void testWholeMap() throws Exception { - TopDocs td = bboxQuery(GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LAT_INCL, 20); - assertEquals("testWholeMap failed", 24, td.totalHits); - td = polygonQuery(new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL}, - new double[] {GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MIN_LAT_INCL}, 20); + TopDocs td = bboxQuery(GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, 20); assertEquals("testWholeMap failed", 24, td.totalHits); + td = polygonQuery(new double[] {GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MIN_LAT_INCL}, + new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL}, + 20); assertEquals("testWholeMap failed", 24, td.totalHits); } public void smallTest() throws Exception { - TopDocs td = geoDistanceQuery(-73.998776, 40.720611, 1, 20); + TopDocs td = geoDistanceQuery(40.720611, -73.998776, 1, 20); assertEquals("smallTest failed", 2, td.totalHits); } // GeoBoundingBox should not accept invalid lat/lon public void testInvalidBBox() throws Exception { expectThrows(Exception.class, () -> { - bboxQuery(179.0, -92.0, 181.0, -91.0, 20); + bboxQuery(-92.0, -91.0, 179.0, 181.0, 20); }); } public void testGeoDistanceQuery() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000, 20); assertEquals("GeoDistanceQuery failed", 2, td.totalHits); } /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ public void testNonEmptyTermsEnum() throws Exception { - TopDocs td = geoDistanceQuery(-177.23537676036358, -88.56029371730983, 7757.999232959935, 20); + TopDocs td = geoDistanceQuery(-88.56029371730983, -177.23537676036358, 7757.999232959935, 20); assertEquals("GeoDistanceQuery failed", 2, td.totalHits); } public void testMultiValuedQuery() throws Exception { - TopDocs td = bboxQuery(-96.4538113027811, 32.7559529921407, -96.7706036567688, 32.7756745755423, 20); + TopDocs td = bboxQuery(32.7559529921407, 32.7756745755423, -96.4538113027811, -96.7706036567688, 20); // 3 single valued docs + 2 multi-valued docs assertEquals("testMultiValuedQuery failed", 5, td.totalHits); } public void testTooBigRadius() throws Exception { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - geoDistanceQuery(0.0, 85.0, 4000000, 20); + geoDistanceQuery(85.0, 0.0, 4000000, 20); }); assertTrue(expected.getMessage().contains("exceeds maxRadius")); } @@ -217,19 +219,19 @@ public void testTooBigRadius() throws Exception { * Explicitly large */ public void testGeoDistanceQueryHuge() throws Exception { - TopDocs td = geoDistanceQuery(-96.4538113027811, 32.94823588839368, 6000000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000000, 20); assertEquals("GeoDistanceQuery failed", 16, td.totalHits); } public void testGeoDistanceQueryCrossDateline() throws Exception { - TopDocs td = geoDistanceQuery(-179.9538113027811, 32.94823588839368, 120000, 20); + TopDocs td = geoDistanceQuery(32.94823588839368, -179.9538113027811, 120000, 20); assertEquals("GeoDistanceQuery failed", 3, td.totalHits); } // GeoDistanceQuery should not accept invalid lat/lon as origin public void testInvalidGeoDistanceQuery() throws Exception { expectThrows(Exception.class, () -> { - geoDistanceQuery(181.0, 92.0, 120000, 20); + geoDistanceQuery(92.0, 181.0, 120000, 20); }); } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index c20faffd0b28..90bfb107e347 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -42,27 +42,27 @@ protected boolean forceSmall() { @Override protected void addPointToDoc(String field, Document doc, double lat, double lon) { - doc.add(new GeoPointField(field, lon, lat, GeoPointField.NUMERIC_TYPE_NOT_STORED)); + doc.add(new GeoPointField(field, lat, lon, GeoPointField.NUMERIC_TYPE_NOT_STORED)); } @Override protected Query newRectQuery(String field, GeoRect rect) { - return new GeoPointInBBoxQuery(field, TermEncoding.NUMERIC, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + return new GeoPointInBBoxQuery(field, TermEncoding.NUMERIC, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); } @Override protected Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters) { - return new GeoPointDistanceQuery(field, TermEncoding.NUMERIC, centerLon, centerLat, radiusMeters); + return new GeoPointDistanceQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, radiusMeters); } @Override protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLon, centerLat, minRadiusMeters, radiusMeters); + return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, minRadiusMeters, radiusMeters); } @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lons, lats); + return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lats, lons); } @Override @@ -76,11 +76,11 @@ protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointL } if (rect.minLon < rect.maxLon) { - return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, rect.maxLon, rect.maxLat); + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); } else { // Rect crosses dateline: - return GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, -180.0, rect.minLat, rect.maxLon, rect.maxLat) - || GeoRelationUtils.pointInRectPrecise(pointLon, pointLat, rect.minLon, rect.minLat, 180.0, rect.maxLat); + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) + || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); } } @@ -111,10 +111,10 @@ protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, double ptLon, double ptLat, final double radius) { - final long hashedCntr = GeoEncodingUtils.mortonHash(centerLon, centerLat); + final long hashedCntr = GeoEncodingUtils.mortonHash(centerLat, centerLon); centerLon = GeoEncodingUtils.mortonUnhashLon(hashedCntr); centerLat = GeoEncodingUtils.mortonUnhashLat(hashedCntr); - final long hashedPt = GeoEncodingUtils.mortonHash(ptLon, ptLat); + final long hashedPt = GeoEncodingUtils.mortonHash(ptLat, ptLon); ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index e14ffbde2e08..a4f70d911e30 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -466,7 +466,7 @@ protected GeoRect randomRect(boolean small, boolean canCrossDateLine) { lon1 = x; } - return new GeoRect(lon0, lon1, lat0, lat1); + return new GeoRect(lat0, lat1, lon0, lon1); } protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 37938328725d..f9dde508c68d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -66,154 +66,13 @@ public static void beforeClass() throws Exception { } } - public void testGeoHash() { - int numPoints = atLeast(100); - String randomGeoHashString; - String mortonGeoHash; - long mortonLongFromGHLong, geoHashLong, mortonLongFromGHString; - int randomLevel; - for (int i = 0; i < numPoints; ++i) { - // random point - double lat = randomLat(false); - double lon = randomLon(false); - - // compute geohash straight from lat/lon and from morton encoded value to ensure they're the same - randomGeoHashString = GeoHashUtils.stringEncode(lon, lat, randomLevel = random().nextInt(12 - 1) + 1); - mortonGeoHash = GeoHashUtils.stringEncodeFromMortonLong(GeoEncodingUtils.mortonHash(lon, lat), randomLevel); - assertEquals(randomGeoHashString, mortonGeoHash); - - // v&v conversion from lat/lon or geohashstring to geohash long and back to geohash string - geoHashLong = (random().nextBoolean()) ? GeoHashUtils.longEncode(lon, lat, randomLevel) : GeoHashUtils.longEncode(randomGeoHashString); - assertEquals(randomGeoHashString, GeoHashUtils.stringEncode(geoHashLong)); - - // v&v conversion from geohash long to morton long - mortonLongFromGHString = GeoHashUtils.mortonEncode(randomGeoHashString); - mortonLongFromGHLong = GeoHashUtils.mortonEncode(geoHashLong); - assertEquals(mortonLongFromGHLong, mortonLongFromGHString); - - // v&v lat/lon from geohash string and geohash long - assertEquals(GeoEncodingUtils.mortonUnhashLat(mortonLongFromGHString), GeoEncodingUtils.mortonUnhashLat(mortonLongFromGHLong), 0); - assertEquals(GeoEncodingUtils.mortonUnhashLon(mortonLongFromGHString), GeoEncodingUtils.mortonUnhashLon(mortonLongFromGHLong), 0); - } - } - - /** - * Pass condition: lat=42.6, lng=-5.6 should be encoded as "ezs42e44yx96", - * lat=57.64911 lng=10.40744 should be encoded as "u4pruydqqvj8" - */ - public void testEncode() { - String hash = GeoHashUtils.stringEncode(-5.6, 42.6, 12); - assertEquals("ezs42e44yx96", hash); - - hash = GeoHashUtils.stringEncode(10.40744, 57.64911, 12); - assertEquals("u4pruydqqvj8", hash); - } - - /** - * Pass condition: lat=52.3738007, lng=4.8909347 should be encoded and then - * decoded within 0.00001 of the original value - */ - public void testDecodePreciseLongitudeLatitude() { - final String geohash = GeoHashUtils.stringEncode(4.8909347, 52.3738007); - final long hash = GeoHashUtils.mortonEncode(geohash); - - assertEquals(52.3738007, GeoEncodingUtils.mortonUnhashLat(hash), 0.00001D); - assertEquals(4.8909347, GeoEncodingUtils.mortonUnhashLon(hash), 0.00001D); - } - - /** - * Pass condition: lat=84.6, lng=10.5 should be encoded and then decoded - * within 0.00001 of the original value - */ - public void testDecodeImpreciseLongitudeLatitude() { - final String geohash = GeoHashUtils.stringEncode(10.5, 84.6); - - final long hash = GeoHashUtils.mortonEncode(geohash); - - assertEquals(84.6, GeoEncodingUtils.mortonUnhashLat(hash), 0.00001D); - assertEquals(10.5, GeoEncodingUtils.mortonUnhashLon(hash), 0.00001D); - } - - public void testDecodeEncode() { - final String geoHash = "u173zq37x014"; - assertEquals(geoHash, GeoHashUtils.stringEncode(4.8909347, 52.3738007)); - final long mortonHash = GeoHashUtils.mortonEncode(geoHash); - final double lon = GeoEncodingUtils.mortonUnhashLon(mortonHash); - final double lat = GeoEncodingUtils.mortonUnhashLat(mortonHash); - assertEquals(52.37380061d, GeoEncodingUtils.mortonUnhashLat(mortonHash), 0.000001d); - assertEquals(4.8909343d, GeoEncodingUtils.mortonUnhashLon(mortonHash), 0.000001d); - - assertEquals(geoHash, GeoHashUtils.stringEncode(lon, lat)); - } - - public void testNeighbors() { - String geohash = "gcpv"; - List expectedNeighbors = new ArrayList<>(); - expectedNeighbors.add("gcpw"); - expectedNeighbors.add("gcpy"); - expectedNeighbors.add("u10n"); - expectedNeighbors.add("gcpt"); - expectedNeighbors.add("u10j"); - expectedNeighbors.add("gcps"); - expectedNeighbors.add("gcpu"); - expectedNeighbors.add("u10h"); - Collection neighbors = new ArrayList<>(); - GeoHashUtils.addNeighbors(geohash, neighbors ); - assertEquals(expectedNeighbors, neighbors); - - // Border odd geohash - geohash = "u09x"; - expectedNeighbors = new ArrayList<>(); - expectedNeighbors.add("u0c2"); - expectedNeighbors.add("u0c8"); - expectedNeighbors.add("u0cb"); - expectedNeighbors.add("u09r"); - expectedNeighbors.add("u09z"); - expectedNeighbors.add("u09q"); - expectedNeighbors.add("u09w"); - expectedNeighbors.add("u09y"); - neighbors = new ArrayList<>(); - GeoHashUtils.addNeighbors(geohash, neighbors); - assertEquals(expectedNeighbors, neighbors); - - // Border even geohash - geohash = "u09tv"; - expectedNeighbors = new ArrayList<>(); - expectedNeighbors.add("u09wh"); - expectedNeighbors.add("u09wj"); - expectedNeighbors.add("u09wn"); - expectedNeighbors.add("u09tu"); - expectedNeighbors.add("u09ty"); - expectedNeighbors.add("u09ts"); - expectedNeighbors.add("u09tt"); - expectedNeighbors.add("u09tw"); - neighbors = new ArrayList<>(); - GeoHashUtils.addNeighbors(geohash, neighbors ); - assertEquals(expectedNeighbors, neighbors); - - // Border even and odd geohash - geohash = "ezzzz"; - expectedNeighbors = new ArrayList<>(); - expectedNeighbors.add("gbpbn"); - expectedNeighbors.add("gbpbp"); - expectedNeighbors.add("u0000"); - expectedNeighbors.add("ezzzy"); - expectedNeighbors.add("spbpb"); - expectedNeighbors.add("ezzzw"); - expectedNeighbors.add("ezzzx"); - expectedNeighbors.add("spbp8"); - neighbors = new ArrayList<>(); - GeoHashUtils.addNeighbors(geohash, neighbors ); - assertEquals(expectedNeighbors, neighbors); - } - public void testClosestPointOnBBox() { double[] result = new double[2]; - GeoDistanceUtils.closestPointOnBBox(20, 30, 40, 50, 70, 70, result); - assertEquals(40.0, result[0], 0.0); - assertEquals(50.0, result[1], 0.0); + GeoDistanceUtils.closestPointOnBBox(30, 50, 20, 40, 70, 70, result); + assertEquals(50.0, result[0], 0.0); + assertEquals(40.0, result[1], 0.0); - GeoDistanceUtils.closestPointOnBBox(-20, -20, 0, 0, 70, 70, result); + GeoDistanceUtils.closestPointOnBBox(-20, 0, -20, 0, 70, 70, result); assertEquals(0.0, result[0], 0.0); assertEquals(0.0, result[1], 0.0); } @@ -331,7 +190,7 @@ private void findMatches(Set hits, PrintWriter log, Cell root, } } else { - if (GeoRelationUtils.rectWithinCircle(cell.minLon, cell.minLat, cell.maxLon, cell.maxLat, centerLon, centerLat, radiusMeters)) { + if (GeoRelationUtils.rectWithinCircle(cell.minLat, cell.maxLat, cell.minLon, cell.maxLon, centerLat, centerLon, radiusMeters, false)) { // Query circle fully contains this cell, just addAll: if (VERBOSE) { log.println(" circle fully contains cell: now addAll"); @@ -345,14 +204,14 @@ private void findMatches(Set hits, PrintWriter log, Cell root, } } continue; - } else if (GeoRelationUtils.rectWithin(root.minLon, root.minLat, root.maxLon, root.maxLat, - cell.minLon, cell.minLat, cell.maxLon, cell.maxLat)) { + } else if (GeoRelationUtils.rectWithin(root.minLat, root.maxLat, root.minLon, root.maxLon, + cell.minLat, cell.maxLat, cell.minLon, cell.maxLon)) { // Fall through below to "recurse" if (VERBOSE) { log.println(" cell fully contains circle: keep splitting"); } - } else if (GeoRelationUtils.rectCrossesCircle(cell.minLon, cell.minLat, cell.maxLon, cell.maxLat, - centerLon, centerLat, radiusMeters)) { + } else if (GeoRelationUtils.rectCrossesCircle(cell.minLat, cell.maxLat, cell.minLon, cell.maxLon, + centerLat, centerLon, radiusMeters, false)) { // Fall through below to "recurse" if (VERBOSE) { log.println(" cell overlaps circle: keep splitting"); @@ -470,7 +329,7 @@ public void testGeoRelations() throws Exception { log.println("\nTEST: iter=" + iter + " radiusMeters=" + radiusMeters + " centerLon=" + centerLon + " centerLat=" + centerLat); } - GeoRect bbox = GeoUtils.circleToBBox(centerLon, centerLat, radiusMeters); + GeoRect bbox = GeoUtils.circleToBBox(centerLat, centerLon, radiusMeters); Set hits = new HashSet<>(); @@ -555,7 +414,7 @@ public void testGeoPrefixCoding() throws Exception { long decodedHash; BytesRefBuilder brb = new BytesRefBuilder(); while (numIters-- >= 0) { - hash = GeoEncodingUtils.mortonHash(randomLon(false), randomLat(false)); + hash = GeoEncodingUtils.mortonHash(randomLat(false), randomLon(false)); for (int i=32; i<64; ++i) { GeoEncodingUtils.geoCodedToPrefixCoded(hash, i, brb); decodedHash = GeoEncodingUtils.prefixCodedToGeoCoded(brb.get()); @@ -565,7 +424,7 @@ public void testGeoPrefixCoding() throws Exception { } public void testMortonEncoding() throws Exception { - long hash = GeoEncodingUtils.mortonHash(180, 90); + long hash = GeoEncodingUtils.mortonHash(90, 180); assertEquals(180.0, GeoEncodingUtils.mortonUnhashLon(hash), 0); assertEquals(90.0, GeoEncodingUtils.mortonUnhashLat(hash), 0); } @@ -577,7 +436,7 @@ public void testEncodeDecode() throws Exception { double lat = randomLat(small); double lon = randomLon(small); - long enc = GeoEncodingUtils.mortonHash(lon, lat); + long enc = GeoEncodingUtils.mortonHash(lat, lon); double latEnc = GeoEncodingUtils.mortonUnhashLat(enc); double lonEnc = GeoEncodingUtils.mortonUnhashLon(enc); @@ -593,11 +452,11 @@ public void testScaleUnscaleIsStable() throws Exception { double lat = randomLat(small); double lon = randomLon(small); - long enc = GeoEncodingUtils.mortonHash(lon, lat); + long enc = GeoEncodingUtils.mortonHash(lat, lon); double latEnc = GeoEncodingUtils.mortonUnhashLat(enc); double lonEnc = GeoEncodingUtils.mortonUnhashLon(enc); - long enc2 = GeoEncodingUtils.mortonHash(lon, lat); + long enc2 = GeoEncodingUtils.mortonHash(lat, lon); double latEnc2 = GeoEncodingUtils.mortonUnhashLat(enc2); double lonEnc2 = GeoEncodingUtils.mortonUnhashLon(enc2); assertEquals(latEnc, latEnc2, 0.0); @@ -631,7 +490,7 @@ public void testRandomCircleToBBox() throws Exception { // TODO: randomly quantize radius too, to provoke exact math errors? - GeoRect bbox = GeoUtils.circleToBBox(centerLon, centerLat, radiusMeters); + GeoRect bbox = GeoUtils.circleToBBox(centerLat, centerLon, radiusMeters); int numPointsToTry = 1000; for(int i=0;i Date: Tue, 22 Mar 2016 16:53:25 -0400 Subject: [PATCH 0192/1113] LUCENE-7127: remove epsilon-based testing from lucene/spatial, fix distance bugs. --- lucene/CHANGES.txt | 4 + .../TestLatLonPointDistanceQuery.java | 108 ------ .../lucene/search/TestLatLonPointQueries.java | 45 --- .../search/GeoPointDistanceQueryImpl.java | 24 +- .../lucene/spatial/util/GeoDistanceUtils.java | 8 +- .../lucene/spatial/util/GeoEncodingUtils.java | 6 - .../spatial/util/GeoProjectionUtils.java | 158 --------- .../lucene/spatial/util/GeoRelationUtils.java | 250 -------------- .../apache/lucene/spatial/util/GeoUtils.java | 44 +-- .../geopoint/search/TestGeoPointQuery.java | 84 +---- .../search/TestLegacyGeoPointQuery.java | 83 ++--- .../spatial/util/BaseGeoPointTestCase.java | 206 +++++++++--- .../lucene/spatial/util/TestGeoUtils.java | 308 ------------------ 13 files changed, 234 insertions(+), 1094 deletions(-) delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 138f6bd069fd..6dbacef7aab8 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -29,6 +29,10 @@ Optimizations * LUCENE-7115: Speed up FieldCache.CacheEntry toString by setting initial StringBuilder capacity (Gregory Chanan) +Bug Fixes + +* LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java index 983b1a7afa94..f015780be2c8 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java @@ -16,29 +16,11 @@ */ package org.apache.lucene.document; -import java.io.IOException; -import java.util.BitSet; - -import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.PointsFormat; -import org.apache.lucene.codecs.PointsReader; -import org.apache.lucene.codecs.PointsWriter; -import org.apache.lucene.codecs.lucene60.Lucene60PointsReader; -import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.SloppyMath; -import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.bkd.BKDWriter; /** Simple tests for {@link LatLonPoint#newDistanceQuery} */ public class TestLatLonPointDistanceQuery extends LuceneTestCase { @@ -97,94 +79,4 @@ public void testInfRadius() { assertTrue(expected.getMessage().contains("radiusMeters")); assertTrue(expected.getMessage().contains("is invalid")); } - - /** Run a few iterations with just 10 docs, hopefully easy to debug */ - public void testRandom() throws Exception { - for (int iters = 0; iters < 100; iters++) { - doRandomTest(10, 100); - } - } - - /** Runs with thousands of docs */ - @Nightly - public void testRandomHuge() throws Exception { - for (int iters = 0; iters < 10; iters++) { - doRandomTest(2000, 100); - } - } - - private void doRandomTest(int numDocs, int numQueries) throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(); - int pointsInLeaf = 2 + random().nextInt(4); - iwc.setCodec(new FilterCodec("Lucene60", TestUtil.getDefaultCodec()) { - @Override - public PointsFormat pointsFormat() { - return new PointsFormat() { - @Override - public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, pointsInLeaf, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); - } - - @Override - public PointsReader fieldsReader(SegmentReadState readState) throws IOException { - return new Lucene60PointsReader(readState); - } - }; - } - }); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); - - for (int i = 0; i < numDocs; i++) { - double latRaw = -90 + 180.0 * random().nextDouble(); - double lonRaw = -180 + 360.0 * random().nextDouble(); - // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons - double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); - double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); - Document doc = new Document(); - doc.add(new LatLonPoint("field", lat, lon)); - doc.add(new StoredField("lat", lat)); - doc.add(new StoredField("lon", lon)); - writer.addDocument(doc); - } - IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); - - for (int i = 0; i < numQueries; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); - double radius = 50000000 * random().nextDouble(); - - BitSet expected = new BitSet(); - for (int doc = 0; doc < reader.maxDoc(); doc++) { - double docLatitude = reader.document(doc).getField("lat").numericValue().doubleValue(); - double docLongitude = reader.document(doc).getField("lon").numericValue().doubleValue(); - double distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude); - if (distance <= radius) { - expected.set(doc); - } - } - - TopDocs topDocs = searcher.search(LatLonPoint.newDistanceQuery("field", lat, lon, radius), reader.maxDoc(), Sort.INDEXORDER); - BitSet actual = new BitSet(); - for (ScoreDoc doc : topDocs.scoreDocs) { - actual.set(doc.doc); - } - - try { - assertEquals(expected, actual); - } catch (AssertionError e) { - for (int doc = 0; doc < reader.maxDoc(); doc++) { - double docLatitude = reader.document(doc).getField("lat").numericValue().doubleValue(); - double docLongitude = reader.document(doc).getField("lon").numericValue().doubleValue(); - double distance = SloppyMath.haversinMeters(lat, lon, docLatitude, docLongitude); - System.out.println("" + doc + ": (" + docLatitude + "," + docLongitude + "), distance=" + distance); - } - throw e; - } - } - reader.close(); - writer.close(); - dir.close(); - } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index df34940db278..71ef7efa9555 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -20,8 +20,6 @@ import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.util.SloppyMath; public class TestLatLonPointQueries extends BaseGeoPointTestCase { @@ -50,19 +48,6 @@ protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return LatLonPoint.newPolygonQuery(FIELD_NAME, lats, lons); } - @Override - protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - assert Double.isNaN(pointLat) == false; - - if (rect.minLon < rect.maxLon) { - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); - } else { - // Rect crosses dateline: - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) - || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); - } - } - @Override protected double quantizeLat(double latRaw) { return LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); @@ -72,34 +57,4 @@ protected double quantizeLat(double latRaw) { protected double quantizeLon(double lonRaw) { return LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); } - - @Override - protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - // TODO write better random polygon tests - - // note: logic must be slightly different than rectContainsPoint, to satisfy - // insideness for cases exactly on boundaries. - - assert Double.isNaN(pointLat) == false; - assert rect.crossesDateline() == false; - double polyLats[] = new double[] { rect.minLat, rect.maxLat, rect.maxLat, rect.minLat, rect.minLat }; - double polyLons[] = new double[] { rect.minLon, rect.minLon, rect.maxLon, rect.maxLon, rect.minLon }; - - // TODO: separately test this method is 100% correct, here treat it like a black box (like haversin) - return GeoRelationUtils.pointInPolygon(polyLats, polyLons, pointLat, pointLon); - } - - @Override - protected Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { - double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - boolean result = distanceMeters <= radiusMeters; - //System.out.println(" shouldMatch? centerLon=" + centerLon + " centerLat=" + centerLat + " pointLon=" + pointLon + " pointLat=" + pointLat + " result=" + result + " distanceMeters=" + (distanceKM * 1000)); - return result; - } - - @Override - protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { - final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - return d >= minRadiusMeters && d <= radiusMeters; - } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index 8f50db5bed6c..d4272a34bf11 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -19,7 +19,6 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoRelationUtils; import org.apache.lucene.util.SloppyMath; /** Package private implementation for the public facing GeoPointDistanceQuery delegate class. @@ -54,15 +53,28 @@ private final class GeoPointRadiusCellComparator extends CellComparator { @Override protected boolean cellCrosses(final double minLat, final double maxLat, final double minLon, final double maxLon) { - return GeoRelationUtils.rectCrossesCircle(minLat, maxLat, minLon, maxLon, - distanceQuery.centerLat, centerLon, distanceQuery.radiusMeters, true); + if (maxLat < GeoPointDistanceQueryImpl.this.minLat || + maxLon < GeoPointDistanceQueryImpl.this.minLon || + minLat > GeoPointDistanceQueryImpl.this.maxLat || + minLon > GeoPointDistanceQueryImpl.this.maxLon) { + return false; + } else { + return true; + } } @Override protected boolean cellWithin(final double minLat, final double maxLat, final double minLon, final double maxLon) { - return GeoRelationUtils.rectWithinCircle(minLat, maxLat, minLon, maxLon, - distanceQuery.centerLat, centerLon, - distanceQuery.radiusMeters, true); + // TODO: we call cellCrosses because of how the termsEnum logic works, helps us avoid some haversin() calls here. + if (cellCrosses(minLat, maxLat, minLon, maxLon) && maxLon - centerLon < 90 && centerLon - minLon < 90 && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, minLon) <= distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, maxLon) <= distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, minLon) <= distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, maxLon) <= distanceQuery.radiusMeters) { + // we are fully enclosed, collect everything within this subtree + return true; + } + return false; } @Override diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java index 8956c4b3a137..32a16cdcc5ae 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java @@ -18,16 +18,12 @@ import org.apache.lucene.util.SloppyMath; -import static org.apache.lucene.util.SloppyMath.TO_RADIANS; - /** * Reusable geo-spatial distance utility methods. * * @lucene.experimental */ public class GeoDistanceUtils { - /** error threshold for point-distance queries (in percent) NOTE: Guideline from USGS is 0.005 **/ - public static final double DISTANCE_PCT_ERR = 0.005; // No instance: private GeoDistanceUtils() { @@ -58,7 +54,7 @@ public static double distanceToDegreesLon(double lat, double distance) { lat = StrictMath.toRadians(lat); // get the diameter at the latitude - final double diameter = 2 * GeoProjectionUtils.SEMIMAJOR_AXIS; + final double diameter = 2 * GeoUtils.SEMIMAJOR_AXIS; // compute inverse haversine double a = StrictMath.sin(distance/diameter); @@ -125,7 +121,7 @@ public static double maxRadialDistanceMeters(final double centerLat, final doubl */ public static double distanceToDegreesLat(double lat, double distance) { // get the diameter at the latitude - final double diameter = 2 * GeoProjectionUtils.SEMIMAJOR_AXIS; + final double diameter = 2 * GeoUtils.SEMIMAJOR_AXIS; // compute inverse haversine double a = StrictMath.sin(distance/diameter); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java index 38eef04b69dd..7bed892bc81b 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java @@ -84,12 +84,6 @@ private static final double unscaleLat(final long val) { return (val / LAT_SCALE) + MIN_LAT_INCL; } - /** Compare two position values within a {@link GeoEncodingUtils#TOLERANCE} factor */ - public static double compare(final double v1, final double v2) { - final double delta = v1-v2; - return Math.abs(delta) <= TOLERANCE ? 0 : delta; - } - /** Convert a geocoded morton long into a prefix coded geo term */ public static void geoCodedToPrefixCoded(long hash, int shift, BytesRefBuilder bytes) { geoCodedToPrefixCodedBytes(hash, shift, bytes); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java deleted file mode 100644 index 61aa09fb9c84..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoProjectionUtils.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.util; - -import static java.lang.StrictMath.sqrt; - -import static org.apache.lucene.util.SloppyMath.asin; -import static org.apache.lucene.util.SloppyMath.cos; -import static org.apache.lucene.util.SloppyMath.TO_DEGREES; -import static org.apache.lucene.util.SloppyMath.TO_RADIANS; - -import static org.apache.lucene.spatial.util.GeoUtils.MAX_LAT_INCL; -import static org.apache.lucene.spatial.util.GeoUtils.MAX_LON_INCL; -import static org.apache.lucene.spatial.util.GeoUtils.MIN_LAT_INCL; -import static org.apache.lucene.spatial.util.GeoUtils.MIN_LON_INCL; -import static org.apache.lucene.spatial.util.GeoUtils.PIO2; -import static org.apache.lucene.spatial.util.GeoUtils.normalizeLat; -import static org.apache.lucene.spatial.util.GeoUtils.normalizeLon; -import static org.apache.lucene.spatial.util.GeoUtils.sloppySin; -import static org.apache.lucene.spatial.util.GeoUtils.sloppyTan; - -/** - * Reusable geo-spatial projection utility methods. - * - * @lucene.experimental - */ -public class GeoProjectionUtils { - // WGS84 earth-ellipsoid parameters - /** major (a) axis in meters */ - public static final double SEMIMAJOR_AXIS = 6_378_137; // [m] - /** earth flattening factor (f) */ - public static final double FLATTENING = 1.0/298.257223563; - /** minor (b) axis in meters */ - public static final double SEMIMINOR_AXIS = SEMIMAJOR_AXIS * (1.0 - FLATTENING); //6_356_752.31420; // [m] - /** first eccentricity (e) */ - public static final double ECCENTRICITY = sqrt((2.0 - FLATTENING) * FLATTENING); - /** major axis squared (a2) */ - public static final double SEMIMAJOR_AXIS2 = SEMIMAJOR_AXIS * SEMIMAJOR_AXIS; - /** minor axis squared (b2) */ - public static final double SEMIMINOR_AXIS2 = SEMIMINOR_AXIS * SEMIMINOR_AXIS; - private static final double E2 = (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2)/(SEMIMAJOR_AXIS2); - private static final double EP2 = (SEMIMAJOR_AXIS2 - SEMIMINOR_AXIS2)/(SEMIMINOR_AXIS2); - - /** min longitude value in radians */ - public static final double MIN_LON_RADIANS = TO_RADIANS * MIN_LON_INCL; - /** min latitude value in radians */ - public static final double MIN_LAT_RADIANS = TO_RADIANS * MIN_LAT_INCL; - /** max longitude value in radians */ - public static final double MAX_LON_RADIANS = TO_RADIANS * MAX_LON_INCL; - /** max latitude value in radians */ - public static final double MAX_LAT_RADIANS = TO_RADIANS * MAX_LAT_INCL; - - // No instance: - private GeoProjectionUtils() { - } - - /** - * Converts from geodesic lat lon alt to geocentric earth-centered earth-fixed - * @param lat geodesic latitude - * @param lon geodesic longitude - * @param alt geodesic altitude - * @param ecf reusable earth-centered earth-fixed result - * @return either a new ecef array or the reusable ecf parameter - */ - public static final double[] llaToECF(double lat, double lon, double alt, double[] ecf) { - lon = TO_RADIANS * lon; - lat = TO_RADIANS * lat; - - final double sl = sloppySin(lat); - final double s2 = sl*sl; - final double cl = cos(lat); - - if (ecf == null) { - ecf = new double[3]; - } - - if (lat < -PIO2 && lat > -1.001D * PIO2) { - lat = -PIO2; - } else if (lat > PIO2 && lat < 1.001D * PIO2) { - lat = PIO2; - } - assert (lat >= -PIO2) || (lat <= PIO2); - - if (lon > StrictMath.PI) { - lon -= (2*StrictMath.PI); - } - - final double rn = SEMIMAJOR_AXIS / StrictMath.sqrt(1.0D - E2 * s2); - ecf[0] = (rn+alt) * cl * cos(lon); - ecf[1] = (rn+alt) * cl * sloppySin(lon); - ecf[2] = ((rn*(1.0-E2))+alt)*sl; - - return ecf; - } - - /** - * Finds a point along a bearing from a given lat,lon geolocation using great circle arc - * - * @param lat origin latitude in degrees - * @param lon origin longitude in degrees - * @param bearing azimuthal bearing in degrees - * @param dist distance in meters - * @param pt resulting point - * @return the point along a bearing at a given distance in meters - */ - public static final double[] pointFromLonLatBearingGreatCircle(double lat, double lon, double bearing, double dist, double[] pt) { - - if (pt == null) { - pt = new double[2]; - } - - lat *= TO_RADIANS; - lon *= TO_RADIANS; - bearing *= TO_RADIANS; - - final double cLat = cos(lat); - final double sLat = sloppySin(lat); - final double sinDoR = sloppySin(dist / GeoProjectionUtils.SEMIMAJOR_AXIS); - final double cosDoR = cos(dist / GeoProjectionUtils.SEMIMAJOR_AXIS); - - pt[1] = asin(sLat*cosDoR + cLat * sinDoR * cos(bearing)); - pt[0] = TO_DEGREES * (lon + Math.atan2(sloppySin(bearing) * sinDoR * cLat, cosDoR - sLat * sloppySin(pt[1]))); - pt[1] *= TO_DEGREES; - - return pt; - } - - /** - * Finds the bearing (in degrees) between 2 geo points (lat, lon) using great circle arc - * @param lat1 first point latitude in degrees - * @param lon1 first point longitude in degrees - * @param lat2 second point latitude in degrees - * @param lon2 second point longitude in degrees - * @return the bearing (in degrees) between the two provided points - */ - public static double bearingGreatCircle(double lat1, double lon1, double lat2, double lon2) { - double dLon = (lon2 - lon1) * TO_RADIANS; - lat2 *= TO_RADIANS; - lat1 *= TO_RADIANS; - double y = sloppySin(dLon) * cos(lat2); - double x = cos(lat1) * sloppySin(lat2) - sloppySin(lat1) * cos(lat2) * cos(dLon); - return Math.atan2(y, x) * TO_DEGREES; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java index 4a31202931d4..ba94e9dfa4c2 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java @@ -16,8 +16,6 @@ */ package org.apache.lucene.spatial.util; -import org.apache.lucene.util.SloppyMath; - /** * Reusable geo-relation utility methods */ @@ -277,252 +275,4 @@ public static boolean rectWithinPolyApprox(final double rMinLat, final double rM return !(rectCrossesPolyApprox(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLats, shapeLons, sMinLat, sMaxLat, sMinLon, sMaxLon) || !pointInPolygon(shapeLats, shapeLons, rMinLat, rMinLon)); } - - ///////////////////////// - // Circle relations - ///////////////////////// - - private static boolean rectAnyCornersInCircle(final double rMinLat, final double rMaxLat, final double rMinLon, - final double rMaxLon, final double centerLat, final double centerLon, - final double radiusMeters, final boolean approx) { - if (approx == true) { - return rectAnyCornersInCircleSloppy(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters); - } - double w = Math.abs(rMaxLon - rMinLon); - if (w <= 90.0) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; - } - // partition - w /= 4; - final double p1 = rMinLon + w; - final double p2 = p1 + w; - final double p3 = p2 + w; - - return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p1) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p1) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p2) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p2) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p3) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p3) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; - } - - private static boolean rectAnyCornersInCircleSloppy(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double centerLat, final double centerLon, final double radiusMeters) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) <= radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) <= radiusMeters; - } - - /** - * Compute whether any of the 4 corners of the rectangle (defined by min/max X/Y) are outside the circle (defined - * by centerLon, centerLat, radiusMeters) - * - * Note: exotic rectangles at the poles (e.g., those whose lat/lon distance ratios greatly deviate from 1) can not - * be determined by using distance alone. For this reason the approx flag may be set to false, in which case the - * space will be further divided to more accurately compute whether the rectangle crosses the circle - */ - private static boolean rectAnyCornersOutsideCircle(final double rMinLat, final double rMaxLat, final double rMinLon, - final double rMaxLon, final double centerLat, final double centerLon, - final double radiusMeters, final boolean approx) { - if (approx == true) { - return rectAnyCornersOutsideCircleSloppy(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters); - } - // if span is less than 70 degrees we can approximate using distance alone - if (Math.abs(rMaxLon - rMinLon) <= 70.0) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) > radiusMeters; - } - return rectCrossesOblateCircle(centerLat, centerLon, - radiusMeters, - rMinLat, rMaxLat, - rMinLon, rMaxLon); - } - - /** - * Compute whether the rectangle (defined by min/max Lat/Lon) crosses a potentially oblate circle - * - * TODO benchmark for replacing existing rectCrossesCircle. - */ - private static boolean rectCrossesOblateCircle(double centerLat, double centerLon, - double radiusMeters, - double rMinLat, double rMaxLat, - double rMinLon, double rMaxLon) { - double w = Math.abs(rMaxLon - rMinLon); - final int segs = (int)Math.ceil(w / 45.0); - w /= segs; - short i = 1; - double p1 = rMinLon; - double maxLon, midLon; - double[] pt = new double[2]; - - do { - maxLon = (i == segs) ? rMaxLon : p1 + w; - - final double d1, d2; - // short-circuit if we find a corner outside the circle - if ( (d1 = SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, p1)) > radiusMeters - || (d2 = SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, maxLon)) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, p1) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, maxLon) > radiusMeters) { - return true; - } - - // else we treat as an oblate circle by slicing the longitude space and checking the azimuthal range - // OPTIMIZATION: this is only executed for latitude values "closeTo" the poles (e.g., 88.0 > lat < -88.0) - if ( (rMaxLat > 88.0 || rMinLat < -88.0) - && (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, p1, - GeoProjectionUtils.bearingGreatCircle(rMinLat, p1, rMaxLat, p1), radiusMeters - d1, pt))[1] < rMinLat || pt[1] < rMaxLat - || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, maxLon, - GeoProjectionUtils.bearingGreatCircle(rMinLat, maxLon, rMaxLat, maxLon), radiusMeters - d2, pt))[1] < rMinLat || pt[1] < rMaxLat - || (pt = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(rMinLat, maxLon, - GeoProjectionUtils.bearingGreatCircle(rMinLat, maxLon, rMaxLat, (midLon = p1 + 0.5*(maxLon - p1))), - radiusMeters - SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, midLon), pt))[1] < rMinLat - || pt[1] < rMaxLat == false ) { - return true; - } - p1 += w; - } while (++i <= segs); - return false; - } - - private static boolean rectAnyCornersOutsideCircleSloppy(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double centerLat, final double centerLon, final double radiusMeters) { - return SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMinLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMinLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMaxLat, rMaxLon) > radiusMeters - || SloppyMath.haversinMeters(centerLat, centerLon, rMinLat, rMaxLon) > radiusMeters; - } - - /** - * Computes whether a rectangle is within a circle. Note: approx == true will be faster but less precise and may - * fail on large rectangles - */ - public static boolean rectWithinCircle(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double centerLat, final double centerLon, final double radiusMeters, - final boolean approx) { - return rectAnyCornersOutsideCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx) == false; - } - - /** - * Computes whether a rectangle crosses a circle. Note: approx == true will be faster but less precise and may - * fail on large rectangles - * - *

    NOTE: this is basic method that does not handle dateline or pole crossing. Unwrapping must be done before - * calling this method. - */ - public static boolean rectCrossesCircle(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double centerLat, final double centerLon, final double radiusMeters, - final boolean approx) { - if (approx == true) { - if (rectAnyCornersInCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { - return true; - } - } else { - if (rectAnyCornersInCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx) && - rectAnyCornersOutsideCircle(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { - return true; - } - } - if (isClosestPointOnRectWithinRange(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, radiusMeters, approx)) { - return true; - } - return false; - } - - private static boolean isClosestPointOnRectWithinRange(final double rMinLat, final double rMaxLat, - final double rMinLon, final double rMaxLon, - final double centerLat, final double centerLon, - final double radiusMeters, - final boolean approx) { - double[] closestPt = {0, 0}; - GeoDistanceUtils.closestPointOnBBox(rMinLat, rMaxLat, rMinLon, rMaxLon, centerLat, centerLon, closestPt); - boolean haverShortCut = SloppyMath.haversinMeters(centerLat, centerLon, closestPt[0], closestPt[1]) <= radiusMeters; - if (approx == true || haverShortCut == true) { - return haverShortCut; - } - double lon1 = rMinLon; - double lon2 = rMaxLon; - double lat1 = rMinLat; - double lat2 = rMaxLat; - if (closestPt[1] == rMinLon || closestPt[1] == rMaxLon) { - lon1 = closestPt[1]; - lon2 = lon1; - } else if (closestPt[0] == rMinLat || closestPt[0] == rMaxLat) { - lat1 = closestPt[0]; - lat2 = lat1; - } - - return lineCrossesSphere(lat1, lon1, 0, - lat2, lon2, 0, - centerLat, centerLon, 0, - radiusMeters); - } - - /** - * Computes whether or a 3dimensional line segment intersects or crosses a sphere - * - * @param lon1 longitudinal location of the line segment start point (in degrees) - * @param lat1 latitudinal location of the line segment start point (in degrees) - * @param alt1 altitude of the line segment start point (in degrees) - * @param lon2 longitudinal location of the line segment end point (in degrees) - * @param lat2 latitudinal location of the line segment end point (in degrees) - * @param alt2 altitude of the line segment end point (in degrees) - * @param centerLon longitudinal location of center search point (in degrees) - * @param centerLat latitudinal location of center search point (in degrees) - * @param centerAlt altitude of the center point (in meters) - * @param radiusMeters search sphere radius (in meters) - * @return whether the provided line segment is a secant of the - */ - private static boolean lineCrossesSphere(double lat1, double lon1, double alt1, - double lat2, double lon2, double alt2, - double centerLat, double centerLon, double centerAlt, - double radiusMeters) { - // convert to cartesian 3d (in meters) - double[] ecf1 = GeoProjectionUtils.llaToECF(lat1, lon1, alt1, null); - double[] ecf2 = GeoProjectionUtils.llaToECF(lat2, lon2, alt2, null); - double[] cntr = GeoProjectionUtils.llaToECF(centerLat, centerLon, centerAlt, null); - - // convert radius from arc radius to cartesian radius - double[] oneEighty = GeoProjectionUtils.pointFromLonLatBearingGreatCircle(centerLat, centerLon, 180.0d, radiusMeters, new double[3]); - GeoProjectionUtils.llaToECF(oneEighty[1], oneEighty[0], 0, oneEighty); - - radiusMeters = GeoDistanceUtils.linearDistance(oneEighty, cntr);// Math.sqrt(oneEighty[0]*cntr[0] + oneEighty[1]*cntr[1] + oneEighty[2]*cntr[2]); - - final double dX = ecf2[0] - ecf1[0]; - final double dY = ecf2[1] - ecf1[1]; - final double dZ = ecf2[2] - ecf1[2]; - final double fX = ecf1[0] - cntr[0]; - final double fY = ecf1[1] - cntr[1]; - final double fZ = ecf1[2] - cntr[2]; - - final double a = dX*dX + dY*dY + dZ*dZ; - final double b = 2 * (fX*dX + fY*dY + fZ*dZ); - final double c = (fX*fX + fY*fY + fZ*fZ) - (radiusMeters*radiusMeters); - - double discrim = (b*b)-(4*a*c); - if (discrim < 0) { - return false; - } - - discrim = StrictMath.sqrt(discrim); - final double a2 = 2*a; - final double t1 = (-b - discrim)/a2; - final double t2 = (-b + discrim)/a2; - - if ( (t1 < 0 || t1 > 1) ) { - return !(t2 < 0 || t2 > 1); - } - - return true; - } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index 935da603c685..a07ea722b716 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -16,10 +16,6 @@ */ package org.apache.lucene.spatial.util; -import java.util.ArrayList; - -import org.apache.lucene.util.SloppyMath; - import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.PI; @@ -30,11 +26,6 @@ import static org.apache.lucene.util.SloppyMath.TO_DEGREES; import static org.apache.lucene.util.SloppyMath.TO_RADIANS; import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.MAX_LAT_RADIANS; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.MAX_LON_RADIANS; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.MIN_LAT_RADIANS; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.MIN_LON_RADIANS; -import static org.apache.lucene.spatial.util.GeoProjectionUtils.SEMIMAJOR_AXIS; /** * Basic reusable geo-spatial utility methods @@ -53,6 +44,19 @@ public final class GeoUtils { /** Maximum latitude value. */ public static final double MAX_LAT_INCL = 90.0D; + + /** min longitude value in radians */ + public static final double MIN_LON_RADIANS = TO_RADIANS * MIN_LON_INCL; + /** min latitude value in radians */ + public static final double MIN_LAT_RADIANS = TO_RADIANS * MIN_LAT_INCL; + /** max longitude value in radians */ + public static final double MAX_LON_RADIANS = TO_RADIANS * MAX_LON_INCL; + /** max latitude value in radians */ + public static final double MAX_LAT_RADIANS = TO_RADIANS * MAX_LAT_INCL; + + // WGS84 earth-ellipsoid parameters + /** major (a) axis in meters */ + public static final double SEMIMAJOR_AXIS = 6_378_137; // [m] // No instance: private GeoUtils() { @@ -153,7 +157,7 @@ public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { // some sloppyish stuff, do we really need this to be done in a sloppy way? // unless it is performance sensitive, we should try to remove. - static final double PIO2 = Math.PI / 2D; + private static final double PIO2 = Math.PI / 2D; /** * Returns the trigonometric sine of an angle converted as a cos operation. @@ -169,26 +173,8 @@ public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { * @see Math#sin(double) */ // TODO: deprecate/remove this? at least its no longer public. - static double sloppySin(double a) { + private static double sloppySin(double a) { return cos(a - PIO2); } - - /** - * Returns the trigonometric tangent of an angle converted in terms of a sin/cos operation. - *

    - * Note that this is probably not quite right (untested) - *

    - * Special cases: - *

      - *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. - *
    - * @param a an angle, in radians. - * @return the tangent of the argument. - * @see Math#sin(double) aand Math#cos(double) - */ - // TODO: deprecate/remove this? at least its no longer public. - static double sloppyTan(double a) { - return sloppySin(a) / cos(a); - } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 53ff25c544c2..747cd1fdc2cc 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -22,23 +22,30 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; +import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.util.SloppyMath; - -import static org.apache.lucene.spatial.util.GeoDistanceUtils.DISTANCE_PCT_ERR; /** * random testing for GeoPoint query logic * * @lucene.experimental */ - public class TestGeoPointQuery extends BaseGeoPointTestCase { @Override - protected boolean forceSmall() { - return false; + protected double maxRadius(double latitude, double longitude) { + // TODO: clean this up + return GeoDistanceUtils.maxRadialDistanceMeters(latitude, longitude); + } + + @Override + protected double quantizeLat(double lat) { + return GeoEncodingUtils.mortonUnhashLat(GeoEncodingUtils.mortonHash(lat, 0)); + } + + @Override + protected double quantizeLon(double lon) { + return GeoEncodingUtils.mortonUnhashLon(GeoEncodingUtils.mortonHash(0, lon)); } @Override @@ -58,7 +65,9 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo @Override protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, radiusMeters); + // LUCENE-7126: currently not valid for multi-valued documents, because it rewrites to a BooleanQuery! + // return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, radiusMeters); + return null; } @Override @@ -66,63 +75,4 @@ protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, lats, lons); } - @Override - protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - if (GeoEncodingUtils.compare(pointLon, rect.minLon) == 0.0 || - GeoEncodingUtils.compare(pointLon, rect.maxLon) == 0.0 || - GeoEncodingUtils.compare(pointLat, rect.minLat) == 0.0 || - GeoEncodingUtils.compare(pointLat, rect.maxLat) == 0.0) { - // Point is very close to rect boundary - return null; - } - - if (rect.minLon < rect.maxLon) { - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); - } else { - // Rect crosses dateline: - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) - || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); - } - } - - @Override - protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - return rectContainsPoint(rect, pointLat, pointLon); - } - - @Override - protected Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { - if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { - return null; - } else { - return SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon) <= radiusMeters; - } - } - - @Override - protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { - if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, minRadiusMeters) - || radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { - return null; - } else { - final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - return d >= minRadiusMeters && d <= radiusMeters; - } - } - - private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, double ptLon, double ptLat, - final double radius) { - final long hashedCntr = GeoEncodingUtils.mortonHash(centerLat, centerLon); - centerLon = GeoEncodingUtils.mortonUnhashLon(hashedCntr); - centerLat = GeoEncodingUtils.mortonUnhashLat(hashedCntr); - final long hashedPt = GeoEncodingUtils.mortonHash(ptLat, ptLon); - ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); - ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); - - double ptDistance = SloppyMath.haversinMeters(centerLat, centerLon, ptLat, ptLon); - double delta = StrictMath.abs(ptDistance - radius); - - // if its within the distance error then it can be wrong - return delta < (ptDistance*DISTANCE_PCT_ERR); - } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index 90bfb107e347..73b30826cfc5 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -22,11 +22,8 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; +import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.util.SloppyMath; - -import static org.apache.lucene.spatial.util.GeoDistanceUtils.DISTANCE_PCT_ERR; /** * random testing for GeoPoint query logic (with deprecated numeric encoding) @@ -36,8 +33,19 @@ public class TestLegacyGeoPointQuery extends BaseGeoPointTestCase { @Override - protected boolean forceSmall() { - return false; + protected double maxRadius(double latitude, double longitude) { + // TODO: clean this up + return GeoDistanceUtils.maxRadialDistanceMeters(latitude, longitude); + } + + @Override + protected double quantizeLat(double lat) { + return GeoEncodingUtils.mortonUnhashLat(GeoEncodingUtils.mortonHash(lat, 0)); + } + + @Override + protected double quantizeLon(double lon) { + return GeoEncodingUtils.mortonUnhashLon(GeoEncodingUtils.mortonHash(0, lon)); } @Override @@ -57,7 +65,9 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo @Override protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, minRadiusMeters, radiusMeters); + // LUCENE-7126: currently not valid for multi-valued documents, because it rewrites to a BooleanQuery! + // return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, minRadiusMeters, radiusMeters); + return null; } @Override @@ -65,63 +75,14 @@ protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lats, lons); } + // legacy encoding is too slow somehow for this random test, its not up to the task. @Override - protected Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - if (GeoEncodingUtils.compare(pointLon, rect.minLon) == 0.0 || - GeoEncodingUtils.compare(pointLon, rect.maxLon) == 0.0 || - GeoEncodingUtils.compare(pointLat, rect.minLat) == 0.0 || - GeoEncodingUtils.compare(pointLat, rect.maxLat) == 0.0) { - // Point is very close to rect boundary - return null; - } - - if (rect.minLon < rect.maxLon) { - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); - } else { - // Rect crosses dateline: - return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) - || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); - } - } - - @Override - protected Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - return rectContainsPoint(rect, pointLat, pointLon); - } - - @Override - protected Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { - if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { - return null; - } else { - return SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon) <= radiusMeters; - } + public void testRandomDistance() throws Exception { + assumeTrue("legacy encoding is too slow/hangs on this test", false); } @Override - protected Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { - if (radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, minRadiusMeters) - || radiusQueryCanBeWrong(centerLat, centerLon, pointLon, pointLat, radiusMeters)) { - return null; - } else { - final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - return d >= minRadiusMeters && d <= radiusMeters; - } - } - - private static boolean radiusQueryCanBeWrong(double centerLat, double centerLon, double ptLon, double ptLat, - final double radius) { - final long hashedCntr = GeoEncodingUtils.mortonHash(centerLat, centerLon); - centerLon = GeoEncodingUtils.mortonUnhashLon(hashedCntr); - centerLat = GeoEncodingUtils.mortonUnhashLat(hashedCntr); - final long hashedPt = GeoEncodingUtils.mortonHash(ptLat, ptLon); - ptLon = GeoEncodingUtils.mortonUnhashLon(hashedPt); - ptLat = GeoEncodingUtils.mortonUnhashLat(hashedPt); - - double ptDistance = SloppyMath.haversinMeters(centerLat, centerLon, ptLat, ptLon); - double delta = StrictMath.abs(ptDistance - radius); - - // if its within the distance error then it can be wrong - return delta < (ptDistance*DISTANCE_PCT_ERR); + public void testRandomDistanceHuge() throws Exception { + assumeTrue("legacy encoding is too slow/hangs on this test", false); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index a4f70d911e30..6f3a73cb822d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -21,6 +21,7 @@ import java.text.DecimalFormatSymbols; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -28,9 +29,16 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.codecs.lucene60.Lucene60PointsReader; +import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -39,16 +47,22 @@ import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.bkd.BKDWriter; import org.junit.BeforeClass; // TODO: cutover TestGeoUtils too? @@ -72,17 +86,8 @@ public static void beforeClassBase() throws Exception { originLat = GeoUtils.normalizeLat(GeoUtils.MIN_LAT_INCL + latRange + (GeoUtils.MAX_LAT_INCL - GeoUtils.MIN_LAT_INCL - 2 * latRange) * random().nextDouble()); } - /** Return true when testing on a non-small region may be too slow (GeoPoint*Query) */ - protected boolean forceSmall() { - return false; - } - // A particularly tricky adversary for BKD tree: public void testSamePointManyTimes() throws Exception { - - // For GeoPointQuery, only run this test nightly: - assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false); - int numPoints = atLeast(1000); boolean small = random().nextBoolean(); @@ -100,12 +105,8 @@ public void testSamePointManyTimes() throws Exception { } public void testAllLatEqual() throws Exception { - - // For GeoPointQuery, only run this test nightly: - assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false); - int numPoints = atLeast(10000); - boolean small = forceSmall() || random().nextBoolean(); + boolean small = random().nextBoolean(); double lat = randomLat(small); double[] lats = new double[numPoints]; double[] lons = new double[numPoints]; @@ -151,12 +152,8 @@ public void testAllLatEqual() throws Exception { } public void testAllLonEqual() throws Exception { - - // For GeoPointQuery, only run this test nightly: - assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false); - int numPoints = atLeast(10000); - boolean small = forceSmall() || random().nextBoolean(); + boolean small = random().nextBoolean(); double theLon = randomLon(small); double[] lats = new double[numPoints]; double[] lons = new double[numPoints]; @@ -204,10 +201,6 @@ public void testAllLonEqual() throws Exception { } public void testMultiValued() throws Exception { - - // For GeoPointQuery, only run this test nightly: - assumeTrue("GeoPoint*Query is too slow otherwise", TEST_NIGHTLY || forceSmall() == false); - int numPoints = atLeast(10000); // Every doc has 2 points: double[] lats = new double[2*numPoints]; @@ -289,19 +282,10 @@ public void collect(int doc) { double latDoc2 = lats[2*docID+1]; double lonDoc2 = lons[2*docID+1]; - Boolean result1 = rectContainsPoint(rect, latDoc1, lonDoc1); - if (result1 == null) { - // borderline case: cannot test - continue; - } - - Boolean result2 = rectContainsPoint(rect, latDoc2, lonDoc2); - if (result2 == null) { - // borderline case: cannot test - continue; - } + boolean result1 = rectContainsPoint(rect, latDoc1, lonDoc1); + boolean result2 = rectContainsPoint(rect, latDoc2, lonDoc2); - boolean expected = result1 == Boolean.TRUE || result2 == Boolean.TRUE; + boolean expected = result1 || result2; if (hits.get(docID) != expected) { String id = s.doc(docID).get("id"); @@ -447,6 +431,10 @@ protected double quantizeLat(double lat) { protected double quantizeLon(double lon) { return lon; } + + protected double maxRadius(double latitude, double longitude) { + return 50000000D; // bigger than earth, shouldnt matter + } protected GeoRect randomRect(boolean small, boolean canCrossDateLine) { double lat0 = randomLat(small); @@ -482,16 +470,44 @@ protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { protected abstract Query newPolygonQuery(String field, double[] lats, double[] lons); - /** Returns null if it's borderline case */ - protected abstract Boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon); + static final boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { + assert Double.isNaN(pointLat) == false; - /** Returns null if it's borderline case */ - protected abstract Boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon); + if (rect.minLon < rect.maxLon) { + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); + } else { + // Rect crosses dateline: + return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, -180.0, rect.maxLon) + || GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, 180.0); + } + } + + static final boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { + // TODO write better random polygon tests + + // note: logic must be slightly different than rectContainsPoint, to satisfy + // insideness for cases exactly on boundaries. + + assert Double.isNaN(pointLat) == false; + assert rect.crossesDateline() == false; + double polyLats[] = new double[] { rect.minLat, rect.maxLat, rect.maxLat, rect.minLat, rect.minLat }; + double polyLons[] = new double[] { rect.minLon, rect.minLon, rect.maxLon, rect.maxLon, rect.minLon }; + + // TODO: separately test this method is 100% correct, here treat it like a black box (like haversin) + return GeoRelationUtils.pointInPolygon(polyLats, polyLons, pointLat, pointLon); + } - /** Returns null if it's borderline case */ - protected abstract Boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon); + static final boolean circleContainsPoint(double centerLat, double centerLon, double radiusMeters, double pointLat, double pointLon) { + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); + boolean result = distanceMeters <= radiusMeters; + //System.out.println(" shouldMatch? centerLon=" + centerLon + " centerLat=" + centerLat + " pointLon=" + pointLon + " pointLat=" + pointLat + " result=" + result + " distanceMeters=" + (distanceKM * 1000)); + return result; + } - protected abstract Boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon); + static final boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { + final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); + return d >= minRadiusMeters && d <= radiusMeters; + } private static abstract class VerifyHits { @@ -525,7 +541,7 @@ public void collect(int doc) { for(int docID=0;docID hits, PrintWriter log, Cell root, - double centerLon, double centerLat, double radiusMeters, - double[] docLons, double[] docLats) { - - if (VERBOSE) { - log.println(" root cell: " + root); - } - - List queue = new ArrayList<>(); - queue.add(root); - - int recurseDepth = RandomInts.randomIntBetween(random(), 5, 15); - - while (queue.size() > 0) { - Cell cell = queue.get(queue.size()-1); - queue.remove(queue.size()-1); - if (VERBOSE) { - log.println(" cycle: " + cell + " queue.size()=" + queue.size()); - } - - if (random().nextInt(10) == 7 || cell.splitCount > recurseDepth) { - if (VERBOSE) { - log.println(" leaf"); - } - // Leaf cell: brute force check all docs that fall within this cell: - for(int docID=0;docID bbox.maxLon) { - // unwrap left - unwrappedLon += -360.0D; - } - findMatches(hits, log, - new Cell(null, - -180, bbox.minLat, - bbox.maxLon, bbox.maxLat, - 0), - unwrappedLon, centerLat, radiusMeters, docLons, docLats); - log.println(" circle crosses dateline; now right query"); - if (unwrappedLon < bbox.maxLon) { - // unwrap right - unwrappedLon += 360.0D; - } - findMatches(hits, log, - new Cell(null, - bbox.minLon, bbox.minLat, - 180, bbox.maxLat, - 0), - unwrappedLon, centerLat, radiusMeters, docLons, docLats); - } else { - // Start with the root cell that fully contains the shape: - findMatches(hits, log, - new Cell(null, - bbox.minLon, bbox.minLat, - bbox.maxLon, bbox.maxLat, - 0), - centerLon, centerLat, radiusMeters, - docLons, docLats); - } - - if (VERBOSE) { - log.println(" " + hits.size() + " hits"); - } - - int failCount = 0; - - // Done matching, now verify: - for(int docID=0;docID Date: Tue, 22 Mar 2016 17:47:09 -0400 Subject: [PATCH 0193/1113] LUCENE-7130: fold optimizations from LatLonPoint to GeoPointField --- .../document/LatLonPointDistanceQuery.java | 8 ++--- .../search/GeoPointDistanceQueryImpl.java | 31 ++++++++++++++++--- .../search/GeoPointPrefixTermsEnum.java | 4 ++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 5e7a04e76572..bba92b3e854c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -104,12 +104,12 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // compute a maximum partial haversin: unless our box is crazy, we can use this bound // to reject edge cases faster in matches() - final double minPartialDistance; + final double maxPartialDistance; if (box.maxLon - longitude < 90 && longitude - box.minLon < 90) { - minPartialDistance = Math.max(SloppyMath.haversinSortKey(latitude, longitude, latitude, box.maxLon), + maxPartialDistance = Math.max(SloppyMath.haversinSortKey(latitude, longitude, latitude, box.maxLon), SloppyMath.haversinSortKey(latitude, longitude, box.maxLat, longitude)); } else { - minPartialDistance = Double.POSITIVE_INFINITY; + maxPartialDistance = Double.POSITIVE_INFINITY; } return new ConstantScoreWeight(this) { @@ -235,7 +235,7 @@ public boolean matches() throws IOException { // first check the partial distance, if its more than that, it can't be <= radiusMeters double h1 = SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude); - if (h1 > minPartialDistance) { + if (h1 > maxPartialDistance) { continue; } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index d4272a34bf11..243f33e26ddc 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -28,12 +28,24 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { private final GeoPointDistanceQuery distanceQuery; private final double centerLon; - + + // optimization, maximum partial haversin needed to be a candidate + private final double maxPartialDistance; + GeoPointDistanceQueryImpl(final String field, final TermEncoding termEncoding, final GeoPointDistanceQuery q, final double centerLonUnwrapped, final GeoRect bbox) { super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); distanceQuery = q; centerLon = centerLonUnwrapped; + + // unless our box is crazy, we can use this bound + // to reject edge cases faster in postFilter() + if (bbox.maxLon - centerLon < 90 && centerLon - bbox.minLon < 90) { + maxPartialDistance = Math.max(SloppyMath.haversinSortKey(distanceQuery.centerLat, centerLon, distanceQuery.centerLat, bbox.maxLon), + SloppyMath.haversinSortKey(distanceQuery.centerLat, centerLon, bbox.maxLat, centerLon)); + } else { + maxPartialDistance = Double.POSITIVE_INFINITY; + } } @Override @@ -65,8 +77,7 @@ protected boolean cellCrosses(final double minLat, final double maxLat, final do @Override protected boolean cellWithin(final double minLat, final double maxLat, final double minLon, final double maxLon) { - // TODO: we call cellCrosses because of how the termsEnum logic works, helps us avoid some haversin() calls here. - if (cellCrosses(minLat, maxLat, minLon, maxLon) && maxLon - centerLon < 90 && centerLon - minLon < 90 && + if (maxLon - centerLon < 90 && centerLon - minLon < 90 && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, minLon) <= distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, maxLon) <= distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, minLon) <= distanceQuery.radiusMeters && @@ -90,7 +101,19 @@ protected boolean cellIntersectsShape(final double minLat, final double maxLat, */ @Override protected boolean postFilter(final double lat, final double lon) { - return SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, lat, lon) <= distanceQuery.radiusMeters; + // check bbox + if (lat < minLat || lat > maxLat || lon < minLon || lon > maxLon) { + return false; + } + + // first check the partial distance, if its more than that, it can't be <= radiusMeters + double h1 = SloppyMath.haversinSortKey(distanceQuery.centerLat, centerLon, lat, lon); + if (h1 > maxPartialDistance) { + return false; + } + + // fully confirm with part 2: + return SloppyMath.haversinMeters(h1) <= distanceQuery.radiusMeters; } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java index eeb58b2e3480..f511392b2eed 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointPrefixTermsEnum.java @@ -99,8 +99,10 @@ private void nextRelation() { maxLon = mortonUnhashLon(currEnd); maxLat = mortonUnhashLat(currEnd); + isWithin = false; // within or a boundary - if ((isWithin = within(minLat, maxLat, minLon, maxLon) == true) || boundary(minLat, maxLat, minLon, maxLon) == true) { + if (boundary(minLat, maxLat, minLon, maxLon) == true) { + isWithin = within(minLat, maxLat, minLon, maxLon); final int m; if (isWithin == false || (m = shift % GeoPointField.PRECISION_STEP) == 0) { setNextRange(isWithin == false); From f2234dccb35ac11e1028890b20b61cdd2c9b9bf7 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 22 Mar 2016 18:21:24 -0400 Subject: [PATCH 0194/1113] LUCENE-7128: fix a few more lon/lat places; remove more dead code --- .../document/GeoPointTokenStream.java | 2 +- .../geopoint/search/GeoBoundingBox.java | 53 -------------- .../search/GeoPointDistanceRangeQuery.java | 2 +- .../search/GeoPointMultiTermQuery.java | 2 +- .../lucene/spatial/util/GeoDistanceUtils.java | 72 ------------------- .../apache/lucene/spatial/util/GeoUtils.java | 38 ++-------- .../spatial/util/BaseGeoPointTestCase.java | 32 +++++++-- .../lucene/spatial/util/TestGeoUtils.java | 27 +++---- 8 files changed, 46 insertions(+), 182 deletions(-) delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoBoundingBox.java diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointTokenStream.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointTokenStream.java index 66216abe1297..c374c4e8b7f1 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointTokenStream.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointTokenStream.java @@ -60,7 +60,7 @@ * document.add(geoPointField); * * // query by bounding box (default uses TermEncoding.PREFIX) - * Query q = new GeoPointInBBoxQuery(fieldName1, minLon, minLat, maxLon, maxLat); + * Query q = new GeoPointInBBoxQuery(fieldName1, minLat, maxLat, minLon, maxLon); * * // using numeric terms * geoPointField = new GeoPointField(fieldName2, lat, lon, GeoPointField.NUMERIC_TYPE_NOT_STORED); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoBoundingBox.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoBoundingBox.java deleted file mode 100644 index 8f30f6049197..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoBoundingBox.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.geopoint.search; - -import org.apache.lucene.spatial.util.GeoUtils; - -/** NOTE: package private; just used so {@link GeoPointInPolygonQuery} can communicate its bounding box to {@link GeoPointInBBoxQuery}. */ -class GeoBoundingBox { - /** minimum longitude value (in degrees) */ - public final double minLon; - /** minimum latitude value (in degrees) */ - public final double maxLon; - /** maximum longitude value (in degrees) */ - public final double minLat; - /** maximum latitude value (in degrees) */ - public final double maxLat; - - /** - * Constructs a bounding box by first validating the provided latitude and longitude coordinates - */ - public GeoBoundingBox(double minLon, double maxLon, double minLat, double maxLat) { - if (GeoUtils.isValidLon(minLon) == false) { - throw new IllegalArgumentException("invalid minLon " + minLon); - } - if (GeoUtils.isValidLon(maxLon) == false) { - throw new IllegalArgumentException("invalid maxLon " + minLon); - } - if (GeoUtils.isValidLat(minLat) == false) { - throw new IllegalArgumentException("invalid minLat " + minLat); - } - if (GeoUtils.isValidLat(maxLat) == false) { - throw new IllegalArgumentException("invalid maxLat " + minLat); - } - this.minLon = minLon; - this.maxLon = maxLon; - this.minLat = minLat; - this.maxLat = maxLat; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java index bdfbb882fc49..5cc778a5e7c7 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java @@ -30,7 +30,7 @@ * @lucene.experimental */ public final class GeoPointDistanceRangeQuery extends GeoPointDistanceQuery { - /** minimum distance range (in meters) from lon, lat center location, maximum is inherited */ + /** minimum distance range (in meters) from lat, lon center location, maximum is inherited */ protected final double minRadiusMeters; /** diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java index a63c4d6d7542..c23ceb8244b9 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java @@ -98,7 +98,7 @@ protected TermsEnum getTermsEnum(final Terms terms, AttributeSource atts) throws * Computes the maximum shift based on the diagonal distance of the bounding box */ protected short computeMaxShift() { - // in this case a factor of 4 brings the detail level to ~0.002/0.001 degrees lon/lat respectively (or ~222m/111m) + // in this case a factor of 4 brings the detail level to ~0.001/0.002 degrees lat/lon respectively (or ~111m/222m) final short shiftFactor; // compute diagonal distance diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java index 32a16cdcc5ae..1a412aca03e1 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java @@ -29,20 +29,6 @@ public class GeoDistanceUtils { private GeoDistanceUtils() { } - /** - * Computes distance between two points in a cartesian (x, y, {z - optional}) coordinate system - */ - public static double linearDistance(double[] pt1, double[] pt2) { - assert pt1 != null && pt2 != null && pt1.length == pt2.length && pt1.length > 1; - final double d0 = pt1[0] - pt2[0]; - final double d1 = pt1[1] - pt2[1]; - if (pt1.length == 3) { - final double d2 = pt1[2] - pt2[2]; - return Math.sqrt(d0*d0 + d1*d1 + d2*d2); - } - return Math.sqrt(d0*d0 + d1*d1); - } - /** * Compute the inverse haversine to determine distance in degrees longitude for provided distance in meters * @param lat latitude to compute delta degrees lon @@ -65,46 +51,6 @@ public static double distanceToDegreesLon(double lat, double distance) { return StrictMath.toDegrees(StrictMath.acos(1-((2d*h)/(cLat*cLat)))); } - /** - * Finds the closest point within a rectangle (defined by rMinX, rMinY, rMaxX, rMaxY) to the given (lon, lat) point - * the result is provided in closestPt (lat, lon). When the point is outside the rectangle, the closest point is on an edge - * or corner of the rectangle; else, the closest point is the point itself. - */ - public static void closestPointOnBBox(final double rMinY, final double rMaxY, final double rMinX, final double rMaxX, - final double lat, final double lon, double[] closestPt) { - assert closestPt != null && closestPt.length == 2; - - closestPt[0] = 0; - closestPt[1] = 0; - - boolean xSet = true; - boolean ySet = true; - - if (lon > rMaxX) { - closestPt[1] = rMaxX; - } else if (lon < rMinX) { - closestPt[1] = rMinX; - } else { - xSet = false; - } - - if (lat > rMaxY) { - closestPt[0] = rMaxY; - } else if (lat < rMinY) { - closestPt[0] = rMinY; - } else { - ySet = false; - } - - if (closestPt[0] == 0 && ySet == false) { - closestPt[0] = lat; - } - - if (closestPt[1] == 0 && xSet == false) { - closestPt[1] = lon; - } - } - /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */ public static double maxRadialDistanceMeters(final double centerLat, final double centerLon) { if (Math.abs(centerLat) == GeoUtils.MAX_LAT_INCL) { @@ -112,22 +58,4 @@ public static double maxRadialDistanceMeters(final double centerLat, final doubl } return SloppyMath.haversinMeters(centerLat, centerLon, centerLat, (GeoUtils.MAX_LON_INCL + centerLon) % 360); } - - /** - * Compute the inverse haversine to determine distance in degrees longitude for provided distance in meters - * @param lat latitude to compute delta degrees lon - * @param distance distance in meters to convert to degrees lon - * @return Sloppy distance in degrees longitude for provided distance in meters - */ - public static double distanceToDegreesLat(double lat, double distance) { - // get the diameter at the latitude - final double diameter = 2 * GeoUtils.SEMIMAJOR_AXIS; - - // compute inverse haversine - double a = StrictMath.sin(distance/diameter); - double h = StrictMath.min(1, a); - h *= h; - - return StrictMath.toDegrees(StrictMath.acos(1-(2d*h))); - } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index a07ea722b716..08c36155023b 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -72,30 +72,6 @@ public static boolean isValidLon(double lon) { return Double.isNaN(lon) == false && lon >= MIN_LON_INCL && lon <= MAX_LON_INCL; } - /** Puts longitude in range of -180 to +180. */ - public static double normalizeLon(double lon_deg) { - if (lon_deg >= -180 && lon_deg <= 180) { - return lon_deg; //common case, and avoids slight double precision shifting - } - double off = (lon_deg + 180) % 360; - if (off < 0) { - return 180 + off; - } else if (off == 0 && lon_deg > 0) { - return 180; - } else { - return -180 + off; - } - } - - /** Puts latitude in range of -90 to 90. */ - public static double normalizeLat(double lat_deg) { - if (lat_deg >= -90 && lat_deg <= 90) { - return lat_deg; //common case, and avoids slight double precision shifting - } - double off = abs((lat_deg + 90) % 360); - return (off <= 180 ? off : 360-off) - 90; - } - /** Compute Bounding Box for a circle using WGS-84 parameters */ public static GeoRect circleToBBox(final double centerLat, final double centerLon, final double radiusMeters) { final double radLat = TO_RADIANS * centerLat; @@ -129,8 +105,8 @@ public static GeoRect circleToBBox(final double centerLat, final double centerLo /** Compute Bounding Box for a polygon using WGS-84 parameters */ public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { - if (polyLons.length != polyLats.length) { - throw new IllegalArgumentException("polyLons and polyLats must be equal length"); + if (polyLats.length != polyLons.length) { + throw new IllegalArgumentException("polyLats and polyLons must be equal length"); } double minLon = Double.POSITIVE_INFINITY; @@ -139,16 +115,16 @@ public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { double maxLat = Double.NEGATIVE_INFINITY; for (int i=0;i Date: Wed, 23 Mar 2016 04:42:15 -0400 Subject: [PATCH 0195/1113] GeoPointField now validates incoming lat/lon --- .../spatial/geopoint/document/GeoPointField.java | 13 +++++++++++-- .../geopoint/search/TestGeoPointField.java | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java index 688315ac1348..00db23a8d0f5 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java @@ -16,13 +16,14 @@ */ package org.apache.lucene.spatial.geopoint.document; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.util.GeoUtils; /** *

    @@ -162,6 +163,14 @@ public GeoPointField(String name, double lat, double lon, TermEncoding termEncod public GeoPointField(String name, double lat, double lon, FieldType type) { super(name, type); + if (GeoUtils.isValidLat(lat) == false) { + throw new IllegalArgumentException("invalid lat=" + lat + " for field \"" + name + "\""); + } + + if (GeoUtils.isValidLon(lon) == false) { + throw new IllegalArgumentException("invalid lon=" + lon + " for field \"" + name + "\""); + } + // field must be indexed // todo does it make sense here to provide the ability to store a GeoPointField but not index? if (type.indexOptions() == IndexOptions.NONE && type.stored() == false) { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java index 41136b954133..123769ee984e 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java @@ -237,4 +237,19 @@ public void testMaxDistanceRangeQuery() throws Exception { TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); } + + public void testInvalidLatLon() throws Exception { + IllegalArgumentException e; + e= expectThrows(IllegalArgumentException.class, + () -> { + new GeoPointField("field", 180.0, 0.0, Field.Store.NO); + }); + assertEquals("invalid lat=180.0 for field \"field\"", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, + () -> { + new GeoPointField("field", 0.0, 190.0, Field.Store.NO); + }); + assertEquals("invalid lon=190.0 for field \"field\"", e.getMessage()); + } } From 855763b063a2fc2e5e3f58f4738f85b41b488f8f Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 23 Mar 2016 06:20:26 -0400 Subject: [PATCH 0196/1113] disable query cache when this test uses threads, so seeds reliably reproduce --- .../lucene/spatial/util/BaseGeoPointTestCase.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index f0334b2a9f8c..0efe7d6165c7 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -656,6 +656,13 @@ protected void verify(boolean small, double[] lats, double[] lons, boolean useTh // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat: IndexSearcher s = newSearcher(r, false); + if (useThreads) { + // We must disable query cache otherwise test seed may not reproduce since different + // threads may or may not get a cache hit or miss depending on order the JVM + // schedules the threads: + s.setQueryCache(null); + } + // Make sure queries are thread safe: int numThreads; if (useThreads) { @@ -765,8 +772,8 @@ protected boolean shouldMatch(double pointLat, double pointLon) { @Override protected void describe(int docID, double pointLat, double pointLon) { double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - System.out.println(" docID=" + docID + " centerLon=" + centerLon + " centerLat=" + centerLat - + " pointLon=" + pointLon + " pointLat=" + pointLat + " distanceMeters=" + distanceMeters + System.out.println(" docID=" + docID + " centerLat=" + centerLat + " centerLon=" + centerLon + + " pointLat=" + pointLat + " pointLon=" + pointLon + " distanceMeters=" + distanceMeters + " vs" + ((rangeQuery == true) ? " minRadiusMeters=" + minRadiusMeters : "") + " radiusMeters=" + radiusMeters); } }; From 82a537795ff9e85cd6834d6f46c41851b435bf7c Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 23 Mar 2016 11:56:00 -0400 Subject: [PATCH 0197/1113] SOLR-8886: fix TrieField.toObject(IndexableField) for docValues --- solr/CHANGES.txt | 3 ++ .../org/apache/solr/schema/TrieField.java | 40 ++++++++++++++++++- .../collection1/conf/schema-docValues.xml | 11 +++++ .../org/apache/solr/schema/DocValuesTest.java | 22 ++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 634f218dbb79..7e0fbe477ed2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -345,6 +345,9 @@ Bug Fixes numeric field. For more complex functions, FunctionValues.exists() must also return true for the document to match. (yonik) +* SOLR-8886: Fix TrieField.toObject(IndexableField) to work for field with docValues + enabled. (yonik) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index 1580b0037edc..506d8ad9b9ac 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -33,6 +33,7 @@ import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; @@ -121,11 +122,46 @@ protected void init(IndexSchema schema, Map args) { public Object toObject(IndexableField f) { final Number val = f.numericValue(); if (val != null) { + + if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.NUMERIC ) { + long bits = val.longValue(); + switch (type) { + case INTEGER: + return (int)bits; + case FLOAT: + return Float.intBitsToFloat((int)bits); + case LONG: + return bits; + case DOUBLE: + return Double.longBitsToDouble(bits); + case DATE: + return new Date(bits); + default: + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); + } + } + + // normal stored case return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val; } else { - // the old BinaryField encoding is no longer supported - return badFieldString(f); + // multi-valued numeric docValues currently use SortedSet on the indexed terms. + BytesRef term = f.binaryValue(); + switch (type) { + case INTEGER: + return LegacyNumericUtils.prefixCodedToInt(term); + case FLOAT: + return NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(term)); + case LONG: + return LegacyNumericUtils.prefixCodedToLong(term); + case DOUBLE: + return NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(term)); + case DATE: + return new Date(LegacyNumericUtils.prefixCodedToLong(term)); + default: + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); + } } + } @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml index cbbdf6ed0ae5..680ac0421036 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -67,6 +67,17 @@ + + + + + + + + + + + id diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java index f1d0196c2262..2537c8f0f6d2 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -16,6 +16,7 @@ */ package org.apache.solr.schema; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfos; @@ -33,6 +34,7 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.function.Function; import java.util.function.Supplier; @@ -102,12 +104,32 @@ public void testDocValues() throws IOException { values = longDv.getType().getValueSource(longDv, null).getValues(null, searcher.getLeafReader().leaves().get(0)); assertEquals(4L, values.longVal(0)); assertEquals(4L, values.objectVal(0)); + + // check reversability of created fields + tstToObj(schema.getField("floatdv"), -1.5f); + tstToObj(schema.getField("floatdvs"), -1.5f); + tstToObj(schema.getField("doubledv"), -1.5d); + tstToObj(schema.getField("doubledvs"), -1.5d); + tstToObj(schema.getField("intdv"), -7); + tstToObj(schema.getField("intdvs"), -7); + tstToObj(schema.getField("longdv"), -11L); + tstToObj(schema.getField("longdvs"), -11L); + tstToObj(schema.getField("datedv"), new Date(1000)); + tstToObj(schema.getField("datedvs"), new Date(1000)); + } finally { searcherRef.decref(); } } } + private void tstToObj(SchemaField sf, Object o) { + List fields = sf.createFields(o, 1.0f); + for (IndexableField field : fields) { + assertEquals( sf.getType().toObject(field), o); + } + } + public void testDocValuesSorting() { assertU(adoc("id", "1", "floatdv", "2", "intdv", "3", "doubledv", "4", "longdv", "5", "datedv", "1995-12-31T23:59:59.999Z", "stringdv", "b")); assertU(adoc("id", "2", "floatdv", "5", "intdv", "4", "doubledv", "3", "longdv", "2", "datedv", "1997-12-31T23:59:59.999Z", "stringdv", "a")); From 39270d9df8959067a18edfe7c1e3cdb24adce543 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Wed, 23 Mar 2016 15:49:21 -0500 Subject: [PATCH 0198/1113] SOLR-8819: Implement DatabaseMetaDataImpl getTables() and fix getSchemas(). (Trey Cahill, Joel Bernstein, Kevin Risden) --- solr/CHANGES.txt | 3 + .../org/apache/solr/handler/SQLHandler.java | 79 ++++++++++++++----- .../apache/solr/handler/TestSQLHandler.java | 73 ++++++++++++++++- .../client/solrj/io/sql/ConnectionImpl.java | 10 ++- .../solrj/io/sql/DatabaseMetaDataImpl.java | 2 +- .../client/solrj/io/sql/StatementImpl.java | 4 +- .../solr/client/solrj/io/sql/JdbcTest.java | 21 +++-- 7 files changed, 156 insertions(+), 36 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7e0fbe477ed2..d7b30949bd94 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -495,6 +495,9 @@ Other Changes * SOLR-8740: docValues are now enabled by default for most non-text (string, date, and numeric) fields in the schema templates. (yonik) +* SOLR-8819: Implement DatabaseMetaDataImpl getTables() and fix getSchemas(). (Trey Cahill, + Joel Bernstein, Kevin Risden) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java index 11fee9358478..28b033ac15d8 100644 --- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java @@ -52,6 +52,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.client.solrj.io.stream.metrics.*; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -173,17 +174,11 @@ public static TupleStream parse(String sql, TupleStream sqlStream = null; if(sqlVistor.table.toUpperCase(Locale.ROOT).contains("_CATALOGS_")) { - if (!sqlVistor.fields.contains("TABLE_CAT")) { - throw new IOException("When querying _CATALOGS_, fields must contain column TABLE_CAT"); - } - - sqlStream = new CatalogsStream(defaultZkhost); + sqlStream = new SelectStream(new CatalogsStream(defaultZkhost), sqlVistor.columnAliases); } else if(sqlVistor.table.toUpperCase(Locale.ROOT).contains("_SCHEMAS_")) { - if (!sqlVistor.fields.contains("TABLE_SCHEM") || !sqlVistor.fields.contains("TABLE_CATALOG")) { - throw new IOException("When querying _SCHEMAS_, fields must contain both TABLE_SCHEM and TABLE_CATALOG"); - } - - sqlStream = new SchemasStream(defaultZkhost); + sqlStream = new SelectStream(new SchemasStream(defaultZkhost), sqlVistor.columnAliases); + } else if(sqlVistor.table.toUpperCase(Locale.ROOT).contains("_TABLES_")) { + sqlStream = new SelectStream(new TableStream(defaultZkhost), sqlVistor.columnAliases); } else if(sqlVistor.groupByQuery) { if(aggregationMode == AggregationMode.FACET) { sqlStream = doGroupByWithAggregatesFacets(sqlVistor); @@ -1378,7 +1373,7 @@ private static class CatalogsStream extends TupleStream { private int currentIndex = 0; private List catalogs; - public CatalogsStream(String zkHost) { + CatalogsStream(String zkHost) { this.zkHost = zkHost; } @@ -1392,7 +1387,7 @@ public void open() throws IOException { } public Tuple read() throws IOException { - Map fields = new HashMap<>(); + Map fields = new HashMap<>(); if (this.currentIndex < this.catalogs.size()) { fields.put("TABLE_CAT", this.catalogs.get(this.currentIndex)); this.currentIndex += 1; @@ -1418,10 +1413,45 @@ public void setStreamContext(StreamContext context) { private static class SchemasStream extends TupleStream { private final String zkHost; private StreamContext context; + + SchemasStream(String zkHost) { + this.zkHost = zkHost; + } + + public List children() { + return new ArrayList<>(); + } + + public void open() throws IOException { + + } + + public Tuple read() throws IOException { + Map fields = new HashMap<>(); + fields.put("EOF", "true"); + return new Tuple(fields); + } + + public StreamComparator getStreamSort() { + return null; + } + + public void close() throws IOException { + + } + + public void setStreamContext(StreamContext context) { + this.context = context; + } + } + + private static class TableStream extends TupleStream { + private final String zkHost; + private StreamContext context; private int currentIndex = 0; - private List schemas; + private List tables; - public SchemasStream(String zkHost) { + TableStream(String zkHost) { this.zkHost = zkHost; } @@ -1430,18 +1460,25 @@ public List children() { } public void open() throws IOException { - this.schemas = new ArrayList<>(); + this.tables = new ArrayList<>(); CloudSolrClient cloudSolrClient = this.context.getSolrClientCache().getCloudSolrClient(this.zkHost); - this.schemas.addAll(cloudSolrClient.getZkStateReader().getClusterState().getCollections()); - Collections.sort(this.schemas); + cloudSolrClient.connect(); + ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); + if (zkStateReader.getClusterState().getCollections().size() != 0) { + this.tables.addAll(zkStateReader.getClusterState().getCollections()); + } + Collections.sort(this.tables); } public Tuple read() throws IOException { - Map fields = new HashMap<>(); - if (this.currentIndex < this.schemas.size()) { - fields.put("TABLE_SCHEM", this.schemas.get(this.currentIndex)); - fields.put("TABLE_CATALOG", this.zkHost); + Map fields = new HashMap<>(); + if (this.currentIndex < this.tables.size()) { + fields.put("TABLE_CAT", this.zkHost); + fields.put("TABLE_SCHEM", null); + fields.put("TABLE_NAME", this.tables.get(this.currentIndex)); + fields.put("TABLE_TYPE", "TABLE"); + fields.put("REMARKS", null); this.currentIndex += 1; } else { fields.put("EOF", "true"); diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java index 0e5cbaf94990..9eca6db26b8a 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,7 +34,6 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; public class TestSQLHandler extends AbstractFullDistribZkTestBase { @@ -99,6 +99,9 @@ public void doTest() throws Exception { testParallelBasicGrouping(); testParallelSelectDistinct(); testParallelTimeSeriesGrouping(); + testCatalogStream(); + testSchemasStream(); + testTablesStream(); } private void testPredicate() throws Exception { @@ -2418,6 +2421,74 @@ private void testParallelTimeSeriesGrouping() throws Exception { } } + private void testCatalogStream() throws Exception { + CloudJettyRunner jetty = this.cloudJettys.get(0); + + Map params = new HashMap<>(); + params.put(CommonParams.QT, "/sql"); + params.put("numWorkers", 2); + params.put("stmt", "select TABLE_CAT from _CATALOGS_"); + + SolrStream solrStream = new SolrStream(jetty.url, params); + List tuples = getTuples(solrStream); + + assertEquals(tuples.size(), 1); + assertEquals(tuples.get(0).getString("TABLE_CAT"), zkServer.getZkAddress()); + } + + private void testSchemasStream() throws Exception { + CloudJettyRunner jetty = this.cloudJettys.get(0); + + Map params = new HashMap<>(); + params.put(CommonParams.QT, "/sql"); + params.put("numWorkers", 2); + params.put("stmt", "select TABLE_SCHEM, TABLE_CATALOG from _SCHEMAS_"); + + SolrStream solrStream = new SolrStream(jetty.url, params); + List tuples = getTuples(solrStream); + + assertEquals(tuples.size(), 0); + } + + private void testTablesStream() throws Exception { + CloudJettyRunner jetty = this.cloudJettys.get(0); + + Map params = new HashMap<>(); + params.put(CommonParams.QT, "/sql"); + params.put("numWorkers", 2); + params.put("stmt", "select TABLE_CAT, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE, REMARKS from _TABLES_"); + + SolrStream solrStream = new SolrStream(jetty.url, params); + List tuples = getTuples(solrStream); + + assertEquals(2, tuples.size()); + + List collections = new ArrayList<>(); + collections.addAll(cloudClient.getZkStateReader().getClusterState().getCollections()); + Collections.sort(collections); + for (Tuple tuple : tuples) { + assertEquals(zkServer.getZkAddress(), tuple.getString("TABLE_CAT")); + assertNull(tuple.get("TABLE_SCHEM")); + assertTrue(collections.contains(tuple.getString("TABLE_NAME"))); + assertEquals("TABLE", tuple.getString("TABLE_TYPE")); + assertNull(tuple.get("REMARKS")); + } + + tuples = getTuples(solrStream); + assertEquals(2, tuples.size()); + + collections = new ArrayList<>(); + collections.addAll(cloudClient.getZkStateReader().getClusterState().getCollections()); + Collections.sort(collections); + for (Tuple tuple : tuples) { + assertEquals(zkServer.getZkAddress(), tuple.getString("TABLE_CAT")); + assertNull(tuple.get("TABLE_SCHEM")); + assertTrue(collections.contains(tuple.getString("TABLE_NAME"))); + assertEquals("TABLE", tuple.getString("TABLE_TYPE")); + assertNull(tuple.get("REMARKS")); + } + } + protected List getTuples(TupleStream tupleStream) throws IOException { tupleStream.open(); List tuples = new ArrayList(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java index 95105c8e33f6..d1e79eb53082 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java @@ -55,7 +55,7 @@ class ConnectionImpl implements Connection { ConnectionImpl(String url, String zkHost, String collection, Properties properties) throws SQLException { this.url = url; this.client = this.solrClientCache.getCloudSolrClient(zkHost); - this.setSchema(collection); + this.collection = collection; this.properties = properties; this.connectionStatement = createStatement(); this.databaseMetaData = new DatabaseMetaDataImpl(this, this.connectionStatement); @@ -69,6 +69,10 @@ CloudSolrClient getClient() { return client; } + String getCollection() { + return collection; + } + Properties getProperties() { return properties; } @@ -345,12 +349,12 @@ public Struct createStruct(String typeName, Object[] attributes) throws SQLExcep @Override public void setSchema(String schema) throws SQLException { - this.collection = schema; + } @Override public String getSchema() throws SQLException { - return this.collection; + return null; } @Override diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java index 62f2a1869d8f..d316c83df09d 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java @@ -705,7 +705,7 @@ public ResultSet getProcedureColumns(String catalog, String schemaPattern, Strin @Override public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { - return null; + return this.connectionStatement.executeQuery("select TABLE_CAT, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE, REMARKS from _TABLES_"); } @Override diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java index 1adcea847890..7f1e2f9a41a5 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java @@ -76,10 +76,10 @@ protected SolrStream constructStream(String sql) throws IOException { try { ZkStateReader zkStateReader = this.connection.getClient().getZkStateReader(); ClusterState clusterState = zkStateReader.getClusterState(); - Collection slices = clusterState.getActiveSlices(this.connection.getSchema()); + Collection slices = clusterState.getActiveSlices(this.connection.getCollection()); if(slices == null) { - throw new Exception("Collection not found:"+this.connection.getSchema()); + throw new Exception("Collection not found:"+this.connection.getCollection()); } List shuffler = new ArrayList<>(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java index 572491e31e8f..393d63a64ef9 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.Set; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; @@ -36,7 +35,6 @@ import org.apache.solr.cloud.AbstractZkTestCase; import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -81,7 +79,6 @@ public void distribSetUp() throws Exception { super.distribSetUp(); } - @Override @After public void tearDown() throws Exception { @@ -444,9 +441,9 @@ private void testJDBCMethods(String collection, String connectionString, Propert con.setCatalog(zkServer.getZkAddress()); assertEquals(zkServer.getZkAddress(), con.getCatalog()); - assertEquals(collection, con.getSchema()); - con.setSchema(collection); - assertEquals(collection, con.getSchema()); + assertEquals(null, con.getSchema()); + con.setSchema("myschema"); + assertEquals(null, con.getSchema()); DatabaseMetaData databaseMetaData = con.getMetaData(); assertNotNull(databaseMetaData); @@ -478,11 +475,19 @@ private void testJDBCMethods(String collection, String connectionString, Propert List collections = new ArrayList<>(); collections.addAll(cloudClient.getZkStateReader().getClusterState().getCollections()); Collections.sort(collections); + try(ResultSet rs = databaseMetaData.getSchemas()) { + assertFalse(rs.next()); + } + + try(ResultSet rs = databaseMetaData.getTables(zkServer.getZkAddress(), null, "%", null)) { for(String acollection : collections) { assertTrue(rs.next()); - assertEquals(acollection, rs.getString("TABLE_SCHEM")); - assertEquals(zkServer.getZkAddress(), rs.getString("TABLE_CATALOG")); + assertEquals(zkServer.getZkAddress(), rs.getString("TABLE_CAT")); + assertNull(rs.getString("TABLE_SCHEM")); + assertEquals(acollection, rs.getString("TABLE_NAME")); + assertEquals("TABLE", rs.getString("TABLE_TYPE")); + assertNull(rs.getString("REMARKS")); } assertFalse(rs.next()); } From 064f599cc8becb6a7383b45b8effe0ae01711661 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 23 Mar 2016 17:12:14 -0400 Subject: [PATCH 0199/1113] LUCENE-7133: check field name in equals/hashCode for point queries --- .../apache/lucene/search/PointInSetQuery.java | 4 +- .../apache/lucene/search/PointRangeQuery.java | 9 +++- .../lucene/search/TestPointQueries.java | 20 ++++++--- .../standard/TestPointQueryParser.java | 2 +- .../document/LatLonPointDistanceQuery.java | 2 +- .../document/LatLonPointInPolygonQuery.java | 6 ++- .../lucene/document/TestBigIntegerPoint.java | 6 ++- .../lucene/document/TestInetAddressPoint.java | 6 ++- .../lucene/search/TestLatLonPointQueries.java | 2 +- .../spatial/util/BaseGeoPointTestCase.java | 43 +++++++++++++++++++ .../spatial3d/PointInGeo3DShapeQuery.java | 4 ++ .../lucene/spatial3d/TestGeo3DPoint.java | 1 + 12 files changed, 88 insertions(+), 17 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java index bee864f8966d..3314e29d49ff 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointInSetQuery.java @@ -304,6 +304,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { @Override public final int hashCode() { int hash = super.hashCode(); + hash = 31 * hash + field.hashCode(); hash = 31 * hash + sortedPackedPointsHashCode; hash = 31 * hash + numDims; hash = 31 * hash + bytesPerDim; @@ -314,7 +315,8 @@ public final int hashCode() { public final boolean equals(Object other) { if (super.equals(other)) { final PointInSetQuery q = (PointInSetQuery) other; - return q.numDims == numDims && + return q.field.equals(field) && + q.numDims == numDims && q.bytesPerDim == bytesPerDim && q.sortedPackedPointsHashCode == sortedPackedPointsHashCode && q.sortedPackedPoints.equals(sortedPackedPoints); diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index cfa644fea9b3..eb183e258407 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -219,6 +219,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { @Override public final int hashCode() { int hash = super.hashCode(); + hash = 31 * hash + field.hashCode(); hash = 31 * hash + Arrays.hashCode(lowerPoint); hash = 31 * hash + Arrays.hashCode(upperPoint); hash = 31 * hash + numDims; @@ -233,6 +234,10 @@ public final boolean equals(Object other) { } final PointRangeQuery q = (PointRangeQuery) other; + if (field.equals(q.field) == false) { + return false; + } + if (q.numDims != numDims) { return false; } @@ -241,11 +246,11 @@ public final boolean equals(Object other) { return false; } - if (!Arrays.equals(lowerPoint, q.lowerPoint)) { + if (Arrays.equals(lowerPoint, q.lowerPoint) == false) { return false; } - if (!Arrays.equals(upperPoint, q.upperPoint)) { + if (Arrays.equals(upperPoint, q.upperPoint) == false) { return false; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index b4a586d6b376..c2136c4d6630 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -1896,11 +1896,14 @@ public void testRangeOptimizesIfAllPointsMatch() throws IOException { } public void testPointRangeEquals() { - Query q1 = IntPoint.newRangeQuery("a", 0, 1000); - Query q2 = IntPoint.newRangeQuery("a", 0, 1000); + Query q1, q2; + + q1 = IntPoint.newRangeQuery("a", 0, 1000); + q2 = IntPoint.newRangeQuery("a", 0, 1000); assertEquals(q1, q2); assertEquals(q1.hashCode(), q2.hashCode()); assertFalse(q1.equals(IntPoint.newRangeQuery("a", 1, 1000))); + assertFalse(q1.equals(IntPoint.newRangeQuery("b", 0, 1000))); q1 = LongPoint.newRangeQuery("a", 0, 1000); q2 = LongPoint.newRangeQuery("a", 0, 1000); @@ -1933,11 +1936,14 @@ public void testPointRangeEquals() { } public void testPointExactEquals() { - Query q1 = IntPoint.newExactQuery("a", 1000); - Query q2 = IntPoint.newExactQuery("a", 1000); + Query q1, q2; + + q1 = IntPoint.newExactQuery("a", 1000); + q2 = IntPoint.newExactQuery("a", 1000); assertEquals(q1, q2); assertEquals(q1.hashCode(), q2.hashCode()); assertFalse(q1.equals(IntPoint.newExactQuery("a", 1))); + assertFalse(q1.equals(IntPoint.newExactQuery("b", 1000))); q1 = LongPoint.newExactQuery("a", 1000); q2 = LongPoint.newExactQuery("a", 1000); @@ -1969,11 +1975,13 @@ public void testPointExactEquals() { } public void testPointInSetEquals() { - Query q1 = IntPoint.newSetQuery("a", 0, 1000, 17); - Query q2 = IntPoint.newSetQuery("a", 17, 0, 1000); + Query q1, q2; + q1 = IntPoint.newSetQuery("a", 0, 1000, 17); + q2 = IntPoint.newSetQuery("a", 17, 0, 1000); assertEquals(q1, q2); assertEquals(q1.hashCode(), q2.hashCode()); assertFalse(q1.equals(IntPoint.newSetQuery("a", 1, 17, 1000))); + assertFalse(q1.equals(IntPoint.newSetQuery("b", 0, 1000, 17))); q1 = LongPoint.newSetQuery("a", 0, 1000, 17); q2 = LongPoint.newSetQuery("a", 17, 0, 1000); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java index 323b0ffba9fd..a301de0a7f1c 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestPointQueryParser.java @@ -75,7 +75,7 @@ public void testDoubles() throws Exception { assertEquals(DoublePoint.newRangeQuery("doubleField", 1.5D, 3.6D), parser.parse("doubleField:[1.5 TO 3.6]", "body")); - assertEquals(DoublePoint.newRangeQuery("floatField", 1.5D, 1.5D), + assertEquals(DoublePoint.newRangeQuery("doubleField", 1.5D, 1.5D), parser.parse("doubleField:1.5", "body")); } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index bba92b3e854c..10d11c6f27b4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -295,7 +295,7 @@ public boolean equals(Object obj) { if (!super.equals(obj)) return false; if (getClass() != obj.getClass()) return false; LatLonPointDistanceQuery other = (LatLonPointDistanceQuery) obj; - if (!field.equals(other.field)) return false; + if (field.equals(other.field) == false) return false; if (Double.doubleToLongBits(latitude) != Double.doubleToLongBits(other.latitude)) return false; if (Double.doubleToLongBits(longitude) != Double.doubleToLongBits(other.longitude)) return false; if (Double.doubleToLongBits(radiusMeters) != Double.doubleToLongBits(other.radiusMeters)) return false; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 68c734e9a492..494f42733c9f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -258,6 +258,9 @@ public boolean equals(Object o) { LatLonPointInPolygonQuery that = (LatLonPointInPolygonQuery) o; + if (field.equals(that.field) == false) { + return false; + } if (Arrays.equals(polyLons, that.polyLons) == false) { return false; } @@ -269,8 +272,9 @@ public boolean equals(Object o) { } @Override - public final int hashCode() { + public int hashCode() { int result = super.hashCode(); + result = 31 * result + field.hashCode(); result = 31 * result + Arrays.hashCode(polyLons); result = 31 * result + Arrays.hashCode(polyLats); return result; diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java index 7d9281b77dab..98c6a679857d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestBigIntegerPoint.java @@ -96,11 +96,13 @@ public void testToString() throws Exception { } public void testQueryEquals() throws Exception { - Query q1 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); - Query q2 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); + Query q1, q2; + q1 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); + q2 = BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(0), BigInteger.valueOf(1000)); assertEquals(q1, q2); assertEquals(q1.hashCode(), q2.hashCode()); assertFalse(q1.equals(BigIntegerPoint.newRangeQuery("a", BigInteger.valueOf(1), BigInteger.valueOf(1000)))); + assertFalse(q1.equals(BigIntegerPoint.newRangeQuery("b", BigInteger.valueOf(0), BigInteger.valueOf(1000)))); q1 = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); q2 = BigIntegerPoint.newExactQuery("a", BigInteger.valueOf(1000)); diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java index ee34ecc5b448..673ee296a48a 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestInetAddressPoint.java @@ -92,11 +92,13 @@ public void testToString() throws Exception { } public void testQueryEquals() throws Exception { - Query q1 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); - Query q2 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + Query q1, q2; + q1 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); + q2 = InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")); assertEquals(q1, q2); assertEquals(q1.hashCode(), q2.hashCode()); assertFalse(q1.equals(InetAddressPoint.newRangeQuery("a", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.7")))); + assertFalse(q1.equals(InetAddressPoint.newRangeQuery("b", InetAddress.getByName("1.2.3.3"), InetAddress.getByName("1.2.3.5")))); q1 = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); q2 = InetAddressPoint.newPrefixQuery("a", InetAddress.getByName("1.2.3.3"), 16); diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 71ef7efa9555..b13677689935 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -45,7 +45,7 @@ protected Query newDistanceRangeQuery(String field, double centerLat, double cen @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return LatLonPoint.newPolygonQuery(FIELD_NAME, lats, lons); + return LatLonPoint.newPolygonQuery(field, lats, lons); } @Override diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 0efe7d6165c7..3943a92039ff 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -971,4 +971,47 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException writer.close(); dir.close(); } + + public void testEquals() throws Exception { + Query q1, q2; + + GeoRect rect = randomRect(false, true); + + q1 = newRectQuery("field", rect); + q2 = newRectQuery("field", rect); + assertEquals(q1, q2); + assertFalse(q1.equals(newRectQuery("field2", rect))); + + double lat = randomLat(false); + double lon = randomLon(false); + q1 = newDistanceQuery("field", lat, lon, 10000.0); + q2 = newDistanceQuery("field", lat, lon, 10000.0); + assertEquals(q1, q2); + assertFalse(q1.equals(newDistanceQuery("field2", lat, lon, 10000.0))); + + q1 = newDistanceRangeQuery("field", lat, lon, 10000.0, 100000.0); + if (q1 != null) { + // Not all subclasses can make distance range query! + q2 = newDistanceRangeQuery("field", lat, lon, 10000.0, 100000.0); + assertEquals(q1, q2); + assertFalse(q1.equals(newDistanceRangeQuery("field2", lat, lon, 10000.0, 100000.0))); + } + + double[] lats = new double[5]; + double[] lons = new double[5]; + lats[0] = rect.minLat; + lons[0] = rect.minLon; + lats[1] = rect.maxLat; + lons[1] = rect.minLon; + lats[2] = rect.maxLat; + lons[2] = rect.maxLon; + lats[3] = rect.minLat; + lons[3] = rect.maxLon; + lats[4] = rect.minLat; + lons[4] = rect.minLon; + q1 = newPolygonQuery("field", lats, lons); + q2 = newPolygonQuery("field", lats, lons); + assertEquals(q1, q2); + assertFalse(q1.equals(newPolygonQuery("field2", lats, lons))); + } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index c9b5e4e635bb..3cc95304a67d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -187,6 +187,9 @@ public boolean equals(Object o) { if (!super.equals(o)) return false; PointInGeo3DShapeQuery that = (PointInGeo3DShapeQuery) o; + if (field.equals(that.field) == false) { + return false; + } return shape.equals(that.shape); } @@ -194,6 +197,7 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = super.hashCode(); + result = 31 * result + field.hashCode(); result = 31 * result + shape.hashCode(); return result; } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 3061b763f856..5c0044f43714 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -812,6 +812,7 @@ public void testEquals() { GeoShape shape = randomShape(PlanetModel.WGS84); Query q = Geo3DPoint.newShapeQuery("point", shape); assertEquals(q, Geo3DPoint.newShapeQuery("point", shape)); + assertFalse(q.equals(Geo3DPoint.newShapeQuery("point2", shape))); // make a different random shape: GeoShape shape2; From 2b13210c3d113b224ccc0878f8c792942e078fd4 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Wed, 23 Mar 2016 17:02:05 -0500 Subject: [PATCH 0200/1113] SOLR-8810: Implement Connection.setReadOnly, Statement.set/getFetchSize, ResultSet.getType --- solr/CHANGES.txt | 3 +++ .../solr/client/solrj/io/sql/ConnectionImpl.java | 2 +- .../apache/solr/client/solrj/io/sql/ResultSetImpl.java | 2 +- .../apache/solr/client/solrj/io/sql/StatementImpl.java | 5 ++--- .../org/apache/solr/client/solrj/io/sql/JdbcTest.java | 10 ++++++++++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d7b30949bd94..ff06ae857a0f 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -498,6 +498,9 @@ Other Changes * SOLR-8819: Implement DatabaseMetaDataImpl getTables() and fix getSchemas(). (Trey Cahill, Joel Bernstein, Kevin Risden) +* SOLR-8810: Implement Connection.setReadOnly, Statement.set/getFetchSize, + ResultSet.getType (Kevin Risden) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java index d1e79eb53082..aa78e14ecb50 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ConnectionImpl.java @@ -152,7 +152,7 @@ public DatabaseMetaData getMetaData() throws SQLException { @Override public void setReadOnly(boolean readOnly) throws SQLException { - throw new UnsupportedOperationException(); + } @Override diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java index 0aa3a4bc5173..e4dcaed82004 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java @@ -560,7 +560,7 @@ public int getFetchSize() throws SQLException { @Override public int getType() throws SQLException { - throw new UnsupportedOperationException(); + return ResultSet.TYPE_FORWARD_ONLY; } @Override diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java index 7f1e2f9a41a5..6af58dae527c 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/StatementImpl.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.HashMap; import java.util.Random; @@ -240,12 +239,12 @@ public int getFetchDirection() throws SQLException { @Override public void setFetchSize(int rows) throws SQLException { - throw new UnsupportedOperationException(); + } @Override public int getFetchSize() throws SQLException { - throw new UnsupportedOperationException(); + return 0; } @Override diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java index 393d63a64ef9..200c971c4bf6 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java @@ -492,6 +492,10 @@ private void testJDBCMethods(String collection, String connectionString, Propert assertFalse(rs.next()); } + assertTrue(con.isReadOnly()); + con.setReadOnly(true); + assertTrue(con.isReadOnly()); + assertNull(con.getWarnings()); con.clearWarnings(); assertNull(con.getWarnings()); @@ -503,6 +507,10 @@ private void testJDBCMethods(String collection, String connectionString, Propert statement.clearWarnings(); assertNull(statement.getWarnings()); + assertEquals(0, statement.getFetchSize()); + statement.setFetchSize(0); + assertEquals(0, statement.getFetchSize()); + try (ResultSet rs = statement.executeQuery(sql)) { assertEquals(statement, rs.getStatement()); @@ -564,6 +572,8 @@ private void checkResultSetMetadata(ResultSet rs) throws Exception { } private void checkResultSet(ResultSet rs) throws Exception { + assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + assertNull(rs.getWarnings()); rs.clearWarnings(); assertNull(rs.getWarnings()); From b978666d572e4e662d8a6dee9e5daace257e5e90 Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 23 Mar 2016 20:43:30 -0400 Subject: [PATCH 0201/1113] Fix StrField.toObject and toExternal to work with docValue IndexableField instances, optimize createFields --- solr/CHANGES.txt | 3 +++ .../org/apache/solr/schema/FieldType.java | 7 ++++- .../java/org/apache/solr/schema/StrField.java | 26 ++++++++++++------- .../org/apache/solr/schema/DocValuesTest.java | 4 ++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ff06ae857a0f..a9731e16f9a2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -348,6 +348,9 @@ Bug Fixes * SOLR-8886: Fix TrieField.toObject(IndexableField) to work for field with docValues enabled. (yonik) +* SOLR-8891: Fix StrField.toObject and toExternal to work with docValue IndexableField + instances. (yonik) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index fbcebcda1b16..ab89f94c5754 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -353,7 +353,12 @@ public String toExternal(IndexableField f) { // currently used in writing XML of the search result (but perhaps // a more efficient toXML(IndexableField f, Writer w) should be used // in the future. - return f.stringValue(); + String val = f.stringValue(); + if (val == null) { + // docValues will use the binary value + val = f.binaryValue().utf8ToString(); + } + return val; } /** diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java index 8aa8ecfff069..7bda94336dd4 100644 --- a/solr/core/src/java/org/apache/solr/schema/StrField.java +++ b/solr/core/src/java/org/apache/solr/schema/StrField.java @@ -40,21 +40,29 @@ protected void init(IndexSchema schema, Map args) { } @Override - public List createFields(SchemaField field, Object value, - float boost) { + public List createFields(SchemaField field, Object value, float boost) { + IndexableField fval = createField(field, value, boost); + if (field.hasDocValues()) { - List fields = new ArrayList<>(); - fields.add(createField(field, value, boost)); + IndexableField docval; final BytesRef bytes = new BytesRef(value.toString()); if (field.multiValued()) { - fields.add(new SortedSetDocValuesField(field.getName(), bytes)); + docval = new SortedSetDocValuesField(field.getName(), bytes); } else { - fields.add(new SortedDocValuesField(field.getName(), bytes)); + docval = new SortedDocValuesField(field.getName(), bytes); } - return fields; - } else { - return Collections.singletonList(createField(field, value, boost)); + + // Only create a list of we have 2 values... + if (fval != null) { + List fields = new ArrayList<>(2); + fields.add(fval); + fields.add(docval); + return fields; + } + + fval = docval; } + return Collections.singletonList(fval); } @Override diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java index 2537c8f0f6d2..6411a834b35b 100644 --- a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -116,7 +116,9 @@ public void testDocValues() throws IOException { tstToObj(schema.getField("longdvs"), -11L); tstToObj(schema.getField("datedv"), new Date(1000)); tstToObj(schema.getField("datedvs"), new Date(1000)); - + tstToObj(schema.getField("stringdv"), "foo"); + tstToObj(schema.getField("stringdvs"), "foo"); + } finally { searcherRef.decref(); } From 99d1478810d689506b91ace1a6b53119458b7c5d Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 23 Mar 2016 23:04:50 -0400 Subject: [PATCH 0202/1113] LUCENE-7119: allow bypass of docvalues check in DocTermOrds --- .../src/java/org/apache/lucene/uninverting/DocTermOrds.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java b/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java index 6ef09ce2da12..3600acd01a6f 100644 --- a/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java +++ b/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java @@ -163,6 +163,10 @@ public class DocTermOrds implements Accountable { /** Used while uninverting. */ protected PostingsEnum postingsEnum; + /** If true, check and throw an exception if the field has docValues enabled. + * Normally, docValues should be used in preference to DocTermOrds. */ + protected boolean checkForDocValues = true; + /** Returns total bytes used. */ public long ramBytesUsed() { // can cache the mem size since it shouldn't change @@ -268,7 +272,7 @@ protected void setActualDocFreq(int termNum, int df) throws IOException { /** Call this only once (if you subclass!) */ protected void uninvert(final LeafReader reader, Bits liveDocs, final BytesRef termPrefix) throws IOException { final FieldInfo info = reader.getFieldInfos().fieldInfo(field); - if (info != null && info.getDocValuesType() != DocValuesType.NONE) { + if (checkForDocValues && info != null && info.getDocValuesType() != DocValuesType.NONE) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } //System.out.println("DTO uninvert field=" + field + " prefix=" + termPrefix); From d9de158676f1dca7ae17b1ad3aae4aa1e91b3212 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 24 Mar 2016 05:13:59 -0400 Subject: [PATCH 0203/1113] fix test bug: pass the correct point count when creating BKDWriter --- .../org/apache/lucene/util/bkd/TestBKD.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index dc7f8f7e3cf4..1a6e653f4bf4 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -43,17 +43,9 @@ public class TestBKD extends LuceneTestCase { - private long randomPointCount(boolean singleValuePerDoc) { - if (singleValuePerDoc || random().nextBoolean()) { - return random().nextInt(Integer.MAX_VALUE); - } else { - return random().nextLong() & Long.MAX_VALUE; - } - } - public void testBasicInts1D() throws Exception { try (Directory dir = getDirectory(100)) { - BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, randomPointCount(true), true); + BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, 100, true); byte[] scratch = new byte[4]; for(int docID=0;docID<100;docID++) { NumericUtils.intToSortableBytes(docID, scratch, 0); @@ -128,7 +120,7 @@ public void testRandomIntsNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, randomPointCount(true), true); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, 4, maxPointsInLeafNode, maxMB, numDocs, true); if (VERBOSE) { System.out.println("TEST: numDims=" + numDims + " numDocs=" + numDocs); @@ -269,7 +261,7 @@ public void testBigIntNDims() throws Exception { int numDims = TestUtil.nextInt(random(), 1, 5); int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 100); float maxMB = (float) 3.0 + (3*random().nextFloat()); - BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(true), true); + BKDWriter w = new BKDWriter(numDocs, dir, "tmp", numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, numDocs, true); BigInteger[][] docs = new BigInteger[numDocs][]; byte[] scratch = new byte[numBytesPerDim*numDims]; @@ -442,7 +434,7 @@ public void testRandomBinaryBig() throws Exception { public void testTooLittleHeap() throws Exception { try (Directory dir = getDirectory(0)) { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, randomPointCount(true), true); + new BKDWriter(1, dir, "bkd", 1, 16, 1000000, 0.001, 0, true); }); assertTrue(expected.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode")); } @@ -565,7 +557,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi List docIDBases = null; int seg = 0; - BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); + BKDWriter w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false); IndexOutput out = dir.createOutput("bkd", IOContext.DEFAULT); IndexInput in = null; @@ -619,7 +611,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi seg++; maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 1000); maxMB = (float) 3.0 + (3*random().nextDouble()); - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false); lastDocIDBase = docID; } } @@ -634,7 +626,7 @@ private void verify(Directory dir, byte[][][] docValues, int[] docIDs, int numDi out.close(); in = dir.openInput("bkd", IOContext.DEFAULT); seg++; - w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, randomPointCount(false), false); + w = new BKDWriter(numValues, dir, "_" + seg, numDims, numBytesPerDim, maxPointsInLeafNode, maxMB, docValues.length, false); List readers = new ArrayList<>(); for(long fp : toMerge) { in.seek(fp); From 2ca08aa6ad09dd652440c9aab1269e8652f75ae7 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 24 Mar 2016 10:02:48 -0400 Subject: [PATCH 0204/1113] simplify BKDWriter's comparator tie break for its temp files; add test case verifying tie break by docID --- .../org/apache/lucene/util/bkd/BKDWriter.java | 26 ++++-------- .../lucene/util/bkd/OfflinePointReader.java | 4 +- .../lucene/util/bkd/OfflinePointWriter.java | 2 +- .../org/apache/lucene/util/bkd/TestBKD.java | 40 ++++++++++++++++++- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 6dc413aa392e..ffba762654be 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -716,32 +716,20 @@ private PointWriter sort(int dim) throws IOException { @Override public int compare(BytesRef a, BytesRef b) { - - // First compare the bytes on the dimension we are sorting on: + // First compare by the requested dimension we are sorting by: int cmp = StringHelper.compare(bytesPerDim, a.bytes, a.offset + bytesPerDim*dim, b.bytes, b.offset + bytesPerDim*dim); if (cmp != 0) { return cmp; } - // Tie-break by docID: - int offset; - if (singleValuePerDoc) { - offset = 0; - } else if (longOrds) { - offset = Long.BYTES; - } else { - offset = Integer.BYTES; - } - reader.reset(a.bytes, a.offset + packedBytesLength + offset, a.length); - final int docIDA = reader.readInt(); - - reader.reset(b.bytes, b.offset + packedBytesLength + offset, b.length); - final int docIDB = reader.readInt(); + // Tie-break by docID ... no need to tie break on ord, for the case where the same doc has + // the same value in a given dimension indexed more than once: it can't matter at search + // time since we don't write ords into the index: - // No need to tie break on ord, for the case where the same doc has the same value in a given dimension indexed more than once: it - // can't matter at search time since we don't write ords into the index: - return Integer.compare(docIDA, docIDB); + return StringHelper.compare(Integer.BYTES, + a.bytes, a.offset + packedBytesLength, + b.bytes, b.offset + packedBytesLength); } }; diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index ebb2fd14a7f0..f3bf2bc08cb2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -93,15 +93,15 @@ public boolean next() throws IOException { assert countLeft == -1; return false; } + docID = in.readInt(); if (singleValuePerDoc == false) { if (longOrds) { ord = in.readLong(); } else { ord = in.readInt(); } - docID = in.readInt(); } else { - ord = docID = in.readInt(); + ord = docID; } return true; } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 2261b81380d8..2a5952e5c260 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -68,6 +68,7 @@ public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLen public void append(byte[] packedValue, long ord, int docID) throws IOException { assert packedValue.length == packedBytesLength; out.writeBytes(packedValue, 0, packedValue.length); + out.writeInt(docID); if (singleValuePerDoc == false) { if (longOrds) { out.writeLong(ord); @@ -76,7 +77,6 @@ public void append(byte[] packedValue, long ord, int docID) throws IOException { out.writeInt((int) ord); } } - out.writeInt(docID); count++; assert expectedCount == 0 || count <= expectedCount; } diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index 1a6e653f4bf4..1aaa689de24e 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -45,7 +45,7 @@ public class TestBKD extends LuceneTestCase { public void testBasicInts1D() throws Exception { try (Directory dir = getDirectory(100)) { - BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, 100, true); + BKDWriter w = new BKDWriter(100, dir, "tmp", 1, 4, 2, 1.0f, 100, true); byte[] scratch = new byte[4]; for(int docID=0;docID<100;docID++) { NumericUtils.intToSortableBytes(docID, scratch, 0); @@ -889,4 +889,42 @@ private void assertCorruptionDetected(Throwable t) { } fail("did not see a supporessed CorruptIndexException"); } + + public void testTieBreakOrder() throws Exception { + try (Directory dir = newDirectory()) { + int numDocs = 10000; + BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, 4, 2, 0.01f, numDocs, true); + for(int i=0;i Date: Thu, 24 Mar 2016 12:18:40 -0400 Subject: [PATCH 0206/1113] LUCENE-7137: consolidate many tests across Points and GeoPoint queries/fields --- .../lucene/document/TestLatLonPoint.java | 126 +------ .../TestLatLonPointDistanceQuery.java | 82 ---- .../TestLatLonPointInPolygonQuery.java | 49 --- .../lucene/search/TestLatLonPointQueries.java | 5 +- .../geopoint/document/GeoPointField.java | 32 +- .../search/GeoPointDistanceQuery.java | 49 ++- .../geopoint/search/GeoPointInBBoxQuery.java | 18 + .../lucene/spatial/util/GeoDistanceUtils.java | 61 --- .../geopoint/search/TestGeoPointField.java | 255 ------------- .../geopoint/search/TestGeoPointQuery.java | 12 +- .../search/TestLegacyGeoPointField.java | 242 ------------ .../search/TestLegacyGeoPointQuery.java | 12 +- .../spatial/util/BaseGeoPointTestCase.java | 352 +++++++++++++++++- .../lucene/spatial/util/TestGeoUtils.java | 23 ++ 14 files changed, 426 insertions(+), 892 deletions(-) delete mode 100644 lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java delete mode 100644 lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java delete mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java delete mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 1b6a6d23db1d..55c8062a31d1 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -16,37 +16,11 @@ */ package org.apache.lucene.document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; /** Simple tests for {@link LatLonPoint} */ public class TestLatLonPoint extends LuceneTestCase { - /** Add a single point and search for it in a box */ - // NOTE: we don't currently supply an exact search, only ranges, because of the lossiness... - public void testBoxQuery() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - - // add a doc with a point - Document document = new Document(); - document.add(new LatLonPoint("field", 18.313694, -65.227444)); - writer.addDocument(document); - - // search and verify we found our doc - IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); - assertEquals(1, searcher.count(LatLonPoint.newBoxQuery("field", 18, 19, -66, -65))); - - reader.close(); - writer.close(); - dir.close(); - } - public void testToString() throws Exception { // looks crazy due to lossiness assertEquals("LatLonPoint ",(new LatLonPoint("field", 18.313694, -65.227444)).toString()); @@ -60,79 +34,6 @@ public void testToString() throws Exception { // sort field assertEquals("", LatLonPoint.newDistanceSort("field", 18.0, 19.0).toString()); } - - /** Valid values that should not cause exception */ - public void testExtremeValues() { - new LatLonPoint("foo", 90.0, 180.0); - new LatLonPoint("foo", 90.0, -180.0); - new LatLonPoint("foo", -90.0, 180.0); - new LatLonPoint("foo", -90.0, -180.0); - } - - /** Invalid values */ - public void testOutOfRangeValues() { - IllegalArgumentException expected; - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", Math.nextUp(90.0), 50.0); - }); - assertTrue(expected.getMessage().contains("invalid latitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", Math.nextDown(-90.0), 50.0); - }); - assertTrue(expected.getMessage().contains("invalid latitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", 90.0, Math.nextUp(180.0)); - }); - assertTrue(expected.getMessage().contains("invalid longitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", 90.0, Math.nextDown(-180.0)); - }); - assertTrue(expected.getMessage().contains("invalid longitude")); - } - - /** NaN: illegal */ - public void testNaNValues() { - IllegalArgumentException expected; - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", Double.NaN, 50.0); - }); - assertTrue(expected.getMessage().contains("invalid latitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", 50.0, Double.NaN); - }); - assertTrue(expected.getMessage().contains("invalid longitude")); - } - - /** Inf: illegal */ - public void testInfValues() { - IllegalArgumentException expected; - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", Double.POSITIVE_INFINITY, 50.0); - }); - assertTrue(expected.getMessage().contains("invalid latitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", Double.NEGATIVE_INFINITY, 50.0); - }); - assertTrue(expected.getMessage().contains("invalid latitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", 50.0, Double.POSITIVE_INFINITY); - }); - assertTrue(expected.getMessage().contains("invalid longitude")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - new LatLonPoint("foo", 50.0, Double.NEGATIVE_INFINITY); - }); - assertTrue(expected.getMessage().contains("invalid longitude")); - } public void testEncodeDecode() throws Exception { // just for testing quantization error @@ -183,30 +84,5 @@ public void testEncodeDecodeIsStable() throws Exception { assertEquals(latEnc, latEnc2, 0.0); assertEquals(lonEnc, lonEnc2, 0.0); } - } - - public void testQueryEquals() throws Exception { - Query q1 = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); - Query q2 = LatLonPoint.newBoxQuery("field", 50, 70, -40, 20); - assertEquals(q1, q2); - assertEquals(q1.hashCode(), q2.hashCode()); - assertFalse(q1.equals(LatLonPoint.newBoxQuery("field", 50, 70, -40, 10))); - - q1 = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); - q2 = LatLonPoint.newDistanceQuery("field", 50, 70, 10000); - assertEquals(q1, q2); - assertEquals(q1.hashCode(), q2.hashCode()); - assertFalse(q1.equals(LatLonPoint.newDistanceQuery("field", 50, 70, 11000))); - assertFalse(q1.equals(LatLonPoint.newDistanceQuery("field", 50, 60, 10000))); - - - double[] polyLats1 = new double[] {30, 40, 40, 30, 30}; - double[] polyLons1 = new double[] {90, 90, -40, -40, 90}; - double[] polyLats2 = new double[] {20, 40, 40, 20, 20}; - q1 = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); - q2 = LatLonPoint.newPolygonQuery("field", polyLats1, polyLons1); - assertEquals(q1, q2); - assertEquals(q1.hashCode(), q2.hashCode()); - assertFalse(q1.equals(LatLonPoint.newPolygonQuery("field", polyLats2, polyLons1))); - } + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java deleted file mode 100644 index f015780be2c8..000000000000 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceQuery.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.document; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** Simple tests for {@link LatLonPoint#newDistanceQuery} */ -public class TestLatLonPointDistanceQuery extends LuceneTestCase { - - /** test we can search for a point */ - public void testBasics() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - - // add a doc with a location - Document document = new Document(); - document.add(new LatLonPoint("field", 18.313694, -65.227444)); - writer.addDocument(document); - - // search within 50km and verify we found our doc - IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); - assertEquals(1, searcher.count(LatLonPoint.newDistanceQuery("field", 18, -65, 50_000))); - - reader.close(); - writer.close(); - dir.close(); - } - - /** negative distance queries are not allowed */ - public void testNegativeRadius() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - LatLonPoint.newDistanceQuery("field", 18, 19, -1); - }); - assertTrue(expected.getMessage().contains("radiusMeters")); - assertTrue(expected.getMessage().contains("is invalid")); - } - - /** NaN distance queries are not allowed */ - public void testNaNRadius() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - LatLonPoint.newDistanceQuery("field", 18, 19, Double.NaN); - }); - assertTrue(expected.getMessage().contains("radiusMeters")); - assertTrue(expected.getMessage().contains("is invalid")); - } - - /** Inf distance queries are not allowed */ - public void testInfRadius() { - IllegalArgumentException expected; - - expected = expectThrows(IllegalArgumentException.class, () -> { - LatLonPoint.newDistanceQuery("field", 18, 19, Double.POSITIVE_INFINITY); - }); - assertTrue(expected.getMessage().contains("radiusMeters")); - assertTrue(expected.getMessage().contains("is invalid")); - - expected = expectThrows(IllegalArgumentException.class, () -> { - LatLonPoint.newDistanceQuery("field", 18, 19, Double.NEGATIVE_INFINITY); - }); - assertTrue(expected.getMessage().contains("radiusMeters")); - assertTrue(expected.getMessage().contains("is invalid")); - } -} diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java deleted file mode 100644 index de870277a33c..000000000000 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointInPolygonQuery.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.document; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** Simple tests for {@link LatLonPoint#newPolygonQuery} */ -public class TestLatLonPointInPolygonQuery extends LuceneTestCase { - - /** test we can search for a polygon */ - public void testBasics() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - - // add a doc with a point - Document document = new Document(); - document.add(new LatLonPoint("field", 18.313694, -65.227444)); - writer.addDocument(document); - - // search and verify we found our doc - IndexReader reader = writer.getReader(); - IndexSearcher searcher = newSearcher(reader); - assertEquals(1, searcher.count(LatLonPoint.newPolygonQuery("field", - new double[] { 18, 18, 19, 19, 18 }, - new double[] { -66, -65, -65, -66, -66 }))); - - reader.close(); - writer.close(); - dir.close(); - } -} diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index b13677689935..4f2c2d7a23c5 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -19,7 +19,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; -import org.apache.lucene.spatial.util.GeoRect; public class TestLatLonPointQueries extends BaseGeoPointTestCase { @@ -29,8 +28,8 @@ protected void addPointToDoc(String field, Document doc, double lat, double lon) } @Override - protected Query newRectQuery(String field, GeoRect rect) { - return LatLonPoint.newBoxQuery(field, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); + protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) { + return LatLonPoint.newBoxQuery(field, minLat, maxLat, minLon, maxLon); } @Override diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java index 00db23a8d0f5..a477fa90ae25 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java @@ -128,47 +128,47 @@ public enum TermEncoding { /** Creates a stored or un-stored GeoPointField * @param name field name - * @param lat latitude double value [-90.0 : 90.0] - * @param lon longitude double value [-180.0 : 180.0] + * @param latitude latitude double value [-90.0 : 90.0] + * @param longitude longitude double value [-180.0 : 180.0] * @param stored Store.YES if the content should also be stored * @throws IllegalArgumentException if the field name is null. */ - public GeoPointField(String name, double lat, double lon, Store stored) { - this(name, lat, lon, getFieldType(stored)); + public GeoPointField(String name, double latitude, double longitude, Store stored) { + this(name, latitude, longitude, getFieldType(stored)); } /** Creates a stored or un-stored GeoPointField using the specified {@link TermEncoding} method * @param name field name - * @param lat latitude double value [-90.0 : 90.0] - * @param lon longitude double value [-180.0 : 180.0] + * @param latitude latitude double value [-90.0 : 90.0] + * @param longitude longitude double value [-180.0 : 180.0] * @param termEncoding encoding type to use ({@link TermEncoding#NUMERIC} Terms, or {@link TermEncoding#PREFIX} only Terms) * @param stored Store.YES if the content should also be stored * @throws IllegalArgumentException if the field name is null. */ @Deprecated - public GeoPointField(String name, double lat, double lon, TermEncoding termEncoding, Store stored) { - this(name, lat, lon, getFieldType(termEncoding, stored)); + public GeoPointField(String name, double latitude, double longitude, TermEncoding termEncoding, Store stored) { + this(name, latitude, longitude, getFieldType(termEncoding, stored)); } /** Expert: allows you to customize the {@link * FieldType}. * @param name field name - * @param lat latitude double value [-90.0 : 90.0] - * @param lon longitude double value [-180.0 : 180.0] + * @param latitude latitude double value [-90.0 : 90.0] + * @param longitude longitude double value [-180.0 : 180.0] * @param type customized field type: must have {@link FieldType#numericType()} * of {@link org.apache.lucene.document.FieldType.LegacyNumericType#LONG}. * @throws IllegalArgumentException if the field name or type is null, or * if the field type does not have a LONG numericType() */ - public GeoPointField(String name, double lat, double lon, FieldType type) { + public GeoPointField(String name, double latitude, double longitude, FieldType type) { super(name, type); - if (GeoUtils.isValidLat(lat) == false) { - throw new IllegalArgumentException("invalid lat=" + lat + " for field \"" + name + "\""); + if (GeoUtils.isValidLat(latitude) == false) { + throw new IllegalArgumentException("invalid latitude=" + latitude + " for field \"" + name + "\""); } - if (GeoUtils.isValidLon(lon) == false) { - throw new IllegalArgumentException("invalid lon=" + lon + " for field \"" + name + "\""); + if (GeoUtils.isValidLon(longitude) == false) { + throw new IllegalArgumentException("invalid longitude=" + longitude + " for field \"" + name + "\""); } // field must be indexed @@ -190,7 +190,7 @@ public GeoPointField(String name, double lat, double lon, FieldType type) { } // set field data - fieldsData = GeoEncodingUtils.mortonHash(lat, lon); + fieldsData = GeoEncodingUtils.mortonHash(latitude, longitude); } private static FieldType getFieldType(Store stored) { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java index 2486852f87d7..98bb2cab2c94 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java @@ -21,7 +21,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoUtils; @@ -49,12 +48,34 @@ public class GeoPointDistanceQuery extends GeoPointInBBoxQuery { /** distance (in meters) from lat, lon center location */ protected final double radiusMeters; + // we must check these before passing to superclass or circleToBBox, or users can get a strange exception! + private static double checkRadius(double radiusMeters) { + if (Double.isFinite(radiusMeters) == false || radiusMeters < 0) { + throw new IllegalArgumentException("invalid radiusMeters " + radiusMeters); + } + return radiusMeters; + } + + private static double checkLatitude(double centerLat) { + if (GeoUtils.isValidLat(centerLat) == false) { + throw new IllegalArgumentException("invalid centerLat " + centerLat); + } + return centerLat; + } + + private static double checkLongitude(double centerLon) { + if (GeoUtils.isValidLon(centerLon) == false) { + throw new IllegalArgumentException("invalid centerLon " + centerLon); + } + return centerLon; + } + /** * Constructs a Query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types within a * distance (in meters) from a given point **/ public GeoPointDistanceQuery(final String field, final double centerLat, final double centerLon, final double radiusMeters) { - this(field, TermEncoding.PREFIX, centerLat, centerLon, radiusMeters); + this(field, TermEncoding.PREFIX, checkLatitude(centerLat), checkLongitude(centerLon), checkRadius(radiusMeters)); } /** @@ -63,34 +84,12 @@ public GeoPointDistanceQuery(final String field, final double centerLat, final d * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter **/ public GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, final double radiusMeters) { - this(field, termEncoding, GeoUtils.circleToBBox(centerLat, centerLon, radiusMeters), centerLat, centerLon, radiusMeters); + this(field, termEncoding, GeoUtils.circleToBBox(checkLatitude(centerLat), checkLongitude(centerLon), checkRadius(radiusMeters)), centerLat, centerLon, radiusMeters); } private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final GeoRect bbox, final double centerLat, final double centerLon, final double radiusMeters) { super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); - { - // check longitudinal overlap (restrict distance to maximum longitudinal radius) - // todo this restriction technically shouldn't be needed, - // its only purpose is to ensure the bounding box doesn't self overlap. - final double maxRadius = GeoDistanceUtils.maxRadialDistanceMeters(centerLat, centerLon); - if (radiusMeters > maxRadius) { - throw new IllegalArgumentException("radiusMeters " + radiusMeters + " exceeds maxRadius [" + maxRadius - + "] at location [" + centerLat + " " + centerLon + "]"); - } - } - - if (GeoUtils.isValidLat(centerLat) == false) { - throw new IllegalArgumentException("invalid centerLat " + centerLat); - } - - if (GeoUtils.isValidLon(centerLon) == false) { - throw new IllegalArgumentException("invalid centerLon " + centerLon); - } - - if (radiusMeters <= 0.0) { - throw new IllegalArgumentException("invalid radiusMeters " + radiusMeters); - } this.centerLat = centerLat; this.centerLon = centerLon; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java index f81f37ed9294..e9486d37c628 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java @@ -71,6 +71,24 @@ public GeoPointInBBoxQuery(final String field, final double minLat, final double * defined bounding box. Accepts optional {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter */ public GeoPointInBBoxQuery(final String field, final TermEncoding termEncoding, final double minLat, final double maxLat, final double minLon, final double maxLon) { + if (field == null) { + throw new IllegalArgumentException("field cannot be null"); + } + if (termEncoding == null) { + throw new IllegalArgumentException("termEncoding cannot be null"); + } + if (GeoUtils.isValidLat(minLat) == false) { + throw new IllegalArgumentException("invalid minimum latitude: " + minLat + ", must be -90 to 90"); + } + if (GeoUtils.isValidLat(maxLat) == false) { + throw new IllegalArgumentException("invalid maximum latitude: " + maxLat + ", must be -90 to 90"); + } + if (GeoUtils.isValidLon(minLon) == false) { + throw new IllegalArgumentException("invalid minimum longitude: " + minLon + ", must be -180 to 180"); + } + if (GeoUtils.isValidLon(maxLon) == false) { + throw new IllegalArgumentException("invalid maximum longitude: " + maxLon + ", must be -180 to 180"); + } this.field = field; this.minLat = minLat; this.maxLat = maxLat; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java deleted file mode 100644 index 1a412aca03e1..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoDistanceUtils.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.util; - -import org.apache.lucene.util.SloppyMath; - -/** - * Reusable geo-spatial distance utility methods. - * - * @lucene.experimental - */ -public class GeoDistanceUtils { - - // No instance: - private GeoDistanceUtils() { - } - - /** - * Compute the inverse haversine to determine distance in degrees longitude for provided distance in meters - * @param lat latitude to compute delta degrees lon - * @param distance distance in meters to convert to degrees lon - * @return Sloppy distance in degrees longitude for provided distance in meters - */ - public static double distanceToDegreesLon(double lat, double distance) { - // convert latitude to radians - lat = StrictMath.toRadians(lat); - - // get the diameter at the latitude - final double diameter = 2 * GeoUtils.SEMIMAJOR_AXIS; - - // compute inverse haversine - double a = StrictMath.sin(distance/diameter); - double h = StrictMath.min(1, a); - h *= h; - double cLat = StrictMath.cos(lat); - - return StrictMath.toDegrees(StrictMath.acos(1-((2d*h)/(cLat*cLat)))); - } - - /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */ - public static double maxRadialDistanceMeters(final double centerLat, final double centerLon) { - if (Math.abs(centerLat) == GeoUtils.MAX_LAT_INCL) { - return SloppyMath.haversinMeters(centerLat, centerLon, 0, centerLon); - } - return SloppyMath.haversinMeters(centerLat, centerLon, centerLat, (GeoUtils.MAX_LON_INCL + centerLon) % 360); - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java deleted file mode 100644 index 123769ee984e..000000000000 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointField.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.geopoint.search; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -/** Simple tests for GeoPoint */ -public class TestGeoPointField extends LuceneTestCase { - - private static Directory directory = null; - private static IndexReader reader = null; - private static IndexSearcher searcher = null; - private static final String FIELD_NAME = "point"; - - @BeforeClass - public static void beforeClass() throws Exception { - directory = newDirectory(); - - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random())) - .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) - .setMergePolicy(newLogMergePolicy())); - - // this is a simple systematic test - GeoPointField[] pts = new GeoPointField[] { - new GeoPointField(FIELD_NAME, 32.763420, -96.774, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.7559529921407, -96.7759895324707, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.77866942010977, -96.77701950073242, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.7756745755423, -96.7706036567688, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 27.703618681345585, -139.73458170890808, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, -96.4538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 33.06047141970814, -96.65084838867188, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.778650, -96.7772, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -88.56029371730983, -177.23537676036358, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 33.541429799076354, -26.779373834241003, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 26.774024500421728, -77.35379276106497, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -90.0, -14.796283808944777, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, -178.8538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, 178.8538113027811, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 40.720611, -73.998776, GeoPointField.PREFIX_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -44.5, -179.5, GeoPointField.PREFIX_TYPE_NOT_STORED)}; - - for (GeoPointField p : pts) { - Document doc = new Document(); - doc.add(p); - writer.addDocument(doc); - } - - // add explicit multi-valued docs - for (int i=0; i { - bboxQuery(-92.0, -91.0, 179.0, 181.0, 20); - }); - } - - public void testGeoDistanceQuery() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ - public void testNonEmptyTermsEnum() throws Exception { - TopDocs td = geoDistanceQuery(-88.56029371730983, -177.23537676036358, 7757.999232959935, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - public void testMultiValuedQuery() throws Exception { - TopDocs td = bboxQuery(32.7559529921407, 32.7756745755423, -96.4538113027811, -96.7706036567688, 20); - // 3 single valued docs + 2 multi-valued docs - assertEquals("testMultiValuedQuery failed", 5, td.totalHits); - } - - public void testTooBigRadius() throws Exception { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - geoDistanceQuery(85.0, 0.0, 4000000, 20); - }); - assertTrue(expected.getMessage().contains("exceeds maxRadius")); - } - - /** - * Explicitly large - */ - public void testGeoDistanceQueryHuge() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000000, 20); - assertEquals("GeoDistanceQuery failed", 16, td.totalHits); - } - - public void testGeoDistanceQueryCrossDateline() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -179.9538113027811, 120000, 20); - assertEquals("GeoDistanceQuery failed", 3, td.totalHits); - } - - // GeoDistanceQuery should not accept invalid lat/lon as origin - public void testInvalidGeoDistanceQuery() throws Exception { - expectThrows(Exception.class, () -> { - geoDistanceQuery(92.0, 181.0, 120000, 20); - }); - } - - public void testMaxDistanceRangeQuery() throws Exception { - TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); - assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); - } - - public void testInvalidLatLon() throws Exception { - IllegalArgumentException e; - e= expectThrows(IllegalArgumentException.class, - () -> { - new GeoPointField("field", 180.0, 0.0, Field.Store.NO); - }); - assertEquals("invalid lat=180.0 for field \"field\"", e.getMessage()); - - e = expectThrows(IllegalArgumentException.class, - () -> { - new GeoPointField("field", 0.0, 190.0, Field.Store.NO); - }); - assertEquals("invalid lon=190.0 for field \"field\"", e.getMessage()); - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 747cd1fdc2cc..1a739a3e3fdc 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -22,8 +22,6 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; -import org.apache.lucene.spatial.util.GeoDistanceUtils; -import org.apache.lucene.spatial.util.GeoRect; /** * random testing for GeoPoint query logic @@ -31,12 +29,6 @@ * @lucene.experimental */ public class TestGeoPointQuery extends BaseGeoPointTestCase { - - @Override - protected double maxRadius(double latitude, double longitude) { - // TODO: clean this up - return GeoDistanceUtils.maxRadialDistanceMeters(latitude, longitude); - } @Override protected double quantizeLat(double lat) { @@ -54,8 +46,8 @@ protected void addPointToDoc(String field, Document doc, double lat, double lon) } @Override - protected Query newRectQuery(String field, GeoRect rect) { - return new GeoPointInBBoxQuery(field, TermEncoding.PREFIX, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); + protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) { + return new GeoPointInBBoxQuery(field, TermEncoding.PREFIX, minLat, maxLat, minLon, maxLon); } @Override diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java deleted file mode 100644 index beab5b974774..000000000000 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointField.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.geopoint.search; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -/** Simple tests for GeoPoint with legacy numeric encoding - * @deprecated remove this when TermEncoding.NUMERIC is removed */ -@Deprecated -public class TestLegacyGeoPointField extends LuceneTestCase { - - private static Directory directory = null; - private static IndexReader reader = null; - private static IndexSearcher searcher = null; - private static final String FIELD_NAME = "point"; - - @BeforeClass - public static void beforeClass() throws Exception { - directory = newDirectory(); - - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random())) - .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) - .setMergePolicy(newLogMergePolicy())); - - // this is a simple systematic test - GeoPointField[] pts = new GeoPointField[] { - new GeoPointField(FIELD_NAME, 32.763420, -96.774, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.7559529921407, -96.7759895324707, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.77866942010977, -96.77701950073242, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.7756745755423, -96.7706036567688, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 27.703618681345585, -139.73458170890808, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, -96.4538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 33.06047141970814, -96.65084838867188, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.778650, -96.7772, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -88.56029371730983, -177.23537676036358, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 33.541429799076354, -26.779373834241003, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 26.774024500421728, -77.35379276106497, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -90.0, -14.796283808944777, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, -178.8538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 32.94823588839368, 178.8538113027811, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, 40.720611, -73.998776, GeoPointField.NUMERIC_TYPE_NOT_STORED), - new GeoPointField(FIELD_NAME, -44.5, -179.5, GeoPointField.NUMERIC_TYPE_NOT_STORED)}; - - for (GeoPointField p : pts) { - Document doc = new Document(); - doc.add(p); - writer.addDocument(doc); - } - - // add explicit multi-valued docs - for (int i=0; i { - bboxQuery(-92.0, -91.0, 179.0, 181.0, 20); - }); - } - - public void testGeoDistanceQuery() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - /** see https://issues.apache.org/jira/browse/LUCENE-6905 */ - public void testNonEmptyTermsEnum() throws Exception { - TopDocs td = geoDistanceQuery(-88.56029371730983, -177.23537676036358, 7757.999232959935, 20); - assertEquals("GeoDistanceQuery failed", 2, td.totalHits); - } - - public void testMultiValuedQuery() throws Exception { - TopDocs td = bboxQuery(32.7559529921407, 32.7756745755423, -96.4538113027811, -96.7706036567688, 20); - // 3 single valued docs + 2 multi-valued docs - assertEquals("testMultiValuedQuery failed", 5, td.totalHits); - } - - public void testTooBigRadius() throws Exception { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - geoDistanceQuery(85.0, 0.0, 4000000, 20); - }); - assertTrue(expected.getMessage().contains("exceeds maxRadius")); - } - - /** - * Explicitly large - */ - public void testGeoDistanceQueryHuge() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -96.4538113027811, 6000000, 20); - assertEquals("GeoDistanceQuery failed", 16, td.totalHits); - } - - public void testGeoDistanceQueryCrossDateline() throws Exception { - TopDocs td = geoDistanceQuery(32.94823588839368, -179.9538113027811, 120000, 20); - assertEquals("GeoDistanceQuery failed", 3, td.totalHits); - } - - // GeoDistanceQuery should not accept invalid lat/lon as origin - public void testInvalidGeoDistanceQuery() throws Exception { - expectThrows(Exception.class, () -> { - geoDistanceQuery(92.0, 181.0, 120000, 20); - }); - } - - public void testMaxDistanceRangeQuery() throws Exception { - TopDocs td = geoDistanceRangeQuery(0.0, 0.0, 10, 20000000, 20); - assertEquals("GeoDistanceRangeQuery failed", 24, td.totalHits); - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index 73b30826cfc5..c2f74f16deb6 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -22,8 +22,6 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; -import org.apache.lucene.spatial.util.GeoDistanceUtils; -import org.apache.lucene.spatial.util.GeoRect; /** * random testing for GeoPoint query logic (with deprecated numeric encoding) @@ -31,12 +29,6 @@ */ @Deprecated public class TestLegacyGeoPointQuery extends BaseGeoPointTestCase { - - @Override - protected double maxRadius(double latitude, double longitude) { - // TODO: clean this up - return GeoDistanceUtils.maxRadialDistanceMeters(latitude, longitude); - } @Override protected double quantizeLat(double lat) { @@ -54,8 +46,8 @@ protected void addPointToDoc(String field, Document doc, double lat, double lon) } @Override - protected Query newRectQuery(String field, GeoRect rect) { - return new GeoPointInBBoxQuery(field, TermEncoding.NUMERIC, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); + protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) { + return new GeoPointInBBoxQuery(field, TermEncoding.NUMERIC, minLat, maxLat, minLon, maxLon); } @Override diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 3943a92039ff..4d25451c7045 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -29,6 +29,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PointsReader; @@ -39,6 +40,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -65,8 +67,17 @@ import org.apache.lucene.util.bkd.BKDWriter; import org.junit.BeforeClass; -// TODO: cutover TestGeoUtils too? - +/** + * Abstract class to do basic tests for a geospatial impl (high level + * fields and queries) + * NOTE: This test focuses on geospatial (distance queries, polygon + * queries, etc) indexing and search, not any underlying storage + * format or encoding: it merely supplies two hooks for the encoding + * so that tests can be exact. The [stretch] goal is for this test to be + * so thorough in testing a new geo impl that if this + * test passes, then all Lucene/Solr tests should also pass. Ie, + * if there is some bug in a given geo impl that this + * test fails to catch then this test needs to be improved! */ public abstract class BaseGeoPointTestCase extends LuceneTestCase { protected static final String FIELD_NAME = "point"; @@ -109,6 +120,195 @@ public static double normalizeLat(double lat_deg) { double off = Math.abs((lat_deg + 90) % 360); return (off <= 180 ? off : 360-off) - 90; } + + /** Valid values that should not cause exception */ + public void testIndexExtremeValues() { + Document document = new Document(); + addPointToDoc("foo", document, 90.0, 180.0); + addPointToDoc("foo", document, 90.0, -180.0); + addPointToDoc("foo", document, -90.0, 180.0); + addPointToDoc("foo", document, -90.0, -180.0); + } + + /** Invalid values */ + public void testIndexOutOfRangeValues() { + Document document = new Document(); + IllegalArgumentException expected; + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, Math.nextUp(90.0), 50.0); + }); + assertTrue(expected.getMessage().contains("invalid latitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, Math.nextDown(-90.0), 50.0); + }); + assertTrue(expected.getMessage().contains("invalid latitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, 90.0, Math.nextUp(180.0)); + }); + assertTrue(expected.getMessage().contains("invalid longitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, 90.0, Math.nextDown(-180.0)); + }); + assertTrue(expected.getMessage().contains("invalid longitude")); + } + + /** NaN: illegal */ + public void testIndexNaNValues() { + Document document = new Document(); + IllegalArgumentException expected; + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, Double.NaN, 50.0); + }); + assertTrue(expected.getMessage().contains("invalid latitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, 50.0, Double.NaN); + }); + assertTrue(expected.getMessage().contains("invalid longitude")); + } + + /** Inf: illegal */ + public void testIndexInfValues() { + Document document = new Document(); + IllegalArgumentException expected; + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, Double.POSITIVE_INFINITY, 50.0); + }); + assertTrue(expected.getMessage().contains("invalid latitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, Double.NEGATIVE_INFINITY, 50.0); + }); + assertTrue(expected.getMessage().contains("invalid latitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, 50.0, Double.POSITIVE_INFINITY); + }); + assertTrue(expected.getMessage().contains("invalid longitude")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + addPointToDoc("foo", document, 50.0, Double.NEGATIVE_INFINITY); + }); + assertTrue(expected.getMessage().contains("invalid longitude")); + } + + /** Add a single point and search for it in a box */ + // NOTE: we don't currently supply an exact search, only ranges, because of the lossiness... + public void testBoxBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + assertEquals(1, searcher.count(newRectQuery("field", 18, 19, -66, -65))); + + reader.close(); + writer.close(); + dir.close(); + } + + + // box should not accept invalid lat/lon + public void testBoxInvalidCoordinates() throws Exception { + expectThrows(Exception.class, () -> { + newRectQuery("field", -92.0, -91.0, 179.0, 181.0); + }); + } + + /** test we can search for a point */ + public void testDistanceBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a location + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search within 50km and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + assertEquals(1, searcher.count(newDistanceQuery("field", 18, -65, 50_000))); + + reader.close(); + writer.close(); + dir.close(); + } + + /** distance query should not accept invalid lat/lon as origin */ + public void testDistanceIllegal() throws Exception { + expectThrows(Exception.class, () -> { + newDistanceQuery("field", 92.0, 181.0, 120000); + }); + } + /** negative distance queries are not allowed */ + public void testDistanceNegative() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newDistanceQuery("field", 18, 19, -1); + }); + assertTrue(expected.getMessage().contains("radiusMeters")); + assertTrue(expected.getMessage().contains("invalid")); + } + + /** NaN distance queries are not allowed */ + public void testDistanceNaN() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newDistanceQuery("field", 18, 19, Double.NaN); + }); + assertTrue(expected.getMessage().contains("radiusMeters")); + assertTrue(expected.getMessage().contains("invalid")); + } + + /** Inf distance queries are not allowed */ + public void testDistanceInf() { + IllegalArgumentException expected; + + expected = expectThrows(IllegalArgumentException.class, () -> { + newDistanceQuery("field", 18, 19, Double.POSITIVE_INFINITY); + }); + assertTrue(expected.getMessage().contains("radiusMeters")); + assertTrue(expected.getMessage().contains("invalid")); + + expected = expectThrows(IllegalArgumentException.class, () -> { + newDistanceQuery("field", 18, 19, Double.NEGATIVE_INFINITY); + }); + assertTrue(expected.getMessage(), expected.getMessage().contains("radiusMeters")); + assertTrue(expected.getMessage().contains("invalid")); + } + + /** test we can search for a polygon */ + public void testPolygonBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + assertEquals(1, searcher.count(newPolygonQuery("field", + new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }))); + + reader.close(); + writer.close(); + dir.close(); + } // A particularly tricky adversary for BKD tree: public void testSamePointManyTimes() throws Exception { @@ -275,7 +475,7 @@ public void testMultiValued() throws Exception { System.out.println("\nTEST: iter=" + iter + " rect=" + rect); } - Query query = newRectQuery(FIELD_NAME, rect); + Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); final FixedBitSet hits = new FixedBitSet(r.maxDoc()); s.search(query, new SimpleCollector() { @@ -455,10 +655,6 @@ protected double quantizeLat(double lat) { protected double quantizeLon(double lon) { return lon; } - - protected double maxRadius(double latitude, double longitude) { - return 50000000D; // bigger than earth, shouldnt matter - } protected GeoRect randomRect(boolean small, boolean canCrossDateLine) { double lat0 = randomLat(small); @@ -486,7 +682,7 @@ protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { protected abstract void addPointToDoc(String field, Document doc, double lat, double lon); - protected abstract Query newRectQuery(String field, GeoRect bbox); + protected abstract Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon); protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters); @@ -708,7 +904,7 @@ private void _run() throws Exception { // Rect: don't allow dateline crossing when testing small: final GeoRect rect = randomRect(small, small == false); - query = newRectQuery(FIELD_NAME, rect); + query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); verifyHits = new VerifyHits() { @Override @@ -875,7 +1071,7 @@ public void testRectBoundariesAreInclusive() throws Exception { } IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); - assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect))); + assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon))); r.close(); w.close(); dir.close(); @@ -936,7 +1132,7 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException for (int i = 0; i < numQueries; i++) { double lat = -90 + 180.0 * random().nextDouble(); double lon = -180 + 360.0 * random().nextDouble(); - double radius = maxRadius(lat, lon) * random().nextDouble(); + double radius = 50000000D * random().nextDouble(); BitSet expected = new BitSet(); for (int doc = 0; doc < reader.maxDoc(); doc++) { @@ -977,10 +1173,10 @@ public void testEquals() throws Exception { GeoRect rect = randomRect(false, true); - q1 = newRectQuery("field", rect); - q2 = newRectQuery("field", rect); + q1 = newRectQuery("field", rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); + q2 = newRectQuery("field", rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); assertEquals(q1, q2); - assertFalse(q1.equals(newRectQuery("field2", rect))); + assertFalse(q1.equals(newRectQuery("field2", rect.minLat, rect.maxLat, rect.minLon, rect.maxLon))); double lat = randomLat(false); double lon = randomLon(false); @@ -1014,4 +1210,132 @@ public void testEquals() throws Exception { assertEquals(q1, q2); assertFalse(q1.equals(newPolygonQuery("field2", lats, lons))); } + + /** return topdocs over a small set of points in field "point" */ + private TopDocs searchSmallSet(Query query, int size) throws Exception { + // this is a simple systematic test, indexing these points + double[][] pts = new double[][] { + { 32.763420, -96.774 }, + { 32.7559529921407, -96.7759895324707 }, + { 32.77866942010977, -96.77701950073242 }, + { 32.7756745755423, -96.7706036567688 }, + { 27.703618681345585, -139.73458170890808 }, + { 32.94823588839368, -96.4538113027811 }, + { 33.06047141970814, -96.65084838867188 }, + { 32.778650, -96.7772 }, + { -88.56029371730983, -177.23537676036358 }, + { 33.541429799076354, -26.779373834241003 }, + { 26.774024500421728, -77.35379276106497 }, + { -90.0, -14.796283808944777 }, + { 32.94823588839368, -178.8538113027811 }, + { 32.94823588839368, 178.8538113027811 }, + { 40.720611, -73.998776 }, + { -44.5, -179.5 } + }; + + Directory directory = newDirectory(); + + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, + newIndexWriterConfig(new MockAnalyzer(random())) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setMergePolicy(newLogMergePolicy())); + + for (double p[] : pts) { + Document doc = new Document(); + addPointToDoc("point", doc, p[0], p[1]); + writer.addDocument(doc); + } + + // add explicit multi-valued docs + for (int i=0; i Date: Thu, 24 Mar 2016 14:13:24 -0400 Subject: [PATCH 0207/1113] LUCENE-7136: remove threads from BaseGeoPointTestCase --- .../spatial/util/BaseGeoPointTestCase.java | 310 +++++++----------- 1 file changed, 126 insertions(+), 184 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 4d25451c7045..fd97207ef75a 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -19,14 +19,11 @@ import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; -import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.analysis.MockAnalyzer; @@ -325,7 +322,7 @@ public void testSamePointManyTimes() throws Exception { double[] lons = new double[numPoints]; Arrays.fill(lons, theLon); - verify(small, lats, lons, false); + verify(small, lats, lons); } public void testAllLatEqual() throws Exception { @@ -372,7 +369,7 @@ public void testAllLatEqual() throws Exception { lats[docID] = lat; } - verify(small, lats, lons, false); + verify(small, lats, lons); } public void testAllLonEqual() throws Exception { @@ -421,7 +418,7 @@ public void testAllLonEqual() throws Exception { lons[docID] = theLon; } - verify(small, lats, lons, false); + verify(small, lats, lons); } public void testMultiValued() throws Exception { @@ -535,25 +532,21 @@ public void collect(int doc) { public void testRandomTiny() throws Exception { // Make sure single-leaf-node case is OK: - doTestRandom(10, false); + doTestRandom(10); } public void testRandomMedium() throws Exception { - doTestRandom(10000, false); - } - - public void testRandomWithThreads() throws Exception { - doTestRandom(10000, true); + doTestRandom(10000); } @Nightly public void testRandomBig() throws Exception { assumeFalse("Direct codec can OOME on this test", TestUtil.getDocValuesFormat(FIELD_NAME).equals("Direct")); assumeFalse("Memory codec can OOME on this test", TestUtil.getDocValuesFormat(FIELD_NAME).equals("Memory")); - doTestRandom(200000, false); + doTestRandom(200000); } - private void doTestRandom(int count, boolean useThreads) throws Exception { + private void doTestRandom(int count) throws Exception { int numPoints = atLeast(count); @@ -621,7 +614,7 @@ private void doTestRandom(int count, boolean useThreads) throws Exception { } } - verify(small, lats, lons, useThreads); + verify(small, lats, lons); } public double randomLat(boolean small) { @@ -808,7 +801,7 @@ public void collect(int doc) { protected abstract void describe(int docID, double lat, double lon); } - protected void verify(boolean small, double[] lats, double[] lons, boolean useThreads) throws Exception { + protected void verify(boolean small, double[] lats, double[] lons) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); // Else we can get O(N^2) merging: int mbd = iwc.getMaxBufferedDocs(); @@ -852,189 +845,138 @@ protected void verify(boolean small, double[] lats, double[] lons, boolean useTh // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat: IndexSearcher s = newSearcher(r, false); - if (useThreads) { - // We must disable query cache otherwise test seed may not reproduce since different - // threads may or may not get a cache hit or miss depending on order the JVM - // schedules the threads: - s.setQueryCache(null); - } - - // Make sure queries are thread safe: - int numThreads; - if (useThreads) { - numThreads = TestUtil.nextInt(random(), 2, 5); - } else { - numThreads = 1; - } - - List threads = new ArrayList<>(); final int iters = atLeast(75); - final CountDownLatch startingGun = new CountDownLatch(1); final AtomicBoolean failed = new AtomicBoolean(); - for(int i=0;i tempDir.fileLength(tempFileName)) { throw new IllegalArgumentException("requested slice is beyond the length of this file: start=" + start + " length=" + length + " bytesPerDoc=" + bytesPerDoc + " fileLength=" + tempDir.fileLength(tempFileName) + " tempFileName=" + tempFileName); @@ -134,6 +136,29 @@ public void close() throws IOException { } } + @Override + public void markOrds(long count, LongBitSet ordBitSet) throws IOException { + if (countLeft < count) { + throw new IllegalStateException("only " + countLeft + " points remain, but " + count + " were requested"); + } + long fp = in.getFilePointer() + packedValue.length; + if (singleValuePerDoc == false) { + fp += Integer.BYTES; + } + for(long i=0;i Date: Fri, 25 Mar 2016 07:59:56 -0400 Subject: [PATCH 0209/1113] LUCENE-7139: fix bugs in geo3d's vincenty distance implementation --- lucene/CHANGES.txt | 3 + .../lucene/spatial3d/geom/PlanetModel.java | 96 ++++++++++--------- .../lucene/spatial3d/geom/GeoPointTest.java | 13 +++ 3 files changed, 67 insertions(+), 45 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 6dbacef7aab8..293ffeaf5d66 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -211,6 +211,9 @@ Bug Fixes * LUCENE-7111: DocValuesRangeQuery.newLongRange behaves incorrectly for Long.MAX_VALUE and Long.MIN_VALUE (Ishan Chattopadhyaya via Steve Rowe) +* LUCENE-7139: Fix bugs in geo3d's Vincenty surface distance + implementation (Karl Wright via Mike McCandless) + Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java index d45d7761a8c3..b905b56eb10d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java @@ -189,64 +189,70 @@ public boolean pointOutside(final double x, final double y, final double z) { } /** Compute surface distance between two points. - * @param p1 is the first point. - * @param p2 is the second point. + * @param pt1 is the first point. + * @param pt2 is the second point. * @return the adjusted angle, when multiplied by the mean earth radius, yields a surface distance. This will differ * from GeoPoint.arcDistance() only when the planet model is not a sphere. @see {@link GeoPoint#arcDistance(GeoPoint)} */ - public double surfaceDistance(final GeoPoint p1, final GeoPoint p2) { - final double latA = p1.getLatitude(); - final double lonA = p1.getLongitude(); - final double latB = p2.getLatitude(); - final double lonB = p2.getLongitude(); + public double surfaceDistance(final GeoPoint pt1, final GeoPoint pt2) { + final double L = pt2.getLongitude() - pt1.getLongitude(); + final double U1 = Math.atan((1.0-flattening) * Math.tan(pt1.getLatitude())); + final double U2 = Math.atan((1.0-flattening) * Math.tan(pt2.getLatitude())); + + final double sinU1 = Math.sin(U1); + final double cosU1 = Math.cos(U1); + final double sinU2 = Math.sin(U2); + final double cosU2 = Math.cos(U2); + + final double dCosU1CosU2 = cosU1 * cosU2; + final double dCosU1SinU2 = cosU1 * sinU2; + + final double dSinU1SinU2 = sinU1 * sinU2; + final double dSinU1CosU2 = sinU1 * cosU2; - final double L = lonB - lonA; - final double oF = 1.0 - this.flattening; - final double U1 = Math.atan(oF * Math.tan(latA)); - final double U2 = Math.atan(oF * Math.tan(latB)); - final double sU1 = Math.sin(U1); - final double cU1 = Math.cos(U1); - final double sU2 = Math.sin(U2); - final double cU2 = Math.cos(U2); - double sigma, sinSigma, cosSigma; - double cos2Alpha, cos2SigmaM; - double lambda = L; - double iters = 100; + double lambdaP = Math.PI * 2.0; + int iterLimit = 0; + double cosSqAlpha; + double sinSigma; + double cos2SigmaM; + double cosSigma; + double sigma; + double sinAlpha; + double C; + double sinLambda, cosLambda; do { - final double sinLambda = Math.sin(lambda); - final double cosLambda = Math.cos(lambda); - sinSigma = Math.sqrt((cU2 * sinLambda) * (cU2 * sinLambda) + (cU1 * sU2 - sU1 * cU2 * cosLambda) - * (cU1 * sU2 - sU1 * cU2 * cosLambda)); - if (Math.abs(sinSigma) < Vector.MINIMUM_RESOLUTION) - return 0.0; + sinLambda = Math.sin(lambda); + cosLambda = Math.cos(lambda); + sinSigma = Math.sqrt((cosU2*sinLambda) * (cosU2*sinLambda) + + (dCosU1SinU2 - dSinU1CosU2 * cosLambda) * (dCosU1SinU2 - dSinU1CosU2 * cosLambda)); - cosSigma = sU1 * sU2 + cU1 * cU2 * cosLambda; + if (sinSigma==0.0) { + return 0.0; + } + cosSigma = dSinU1SinU2 + dCosU1CosU2 * cosLambda; sigma = Math.atan2(sinSigma, cosSigma); - final double sinAlpha = cU1 * cU2 * sinLambda / sinSigma; - cos2Alpha = 1.0 - sinAlpha * sinAlpha; - cos2SigmaM = cosSigma - 2.0 * sU1 * sU2 / cos2Alpha; - - final double c = this.flattening * 0.625 * cos2Alpha * (4.0 + this.flattening * (4.0 - 3.0 * cos2Alpha)); - final double lambdaP = lambda; - lambda = L + (1.0 - c) * this.flattening * sinAlpha * (sigma + c * sinSigma * (cos2SigmaM + c * cosSigma * - (-1.0 + 2.0 * cos2SigmaM * cos2SigmaM))); - if (Math.abs(lambda - lambdaP) < Vector.MINIMUM_RESOLUTION) - break; - } while (--iters > 0); + sinAlpha = dCosU1CosU2 * sinLambda / sinSigma; + cosSqAlpha = 1.0 - sinAlpha * sinAlpha; + cos2SigmaM = cosSigma - 2.0 * dSinU1SinU2 / cosSqAlpha; - if (iters == 0) - return 0.0; + if (Double.isNaN(cos2SigmaM)) + cos2SigmaM = 0.0; // equatorial line: cosSqAlpha=0 + C = flattening / 16.0 * cosSqAlpha * (4.0 + flattening * (4.0 - 3.0 * cosSqAlpha)); + lambdaP = lambda; + lambda = L + (1.0 - C) * flattening * sinAlpha * + (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1.0 + 2.0 * cos2SigmaM *cos2SigmaM))); + } while (Math.abs(lambda-lambdaP) > Vector.MINIMUM_RESOLUTION && ++iterLimit < 40); - final double uSq = cos2Alpha * this.squareRatio; - final double A = 1.0 + uSq * 0.00006103515625 * (4096.0 + uSq * (-768.0 + uSq * (320.0 - 175.0 * uSq))); - final double B = uSq * 0.0009765625 * (256.0 + uSq * (-128.0 + uSq * (74.0 - 47.0 * uSq))); - final double deltaSigma = B * sinSigma * (cos2SigmaM + B * 0.25 * (cosSigma * (-1.0 + 2.0 * cos2SigmaM * cos2SigmaM) - B * 0.16666666666666666666667 * cos2SigmaM - * (-3.0 + 4.0 * sinSigma * sinSigma) * (-3.0 + 4.0 * cos2SigmaM * cos2SigmaM))); + final double uSq = cosSqAlpha * this.squareRatio; + final double A = 1.0 + uSq / 16384.0 * (4096.0 + uSq * (-768.0 + uSq * (320.0 - 175.0 * uSq))); + final double B = uSq / 1024.0 * (256.0 + uSq * (-128.0 + uSq * (74.0 - 47.0 * uSq))); + final double deltaSigma = B * sinSigma * (cos2SigmaM + B / 4.0 * (cosSigma * (-1.0 + 2.0 * cos2SigmaM * cos2SigmaM)- + B / 6.0 * cos2SigmaM * (-3.0 + 4.0 * sinSigma * sinSigma) * (-3.0 + 4.0 * cos2SigmaM * cos2SigmaM))); - return this.c * A * (sigma - deltaSigma); + return c * A * (sigma - deltaSigma); } @Override diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java index ed1792862be2..6df35c55c165 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java @@ -67,6 +67,19 @@ public void testSurfaceDistance() { // Compute ellipsoid distance; it should agree for a sphere final double surfaceDistance = PlanetModel.SPHERE.surfaceDistance(p1,p2); assertEquals(arcDistance, surfaceDistance, 1e-6); + // Now try some WGS84 points (taken randomly and compared against a known-good implementation) + assertEquals(1.1444648695765323, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.038203808753702884, -0.6701260455506466), + new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); + assertEquals(1.4345148695890722, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.5220926323378574, 0.6758041581907408), + new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); + assertEquals(2.32418144616446, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.09541335760967473, 1.2091829760623236), + new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); + assertEquals(2.018421047005435, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.3402853531962009, -0.43544195327249957), + new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); } } From 4d2be5d5696c787a07c61a89f9406052c3419de2 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 25 Mar 2016 08:21:44 -0400 Subject: [PATCH 0210/1113] LUCENE-7140: add spatial3d PlanetModel.bisection method --- lucene/CHANGES.txt | 3 +++ .../lucene/spatial3d/geom/PlanetModel.java | 24 +++++++++++++++++++ .../lucene/spatial3d/geom/GeoPointTest.java | 19 +++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 293ffeaf5d66..244609d1ad9e 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -10,6 +10,9 @@ New Features * LUCENE-7099: Add LatLonPoint.newDistanceSort to the sandbox's LatLonPoint. (Robert Muir) +* LUCENE-7140: Add PlanetModel.bisection to spatial3d (Karl Wright via + Mike McCandless) + Optimizations * LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java index b905b56eb10d..9f5385170cc8 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java @@ -188,6 +188,30 @@ public boolean pointOutside(final double x, final double y, final double z) { return (x * x + y * y) * inverseAb * inverseAb + z * z * inverseC * inverseC - 1.0 > Vector.MINIMUM_RESOLUTION; } + /** Compute a GeoPoint that's a bisection between two other GeoPoints. + * @param pt1 is the first point. + * @param pt2 is the second point. + * @return the bisection point, or null if a unique one cannot be found. + */ + public GeoPoint bisection(final GeoPoint pt1, final GeoPoint pt2) { + final double A0 = (pt1.x + pt2.x) * 0.5; + final double B0 = (pt1.y + pt2.y) * 0.5; + final double C0 = (pt1.z + pt2.z) * 0.5; + + final double denom = inverseAbSquared * A0 * A0 + + inverseAbSquared * B0 * B0 + + inverseCSquared * C0 * C0; + + if(denom < Vector.MINIMUM_RESOLUTION) { + // Bisection is undefined + return null; + } + + final double t = Math.sqrt(1.0 / denom); + + return new GeoPoint(t * A0, t * B0, t * C0); + } + /** Compute surface distance between two points. * @param pt1 is the first point. * @param pt2 is the second point. diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java index 6df35c55c165..d7381103ca02 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java @@ -83,6 +83,25 @@ public void testSurfaceDistance() { } } + @Test + public void testBisection() { + final int times = atLeast(100); + for (int i = 0; i < times; i++) { + final double p1Lat = (randomFloat() * 180.0 - 90.0) * DEGREES_TO_RADIANS; + final double p1Lon = (randomFloat() * 360.0 - 180.0) * DEGREES_TO_RADIANS; + final double p2Lat = (randomFloat() * 180.0 - 90.0) * DEGREES_TO_RADIANS; + final double p2Lon = (randomFloat() * 360.0 - 180.0) * DEGREES_TO_RADIANS; + final GeoPoint p1 = new GeoPoint(PlanetModel.WGS84, p1Lat, p1Lon); + final GeoPoint p2 = new GeoPoint(PlanetModel.WGS84, p2Lat, p2Lon); + final GeoPoint pMid = PlanetModel.WGS84.bisection(p1, p2); + if (pMid != null) { + final double arcDistance = p1.arcDistance(p2); + final double sum = pMid.arcDistance(p1) + pMid.arcDistance(p2); + assertEquals(arcDistance, sum, 1e-6); + } + } + } + @Test(expected = IllegalArgumentException.class) public void testBadLatLon() { new GeoPoint(PlanetModel.SPHERE, 50.0, 32.2); From 1d64c47aa79c29a8d31f5536b482ea17995a15eb Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 25 Mar 2016 08:24:08 -0400 Subject: [PATCH 0211/1113] LUCENE-7139: move these fixed tests out of the loop --- .../lucene/spatial3d/geom/GeoPointTest.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java index d7381103ca02..c19f9bfbfb4a 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPointTest.java @@ -67,20 +67,21 @@ public void testSurfaceDistance() { // Compute ellipsoid distance; it should agree for a sphere final double surfaceDistance = PlanetModel.SPHERE.surfaceDistance(p1,p2); assertEquals(arcDistance, surfaceDistance, 1e-6); - // Now try some WGS84 points (taken randomly and compared against a known-good implementation) - assertEquals(1.1444648695765323, PlanetModel.WGS84.surfaceDistance( - new GeoPoint(PlanetModel.WGS84, 0.038203808753702884, -0.6701260455506466), - new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); - assertEquals(1.4345148695890722, PlanetModel.WGS84.surfaceDistance( - new GeoPoint(PlanetModel.WGS84, 0.5220926323378574, 0.6758041581907408), - new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); - assertEquals(2.32418144616446, PlanetModel.WGS84.surfaceDistance( - new GeoPoint(PlanetModel.WGS84, 0.09541335760967473, 1.2091829760623236), - new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); - assertEquals(2.018421047005435, PlanetModel.WGS84.surfaceDistance( - new GeoPoint(PlanetModel.WGS84, 0.3402853531962009, -0.43544195327249957), - new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); } + + // Now try some WGS84 points (taken randomly and compared against a known-good implementation) + assertEquals(1.1444648695765323, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.038203808753702884, -0.6701260455506466), + new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); + assertEquals(1.4345148695890722, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.5220926323378574, 0.6758041581907408), + new GeoPoint(PlanetModel.WGS84, -0.8453720422675458, 0.1737353153814496)), 1e-6); + assertEquals(2.32418144616446, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.09541335760967473, 1.2091829760623236), + new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); + assertEquals(2.018421047005435, PlanetModel.WGS84.surfaceDistance( + new GeoPoint(PlanetModel.WGS84, 0.3402853531962009, -0.43544195327249957), + new GeoPoint(PlanetModel.WGS84, -0.8501591797459979, -2.3044806381627594)), 1e-6); } @Test From ca374d947678a81df665095387dadc0551590729 Mon Sep 17 00:00:00 2001 From: yonik Date: Fri, 25 Mar 2016 10:18:31 -0400 Subject: [PATCH 0212/1113] SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues --- solr/CHANGES.txt | 3 ++ .../component/RealTimeGetComponent.java | 13 ++++- .../solr/collection1/conf/schema15.xml | 1 + .../solr/collection1/conf/schema_latest.xml | 51 ++++++++++++------- .../apache/solr/search/TestRealTimeGet.java | 49 ++++++++++++++++-- 5 files changed, 93 insertions(+), 24 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a9731e16f9a2..4bd24df58851 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -351,6 +351,9 @@ Bug Fixes * SOLR-8891: Fix StrField.toObject and toExternal to work with docValue IndexableField instances. (yonik) +* SOLR-8865: Real-time get sometimes fails to retrieve stored fields from docValues. + (Ishan Chattopadhyaya, yonik) + Optimizations ---------------------- * SOLR-7876: Speed up queries and operations that use many terms when timeAllowed has not been diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index 2f71b3e8a37c..8332c797fa66 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.lucene.document.Document; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -258,6 +259,7 @@ public void process(ResponseBuilder rb) throws IOException if (docid < 0) continue; Document luceneDocument = searcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames()); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); + searcher.decorateDocValueFields(doc, docid, searcher.getNonStoredDVs(true)); if( transformer != null ) { transformer.transform(doc, docid, 0); } @@ -408,8 +410,15 @@ private static SolrDocument toSolrDoc(SolrInputDocument sdoc, IndexSchema schema // copy the stored fields only Document out = new Document(); for (IndexableField f : doc.getFields()) { - if (f.fieldType().stored() ) { - out.add((IndexableField) f); + if (f.fieldType().stored()) { + out.add(f); + } else if (f.fieldType().docValuesType() != DocValuesType.NONE) { + SchemaField schemaField = schema.getFieldOrNull(f.name()); + if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored()) { + out.add(f); + } + } else { + log.debug("Don't know how to handle field " + f); } } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index ea46d85553d4..55ec3f1ac602 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -530,6 +530,7 @@ + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index ef82aa189b70..cae373cdd92c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -110,7 +110,7 @@ - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + @@ -230,10 +251,6 @@ - - - - @@ -258,8 +275,8 @@ - + @@ -695,7 +712,7 @@ - + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + ${solr.hdfs.blockcache.enabled:true} + ${solr.hdfs.blockcache.blocksperbank:1024} + ${solr.hdfs.home:} + ${solr.hdfs.confdir:} + ${solr.hdfs.blockcache.global:false} + + + ${solr.data.dir:} + + + + + + + + + + + + + + + true + + + + + + + + + + ${solr.ulog.dir:} + + + + + + + 10 + + + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml new file mode 100644 index 000000000000..d3b90db50f60 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tolerant-update-minimal.xml @@ -0,0 +1,40 @@ + + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + explicit + true + text + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index d0b547241c3a..bb1cbcfc6139 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -26,6 +26,7 @@ ${tests.luceneMatchVersion:LATEST} + @@ -628,4 +629,20 @@ + + + 10 + + + + + + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java new file mode 100644 index 000000000000..054c0745fd46 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java @@ -0,0 +1,1065 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.io.File; +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.ToleratedUpdateError; +import org.apache.solr.common.ToleratedUpdateError.CmdType; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.util.RevertDefaultThreadHandlerRule; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test of TolerantUpdateProcessor using a MiniSolrCloud. Updates (that include failures which + * should be tolerated) are explicitly tested against various initial nodes to confirm correct + * behavior regardless of routing. + * + *

    + * NOTE: This test sets up a static instance of MiniSolrCloud with a single collection + * and several clients pointed at specific nodes. These are all re-used across multiple test methods, + * and assumes that the state of the cluster is healthy. + *

    + * + */ +public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final int NUM_SHARDS = 2; + private static final int REPLICATION_FACTOR = 2; + private static final int NUM_SERVERS = 5; + + private static final String COLLECTION_NAME = "test_col"; + + /** A basic client for operations at the cloud level, default collection will be set */ + private static CloudSolrClient CLOUD_CLIENT; + + /** A client for talking directly to the leader of shard1 */ + private static HttpSolrClient S_ONE_LEADER_CLIENT; + + /** A client for talking directly to the leader of shard2 */ + private static HttpSolrClient S_TWO_LEADER_CLIENT; + + /** A client for talking directly to a passive replica of shard1 */ + private static HttpSolrClient S_ONE_NON_LEADER_CLIENT; + + /** A client for talking directly to a passive replica of shard2 */ + private static HttpSolrClient S_TWO_NON_LEADER_CLIENT; + + /** A client for talking directly to a node that has no piece of the collection */ + private static HttpSolrClient NO_COLLECTION_CLIENT; + + /** id field doc routing prefix for shard1 */ + private static final String S_ONE_PRE = "abc!"; + + /** id field doc routing prefix for shard2 */ + private static final String S_TWO_PRE = "XYZ!"; + + @BeforeClass + private static void createMiniSolrCloudCluster() throws Exception { + + final String configName = "solrCloudCollectionConfig"; + final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf"); + + configureCluster(NUM_SERVERS) + .addConfig(configName, configDir.toPath()) + .configure(); + assertSpinLoopAllJettyAreRunning(cluster); + + Map collectionProperties = new HashMap<>(); + collectionProperties.put("config", "solrconfig-distrib-update-processor-chains.xml"); + collectionProperties.put("schema", "schema15.xml"); // string id for doc routing prefix + + assertNotNull(cluster.createCollection(COLLECTION_NAME, NUM_SHARDS, REPLICATION_FACTOR, + configName, null, null, collectionProperties)); + + CLOUD_CLIENT = cluster.getSolrClient(); + CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME); + + ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader(); + AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330); + + + // really hackish way to get a URL for specific nodes based on shard/replica hosting + // inspired by TestMiniSolrCloudCluster + HashMap urlMap = new HashMap<>(); + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + URL jettyURL = jetty.getBaseUrl(); + String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/","_"); + urlMap.put(nodeKey, jettyURL.toString()); + } + zkStateReader.updateClusterState(); + ClusterState clusterState = zkStateReader.getClusterState(); + for (Slice slice : clusterState.getSlices(COLLECTION_NAME)) { + String shardName = slice.getName(); + Replica leader = slice.getLeader(); + assertNotNull("slice has null leader: " + slice.toString(), leader); + assertNotNull("slice leader has null node name: " + slice.toString(), leader.getNodeName()); + String leaderUrl = urlMap.remove(leader.getNodeName()); + assertNotNull("could not find URL for " + shardName + " leader: " + leader.getNodeName(), + leaderUrl); + assertEquals("expected two total replicas for: " + slice.getName(), + 2, slice.getReplicas().size()); + + String passiveUrl = null; + + for (Replica replica : slice.getReplicas()) { + if ( ! replica.equals(leader)) { + passiveUrl = urlMap.remove(replica.getNodeName()); + assertNotNull("could not find URL for " + shardName + " replica: " + replica.getNodeName(), + passiveUrl); + } + } + assertNotNull("could not find URL for " + shardName + " replica", passiveUrl); + + if (shardName.equals("shard1")) { + S_ONE_LEADER_CLIENT = new HttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); + S_ONE_NON_LEADER_CLIENT = new HttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); + } else if (shardName.equals("shard2")) { + S_TWO_LEADER_CLIENT = new HttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); + S_TWO_NON_LEADER_CLIENT = new HttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); + } else { + fail("unexpected shard: " + shardName); + } + } + assertEquals("Should be exactly one server left (nost hosting either shard)", 1, urlMap.size()); + NO_COLLECTION_CLIENT = new HttpSolrClient(urlMap.values().iterator().next() + + "/" + COLLECTION_NAME + "/"); + + assertNotNull(S_ONE_LEADER_CLIENT); + assertNotNull(S_TWO_LEADER_CLIENT); + assertNotNull(S_ONE_NON_LEADER_CLIENT); + assertNotNull(S_TWO_NON_LEADER_CLIENT); + assertNotNull(NO_COLLECTION_CLIENT); + + // sanity check that our S_ONE_PRE & S_TWO_PRE really do map to shard1 & shard2 with default routing + assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_ONE_PRE + random().nextInt()), + f("expected_shard_s", "shard1"))).getStatus()); + assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_TWO_PRE + random().nextInt()), + f("expected_shard_s", "shard2"))).getStatus()); + assertEquals(0, CLOUD_CLIENT.commit().getStatus()); + SolrDocumentList docs = CLOUD_CLIENT.query(params("q", "*:*", + "fl","id,expected_shard_s,[shard]")).getResults(); + assertEquals(2, docs.getNumFound()); + assertEquals(2, docs.size()); + for (SolrDocument doc : docs) { + String expected = COLLECTION_NAME + "_" + doc.getFirstValue("expected_shard_s") + "_replica"; + String docShard = doc.getFirstValue("[shard]").toString(); + assertTrue("shard routing prefixes don't seem to be aligned anymore, " + + "did someone change the default routing rules? " + + "and/or the the default core name rules? " + + "and/or the numShards used by this test? ... " + + "couldn't find " + expected + " as substring of [shard] == '" + docShard + + "' ... for docId == " + doc.getFirstValue("id"), + docShard.contains(expected)); + } + } + + @Before + private void clearCollection() throws Exception { + assertEquals(0, CLOUD_CLIENT.deleteByQuery("*:*").getStatus()); + assertEquals(0, CLOUD_CLIENT.commit().getStatus()); + } + + public void testSanity() throws Exception { + + // verify some basic sanity checking of indexing & querying across the collection + // w/o using our custom update processor chain + + assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_ONE_PRE + "1"), + f("foo_i", 42))).getStatus()); + assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_TWO_PRE + "2"), + f("foo_i", 66))).getStatus()); + assertEquals(0, CLOUD_CLIENT.commit().getStatus()); + + for (SolrClient c : Arrays.asList(S_ONE_LEADER_CLIENT, S_TWO_LEADER_CLIENT, + S_ONE_NON_LEADER_CLIENT, S_TWO_NON_LEADER_CLIENT, + NO_COLLECTION_CLIENT, CLOUD_CLIENT)) { + assertQueryDocIds(c, true, S_ONE_PRE + "1", S_TWO_PRE + "2"); + assertQueryDocIds(c, false, "id_not_exists"); + + // verify adding 2 broken docs causes a clint exception + try { + UpdateResponse rsp = update(params(), + doc(f("id", S_ONE_PRE + "X"), f("foo_i", "bogus_val_X")), + doc(f("id", S_TWO_PRE + "Y"), f("foo_i", "bogus_val_Y")) + ).process(c); + fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString()); + } catch (SolrException e) { + assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(), + 400, e.code()); + } + + // verify malformed deleteByQuerys fail + try { + UpdateResponse rsp = update(params()).deleteByQuery("foo_i:not_a_num").process(c); + fail("sanity check for malformed DBQ didn't fail: " + rsp.toString()); + } catch (SolrException e) { + assertEquals("not the expected DBQ failure: " + e.getMessage(), 400, e.code()); + } + + // verify oportunistic concurrency deletions fail as we expect when docs are / aren't present + for (UpdateRequest r : new UpdateRequest[] { + update(params("commit", "true")).deleteById(S_ONE_PRE + "1", -1L), + update(params("commit", "true")).deleteById(S_TWO_PRE + "2", -1L), + update(params("commit", "true")).deleteById("id_not_exists", 1L) }) { + try { + UpdateResponse rsp = r.process(c); + fail("sanity check for oportunistic concurrency delete didn't fail: " + + r.toString() + " => " + rsp.toString()); + } catch (SolrException e) { + assertEquals("not the expected oportunistic concurrency failure code: " + + r.toString() + " => " + e.getMessage(), 409, e.code()); + } + } + } + } + + // + public void testVariousDeletesViaCloudClient() throws Exception { + testVariousDeletes(CLOUD_CLIENT); + } + public void testVariousDeletesViaShard1LeaderClient() throws Exception { + testVariousDeletes(S_ONE_LEADER_CLIENT); + } + public void testVariousDeletesViaShard2LeaderClient() throws Exception { + testVariousDeletes(S_TWO_LEADER_CLIENT); + } + public void testVariousDeletesViaShard1NonLeaderClient() throws Exception { + testVariousDeletes(S_ONE_NON_LEADER_CLIENT); + } + public void testVariousDeletesViaShard2NonLeaderClient() throws Exception { + testVariousDeletes(S_TWO_NON_LEADER_CLIENT); + } + public void testVariousDeletesViaNoCollectionClient() throws Exception { + testVariousDeletes(NO_COLLECTION_CLIENT); + } + + protected static void testVariousDeletes(SolrClient client) throws Exception { + assertNotNull("client not initialized", client); + + // 2 docs, one on each shard + final String docId1 = S_ONE_PRE + "42"; + final String docId2 = S_TWO_PRE + "666"; + + UpdateResponse rsp = null; + + // add 1 doc to each shard + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", docId1), f("foo_i", "2001")), + doc(f("id", docId2), f("foo_i", "1976"))).process(client); + assertEquals(0, rsp.getStatus()); + + // attempt to delete individual doc id(s) that should fail because of oportunistic concurrency constraints + for (String id : new String[] { docId1, docId2 }) { + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true")).deleteById(id, -1L).process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delId="+id, rsp, + delIErr(id)); + } + + // multiple failed deletes from the same shard (via oportunistic concurrent w/ bogus ids) + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true") + ).deleteById(S_ONE_PRE + "X", +1L).deleteById(S_ONE_PRE + "Y", +1L).process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by id for 2 bogus docs", rsp, + delIErr(S_ONE_PRE + "X"), delIErr(S_ONE_PRE + "Y")); + assertQueryDocIds(client, true, docId1, docId2); + + // multiple failed deletes from the diff shards due to oportunistic concurrency constraints + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true")).deleteById(docId2, -1L).deleteById(docId1, -1L).process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by id for 2 docs", rsp, + delIErr(docId1), delIErr(docId2)); + assertQueryDocIds(client, true, docId1, docId2); + + // deleteByQuery using malformed query (fail) + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true")).deleteByQuery("bogus_field:foo").process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by query", rsp, + delQErr("bogus_field:foo")); + assertQueryDocIds(client, true, docId1, docId2); + + // mix 2 deleteByQuery, one malformed (fail), one that doesn't match anything (ok) + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true") + ).deleteByQuery("bogus_field:foo").deleteByQuery("foo_i:23").process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by query", rsp, + delQErr("bogus_field:foo")); + assertQueryDocIds(client, true, docId1, docId2); + + // mix 2 deleteById using _version_=-1, one for real doc1 (fail), one for bogus id (ok) + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true") + ).deleteById(docId1, -1L).deleteById("bogus", -1L).process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by id: exists", rsp, + delIErr(docId1)); + assertQueryDocIds(client, true, docId1, docId2); + + // mix 2 deleteById using _version_=1, one for real doc1 (ok, deleted), one for bogus id (fail) + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true") + ).deleteById(docId1, +1L).deleteById("bogusId", +1L).process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by id: bogus", rsp, + delIErr("bogusId")); + assertQueryDocIds(client, false, docId1); + assertQueryDocIds(client, true, docId2); + + // mix 2 deleteByQuery, one malformed (fail), one that alctaully removes some docs (ok) + assertQueryDocIds(client, true, docId2); + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true") + ).deleteByQuery("bogus_field:foo").deleteByQuery("foo_i:1976").process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed oportunistic concurrent delete by query", rsp, + delQErr("bogus_field:foo")); + assertQueryDocIds(client, false, docId2); + + } + + + // + public void testVariousAddsViaCloudClient() throws Exception { + testVariousAdds(CLOUD_CLIENT); + } + public void testVariousAddsViaShard1LeaderClient() throws Exception { + testVariousAdds(S_ONE_LEADER_CLIENT); + } + public void testVariousAddsViaShard2LeaderClient() throws Exception { + testVariousAdds(S_TWO_LEADER_CLIENT); + } + public void testVariousAddsViaShard1NonLeaderClient() throws Exception { + testVariousAdds(S_ONE_NON_LEADER_CLIENT); + } + public void testVariousAddsViaShard2NonLeaderClient() throws Exception { + testVariousAdds(S_TWO_NON_LEADER_CLIENT); + } + public void testVariousAddsViaNoCollectionClient() throws Exception { + testVariousAdds(NO_COLLECTION_CLIENT); + } + + protected static void testVariousAdds(SolrClient client) throws Exception { + assertNotNull("client not initialized", client); + + UpdateResponse rsp = null; + + // 2 docs that are both on shard1, the first one should fail + for (int maxErrors : new int[] { -1, 2, 47, 10 }) { + // regardless of which of these maxErrors values we use, behavior should be the same... + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "maxErrors", ""+maxErrors, + "commit", "true"), + doc(f("id", S_ONE_PRE + "42"), f("foo_i", "bogus_value")), + doc(f("id", S_ONE_PRE + "666"), f("foo_i", "1976"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("single shard, 1st doc should fail", rsp, S_ONE_PRE + "42"); + assertEquals(0, client.commit().getStatus()); + assertQueryDocIds(client, false, S_ONE_PRE + "42"); + assertQueryDocIds(client, true, S_ONE_PRE + "666"); + + // ...only diff should be that we get an accurate report of the effective maxErrors + assertEquals(maxErrors, rsp.getResponseHeader().get("maxErrors")); + } + + // 2 docs that are both on shard1, the second one should fail + + rsp = update(params("update.chain", "tolerant-chain-max-errors-not-set", + "commit", "true"), + doc(f("id", S_ONE_PRE + "55"), f("foo_i", "1976")), + doc(f("id", S_ONE_PRE + "77"), f("foo_i", "bogus_val"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("single shard, 2nd doc should fail", rsp, S_ONE_PRE + "77"); + assertQueryDocIds(client, false, S_ONE_PRE + "77"); + assertQueryDocIds(client, true, S_ONE_PRE + "666", S_ONE_PRE + "55"); + // since maxErrors is unset, we should get an "unlimited" value back + assertEquals(-1, rsp.getResponseHeader().get("maxErrors")); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // 2 docs on 2 diff shards, first of which should fail + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", S_ONE_PRE + "42"), f("foo_i", "bogus_value")), + doc(f("id", S_TWO_PRE + "666"), f("foo_i", "1976"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("two shards, 1st doc should fail", rsp, S_ONE_PRE + "42"); + assertEquals(0, client.commit().getStatus()); + assertQueryDocIds(client, false, S_ONE_PRE + "42"); + assertQueryDocIds(client, true, S_TWO_PRE + "666"); + + // 2 docs on 2 diff shards, second of which should fail + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", S_ONE_PRE + "55"), f("foo_i", "1976")), + doc(f("id", S_TWO_PRE + "77"), f("foo_i", "bogus_val"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("two shards, 2nd doc should fail", rsp, S_TWO_PRE + "77"); + assertQueryDocIds(client, false, S_TWO_PRE + "77"); + assertQueryDocIds(client, true, S_TWO_PRE + "666", S_ONE_PRE + "55"); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, 1 from each shard should fail + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", S_ONE_PRE + "11")), + doc(f("id", S_TWO_PRE + "21")), + doc(f("id", S_ONE_PRE + "12")), + doc(f("id", S_TWO_PRE + "22"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "13")), + doc(f("id", S_TWO_PRE + "23")), + doc(f("id", S_ONE_PRE + "14")), + doc(f("id", S_TWO_PRE + "24")), + doc(f("id", S_ONE_PRE + "15"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "25")), + doc(f("id", S_ONE_PRE + "16")), + doc(f("id", S_TWO_PRE + "26"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("many docs, 1 from each shard should fail", rsp, + S_ONE_PRE + "15", + S_TWO_PRE + "22"); + assertQueryDocIds(client, false, S_TWO_PRE + "22", S_ONE_PRE + "15"); + assertQueryDocIds(client, true, + S_ONE_PRE + "11", S_TWO_PRE + "21", S_ONE_PRE + "12", + S_ONE_PRE + "13", S_TWO_PRE + "23", S_ONE_PRE + "14", S_TWO_PRE + "24", + S_TWO_PRE + "25", S_ONE_PRE + "16", S_TWO_PRE + "26"); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, 1 from each shard should fail and 1 w/o uniqueKey + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", S_ONE_PRE + "11")), + doc(f("id", S_TWO_PRE + "21")), + doc(f("id", S_ONE_PRE + "12")), + doc(f("id", S_TWO_PRE + "22"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "13")), + doc(f("id", S_TWO_PRE + "23")), + doc(f("foo_i", "42")), // no "id" + doc(f("id", S_ONE_PRE + "14")), + doc(f("id", S_TWO_PRE + "24")), + doc(f("id", S_ONE_PRE + "15"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "25")), + doc(f("id", S_ONE_PRE + "16")), + doc(f("id", S_TWO_PRE + "26"))).process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantAddErrors("many docs, 1 from each shard (+ no id) should fail", rsp, + S_ONE_PRE + "15", + "(unknown)", + S_TWO_PRE + "22"); + assertQueryDocIds(client, false, S_TWO_PRE + "22", S_ONE_PRE + "15"); + assertQueryDocIds(client, true, + S_ONE_PRE + "11", S_TWO_PRE + "21", S_ONE_PRE + "12", + S_ONE_PRE + "13", S_TWO_PRE + "23", S_ONE_PRE + "14", S_TWO_PRE + "24", + S_TWO_PRE + "25", S_ONE_PRE + "16", S_TWO_PRE + "26"); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, more then 10 (total) should fail + + try { + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", S_ONE_PRE + "11")), + doc(f("id", S_TWO_PRE + "21"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "12")), + doc(f("id", S_TWO_PRE + "22"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "13")), + doc(f("id", S_TWO_PRE + "23"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "14"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "24")), + doc(f("id", S_ONE_PRE + "15"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "25")), + doc(f("id", S_ONE_PRE + "16"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "26"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "17")), + doc(f("id", S_TWO_PRE + "27")), + doc(f("id", S_ONE_PRE + "18"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "28"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "19"), f("foo_i", "bogus_val")), + doc(f("id", S_TWO_PRE + "29"), f("foo_i", "bogus_val")), + doc(f("id", S_ONE_PRE + "10")), // may be skipped, more then 10 fails + doc(f("id", S_TWO_PRE + "20")) // may be skipped, more then 10 fails + ).process(client); + + fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString()); + } catch (SolrException e) { + // we can't make any reliable assertions about the error message, because + // it varies based on how the request was routed -- see SOLR-8830 + assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(), + // NOTE: we always expect a 400 because we know that's what we would get from these types of errors + // on a single node setup -- a 5xx type error isn't something we should have triggered + 400, e.code()); + + // verify that the Exceptions metadata can tell us what failed. + NamedList remoteErrMetadata = e.getMetadata(); + assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata); + Set actualKnownErrs + = new LinkedHashSet(remoteErrMetadata.size()); + int actualKnownErrsCount = 0; + for (int i = 0; i < remoteErrMetadata.size(); i++) { + ToleratedUpdateError err = + ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i), + remoteErrMetadata.getVal(i)); + if (null == err) { + // some metadata unrelated to this update processor + continue; + } + actualKnownErrsCount++; + actualKnownErrs.add(err); + } + assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(), + 11, actualKnownErrsCount); + assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(), + actualKnownErrsCount, actualKnownErrs.size()); + for (ToleratedUpdateError err : actualKnownErrs) { + assertEquals("only expected type of error is ADD: " + err, + CmdType.ADD, err.getType()); + assertTrue("failed err msg didn't match expected value: " + err, + err.getMessage().contains("bogus_val")); + } + } + assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish + assertQueryDocIds(client, false + // explicitly failed + , S_TWO_PRE + "21", S_TWO_PRE + "22", S_TWO_PRE + "23", S_ONE_PRE + "14" + , S_ONE_PRE + "15", S_ONE_PRE + "16", S_TWO_PRE + "26", S_ONE_PRE + "18" + , S_TWO_PRE + "28", S_ONE_PRE + "19", S_TWO_PRE + "29" + // + // // we can't assert for sure these docs were skipped + // // depending on shard we hit, they may have been added async before errors were exceeded + // , S_ONE_PRE + "10", S_TWO_PRE + "20" // skipped + ); + assertQueryDocIds(client, true, + S_ONE_PRE + "11", S_ONE_PRE + "12", S_ONE_PRE + "13", S_TWO_PRE + "24", + S_TWO_PRE + "25", S_ONE_PRE + "17", S_TWO_PRE + "27"); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, more then 10 from a single shard (two) should fail + + try { + ArrayList docs = new ArrayList(30); + docs.add(doc(f("id", S_ONE_PRE + "z"))); + docs.add(doc(f("id", S_TWO_PRE + "z"))); + docs.add(doc(f("id", S_ONE_PRE + "y"))); + docs.add(doc(f("id", S_TWO_PRE + "y"))); + for (int i = 0; i < 11; i++) { + docs.add(doc(f("id", S_ONE_PRE + i))); + docs.add(doc(f("id", S_TWO_PRE + i), f("foo_i", "bogus_val"))); + } + docs.add(doc(f("id", S_ONE_PRE + "x"))); // may be skipped, more then 10 fails + docs.add(doc(f("id", S_TWO_PRE + "x"))); // may be skipped, more then 10 fails + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + docs.toArray(new SolrInputDocument[docs.size()])).process(client); + + fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString()); + } catch (SolrException e) { + // we can't make any reliable assertions about the error message, because + // it varies based on how the request was routed -- see SOLR-8830 + assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(), + // NOTE: we always expect a 400 because we know that's what we would get from these types of errors + // on a single node setup -- a 5xx type error isn't something we should have triggered + 400, e.code()); + + // verify that the Exceptions metadata can tell us what failed. + NamedList remoteErrMetadata = e.getMetadata(); + assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata); + Set actualKnownErrs + = new LinkedHashSet(remoteErrMetadata.size()); + int actualKnownErrsCount = 0; + for (int i = 0; i < remoteErrMetadata.size(); i++) { + ToleratedUpdateError err = + ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i), + remoteErrMetadata.getVal(i)); + if (null == err) { + // some metadata unrelated to this update processor + continue; + } + actualKnownErrsCount++; + actualKnownErrs.add(err); + } + assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(), + 11, actualKnownErrsCount); + assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(), + actualKnownErrsCount, actualKnownErrs.size()); + for (ToleratedUpdateError err : actualKnownErrs) { + assertEquals("only expected type of error is ADD: " + err, + CmdType.ADD, err.getType()); + assertTrue("failed id had unexpected prefix: " + err, + err.getId().startsWith(S_TWO_PRE)); + assertTrue("failed err msg didn't match expected value: " + err, + err.getMessage().contains("bogus_val")); + } + + } + assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish + assertQueryDocIds(client, true + , S_ONE_PRE + "z", S_ONE_PRE + "y", S_TWO_PRE + "z", S_TWO_PRE + "y" // first + // + , S_ONE_PRE + "0", S_ONE_PRE + "1", S_ONE_PRE + "2", S_ONE_PRE + "3", S_ONE_PRE + "4" + , S_ONE_PRE + "5", S_ONE_PRE + "6", S_ONE_PRE + "7", S_ONE_PRE + "8", S_ONE_PRE + "9" + ); + assertQueryDocIds(client, false + // explicitly failed + , S_TWO_PRE + "0", S_TWO_PRE + "1", S_TWO_PRE + "2", S_TWO_PRE + "3", S_TWO_PRE + "4" + , S_TWO_PRE + "5", S_TWO_PRE + "6", S_TWO_PRE + "7", S_TWO_PRE + "8", S_TWO_PRE + "9" + // + // // we can't assert for sure these docs were skipped + // // depending on shard we hit, they may have been added async before errors were exceeded + // , S_ONE_PRE + "x", S_TWO_PRE + "x", // skipped + ); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, more then 10 don't have any uniqueKey specified + + try { + ArrayList docs = new ArrayList(30); + docs.add(doc(f("id", S_ONE_PRE + "z"))); + docs.add(doc(f("id", S_TWO_PRE + "z"))); + docs.add(doc(f("id", S_ONE_PRE + "y"))); + docs.add(doc(f("id", S_TWO_PRE + "y"))); + for (int i = 0; i < 11; i++) { + // no "id" field + docs.add(doc(f("foo_i", "" + i))); + } + docs.add(doc(f("id", S_ONE_PRE + "x"))); // may be skipped, more then 10 fails + docs.add(doc(f("id", S_TWO_PRE + "x"))); // may be skipped, more then 10 fails + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + docs.toArray(new SolrInputDocument[docs.size()])).process(client); + + fail("did not get a top level exception when more then 10 docs mising uniqueKey: " + rsp.toString()); + } catch (SolrException e) { + // we can't make any reliable assertions about the error message, because + // it varies based on how the request was routed -- see SOLR-8830 + assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(), + // NOTE: we always expect a 400 because we know that's what we would get from these types of errors + // on a single node setup -- a 5xx type error isn't something we should have triggered + 400, e.code()); + + // verify that the Exceptions metadata can tell us what failed. + NamedList remoteErrMetadata = e.getMetadata(); + assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata); + int actualKnownErrsCount = 0; + for (int i = 0; i < remoteErrMetadata.size(); i++) { + ToleratedUpdateError err = + ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i), + remoteErrMetadata.getVal(i)); + if (null == err) { + // some metadata unrelated to this update processor + continue; + } + actualKnownErrsCount++; + assertEquals("only expected type of error is ADD: " + err, + CmdType.ADD, err.getType()); + assertTrue("failed id didn't match 'unknown': " + err, + err.getId().contains("unknown")); + } + assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(), + 11, actualKnownErrsCount); + } + assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish + assertQueryDocIds(client, true + , S_ONE_PRE + "z", S_ONE_PRE + "y", S_TWO_PRE + "z", S_TWO_PRE + "y" // first + // // we can't assert for sure these docs were skipped or added + // // depending on shard we hit, they may have been added async before errors were exceeded + // , S_ONE_PRE + "x", S_TWO_PRE + "x" // skipped + ); + + // clean slate + assertEquals(0, client.deleteByQuery("*:*").getStatus()); + + // many docs from diff shards, more then 10 from a single shard (two) should fail but + // request should still succeed because of maxErrors=-1 param + + ArrayList docs = new ArrayList(30); + ArrayList expectedErrs = new ArrayList(30); + docs.add(doc(f("id", S_ONE_PRE + "z"))); + docs.add(doc(f("id", S_TWO_PRE + "z"))); + docs.add(doc(f("id", S_ONE_PRE + "y"))); + docs.add(doc(f("id", S_TWO_PRE + "y"))); + for (int i = 0; i < 11; i++) { + docs.add(doc(f("id", S_ONE_PRE + i))); + docs.add(doc(f("id", S_TWO_PRE + i), f("foo_i", "bogus_val"))); + expectedErrs.add(addErr(S_TWO_PRE + i)); + } + docs.add(doc(f("id", S_ONE_PRE + "x"))); + docs.add(doc(f("id", S_TWO_PRE + "x"))); + + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "maxErrors", "-1", + "commit", "true"), + docs.toArray(new SolrInputDocument[docs.size()])).process(client); + assertUpdateTolerantErrors("many docs from shard2 fail, but req should succeed", rsp, + expectedErrs.toArray(new ExpectedErr[expectedErrs.size()])); + assertQueryDocIds(client, true + , S_ONE_PRE + "z", S_ONE_PRE + "y", S_TWO_PRE + "z", S_TWO_PRE + "y" // first + , S_ONE_PRE + "x", S_TWO_PRE + "x" // later + ); + + } + + // + public void testAddsMixedWithDeletesViaCloudClient() throws Exception { + testAddsMixedWithDeletes(CLOUD_CLIENT); + } + public void testAddsMixedWithDeletesViaShard1LeaderClient() throws Exception { + testAddsMixedWithDeletes(S_ONE_LEADER_CLIENT); + } + public void testAddsMixedWithDeletesViaShard2LeaderClient() throws Exception { + testAddsMixedWithDeletes(S_TWO_LEADER_CLIENT); + } + public void testAddsMixedWithDeletesViaShard1NonLeaderClient() throws Exception { + testAddsMixedWithDeletes(S_ONE_NON_LEADER_CLIENT); + } + public void testAddsMixedWithDeletesViaShard2NonLeaderClient() throws Exception { + testAddsMixedWithDeletes(S_TWO_NON_LEADER_CLIENT); + } + public void testAddsMixedWithDeletesViaNoCollectionClient() throws Exception { + testAddsMixedWithDeletes(NO_COLLECTION_CLIENT); + } + + protected static void testAddsMixedWithDeletes(SolrClient client) throws Exception { + assertNotNull("client not initialized", client); + + // 3 doc ids, exactly one on shard1 + final String docId1 = S_ONE_PRE + "42"; + final String docId21 = S_TWO_PRE + "42"; + final String docId22 = S_TWO_PRE + "666"; + + UpdateResponse rsp = null; + + // add 2 docs, one to each shard + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", docId1), f("foo_i", "2001")), + doc(f("id", docId21), f("foo_i", "1976"))).process(client); + assertEquals(0, rsp.getStatus()); + + // add failure on shard2, delete failure on shard1 + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", docId22), f("foo_i", "not_a_num"))) + .deleteById(docId1, -1L) + .process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("shard2 add fail, shard1 delI fail", rsp, + delIErr(docId1, "version conflict"), + addErr(docId22,"not_a_num")); + + // attempt a request containing 4 errors of various types (add, delI, delQ) + for (String maxErrors : new String[] {"4", "-1", "100"}) { + // for all of these maxErrors values, the overall request should still succeed + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "maxErrors", maxErrors, + "commit", "true"), + doc(f("id", docId22), f("foo_i", "bogus_val"))) + .deleteById(docId1, -1L) + .deleteByQuery("malformed:[") + .deleteById(docId21, -1L) + .process(client); + + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("failed variety of updates", rsp, + delIErr(docId1, "version conflict"), + delQErr("malformed:[", "SyntaxError"), + delIErr(docId21,"version conflict"), + addErr(docId22,"bogus_val")); + } + + // attempt a request containing 4 errors of various types (add, delI, delQ) .. 1 too many + try { + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "maxErrors", "3", + "commit", "true"), + doc(f("id", docId22), f("foo_i", "bogus_val"))) + .deleteById(docId1, -1L) + .deleteByQuery("malformed:[") + .deleteById(docId21, -1L) + .process(client); + fail("did not get a top level exception when more then 4 updates failed: " + rsp.toString()); + } catch (SolrException e) { + // we can't make any reliable assertions about the error message, because + // it varies based on how the request was routed -- see SOLR-8830 + + // likewise, we can't make a firm(er) assertion about the response code... + assertTrue("not the type of error we were expecting ("+e.code()+"): " + e.toString(), + // should be one these 2 depending on order that the async errors were hit... + // on a single node setup -- a 5xx type error isn't something we should have triggered + 400 == e.code() || 409 == e.code()); + + // verify that the Exceptions metadata can tell us what failed. + NamedList remoteErrMetadata = e.getMetadata(); + assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata); + Set actualKnownErrs + = new LinkedHashSet(remoteErrMetadata.size()); + int actualKnownErrsCount = 0; + for (int i = 0; i < remoteErrMetadata.size(); i++) { + ToleratedUpdateError err = + ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i), + remoteErrMetadata.getVal(i)); + if (null == err) { + // some metadata unrelated to this update processor + continue; + } + actualKnownErrsCount++; + actualKnownErrs.add(err); + } + assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(), + 4, actualKnownErrsCount); + assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(), + actualKnownErrsCount, actualKnownErrs.size()); + } + + // sanity check our 2 existing docs are still here + assertQueryDocIds(client, true, docId1, docId21); + assertQueryDocIds(client, false, docId22); + + // tolerate some failures along with a DELQ that should succeed + rsp = update(params("update.chain", "tolerant-chain-max-errors-10", + "commit", "true"), + doc(f("id", docId22), f("foo_i", "not_a_num"))) + .deleteById(docId1, -1L) + .deleteByQuery("zot_i:[42 to gibberish...") + .deleteByQuery("foo_i:[50 TO 2000}") + .process(client); + assertEquals(0, rsp.getStatus()); + assertUpdateTolerantErrors("mix fails with one valid DELQ", rsp, + delIErr(docId1, "version conflict"), + delQErr("zot_i:[42 to gibberish..."), + addErr(docId22,"not_a_num")); + // one of our previous docs should have been deleted now + assertQueryDocIds(client, true, docId1); + assertQueryDocIds(client, false, docId21, docId22); + + } + + /** + * HACK: Loops over every Jetty instance in the specified MiniSolrCloudCluster to see if they are running, + * and sleeps small increments until they all report that they are, or a max num iters is reached + * + * (work around for SOLR-8862. Maybe something like this should be promoted into MiniSolrCloudCluster's + * start() method? or SolrCloudTestCase's configureCluster?) + */ + public static void assertSpinLoopAllJettyAreRunning(MiniSolrCloudCluster cluster) throws InterruptedException { + // NOTE: idealy we could use an ExecutorService that tried to open Sockets (with a long timeout) + // to each of the jetty instances in parallel w/o any sleeping -- but since they pick their ports + // dynamically and don't report them until/unless the server is up, that won't neccessarily do us + // any good. + final int numServers = cluster.getJettySolrRunners().size(); + int numRunning = 0; + for (int i = 5; 0 <= i; i--) { + numRunning = 0; + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + if (jetty.isRunning()) { + numRunning++; + } + } + if (numServers == numRunning) { + return; + } else if (0 == i) { + // give up + break; + } + // the more nodes we're waiting on, the longer we should try to sleep (within reason) + Thread.sleep(Math.min((numServers - numRunning) * 100, 1000)); + } + assertEquals("giving up waiting for all jetty instances to be running", + numServers, numRunning); + } + + /** Asserts that the UpdateResponse contains the specified expectedErrs and no others */ + public static void assertUpdateTolerantErrors(String assertionMsgPrefix, + UpdateResponse response, + ExpectedErr... expectedErrs) { + @SuppressWarnings("unchecked") + List> errors = (List>) + response.getResponseHeader().get("errors"); + + assertNotNull(assertionMsgPrefix + ": Null errors: " + response.toString(), errors); + assertEquals(assertionMsgPrefix + ": Num error ids: " + errors.toString(), + expectedErrs.length, errors.size()); + + for (SimpleOrderedMap err : errors) { + String assertErrPre = assertionMsgPrefix + ": " + err.toString(); + + String id = err.get("id"); + assertNotNull(assertErrPre + " ... null id", id); + String type = err.get("type"); + assertNotNull(assertErrPre + " ... null type", type); + String message = err.get("message"); + assertNotNull(assertErrPre + " ... null message", message); + + // inefficient scan, but good nough for the size of sets we're dealing with + boolean found = false; + for (ExpectedErr expected : expectedErrs) { + if (expected.type.equals(type) && expected.id.equals(id) + && (null == expected.msgSubStr || message.contains(expected.msgSubStr))) { + found = true; + break; + } + } + assertTrue(assertErrPre + " ... unexpected err in: " + response.toString(), found); + + } + } + + /** convinience method when the only type of errors you expect are 'add' errors */ + public static void assertUpdateTolerantAddErrors(String assertionMsgPrefix, + UpdateResponse response, + String... errorIdsExpected) { + ExpectedErr[] expected = new ExpectedErr[errorIdsExpected.length]; + for (int i = 0; i < expected.length; i++) { + expected[i] = addErr(errorIdsExpected[i]); + } + assertUpdateTolerantErrors(assertionMsgPrefix, response, expected); + } + + /** + * Asserts that the specified document ids do/do-not exist in the index, using both the specified client, + * and the CLOUD_CLIENT + */ + public static void assertQueryDocIds(SolrClient client, boolean shouldExist, String... ids) throws Exception { + for (String id : ids) { + assertEquals(client.toString() + " should " + (shouldExist ? "" : "not ") + "find id: " + id, + (shouldExist ? 1 : 0), + CLOUD_CLIENT.query(params("q", "{!term f=id}" + id)).getResults().getNumFound()); + } + if (! CLOUD_CLIENT.equals(client) ) { + assertQueryDocIds(CLOUD_CLIENT, shouldExist, ids); + } + } + + public static UpdateRequest update(SolrParams params, SolrInputDocument... docs) { + UpdateRequest r = new UpdateRequest(); + r.setParams(new ModifiableSolrParams(params)); + r.add(Arrays.asList(docs)); + return r; + } + + public static SolrInputDocument doc(SolrInputField... fields) { + SolrInputDocument doc = new SolrInputDocument(); + for (SolrInputField f : fields) { + doc.put(f.getName(), f); + } + return doc; + } + + public static SolrInputField f(String fieldName, Object... values) { + SolrInputField f = new SolrInputField(fieldName); + f.setValue(values, 1.0F); + return f; + } + + /** simple helper struct */ + public static final class ExpectedErr { + final String type; + final String id; + final String msgSubStr; // ignored if null + + public ExpectedErr(String type, String id, String msgSubStr) { + this.type = type; + this.id = id; + this.msgSubStr = msgSubStr; + } + public String toString() { + return "type=<"+type+">,id=<"+id+">,msgSubStr=<"+msgSubStr+">"; + } + } + public static ExpectedErr addErr(String id, String msgSubStr) { + return new ExpectedErr("ADD", id, msgSubStr); + } + public static ExpectedErr delIErr(String id, String msgSubStr) { + return new ExpectedErr("DELID", id, msgSubStr); + } + public static ExpectedErr delQErr(String id, String msgSubStr) { + return new ExpectedErr("DELQ", id, msgSubStr); + } + public static ExpectedErr addErr(String id) { + return addErr(id, null); + } + public static ExpectedErr delIErr(String id) { + return delIErr(id, null); + } + public static ExpectedErr delQErr(String id) { + return delQErr(id, null); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java new file mode 100644 index 000000000000..b3f0423c9999 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.io.File; +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.util.TestUtil; +import org.apache.solr.cloud.SolrCloudTestCase; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.assertUpdateTolerantErrors; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.addErr; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delIErr; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.delQErr; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.f; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.update; +import static org.apache.solr.cloud.TestTolerantUpdateProcessorCloud.ExpectedErr; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_PARAM; +import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_NEXT; +import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_START; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.ToleratedUpdateError; +import org.apache.solr.common.ToleratedUpdateError.CmdType; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.util.RevertDefaultThreadHandlerRule; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test of TolerantUpdateProcessor using a randomized MiniSolrCloud. + * Reuses some utility methods in {@link TestTolerantUpdateProcessorCloud} + * + *

    + * NOTE: This test sets up a static instance of MiniSolrCloud with a single collection + * and several clients pointed at specific nodes. These are all re-used across multiple test methods, + * and assumes that the state of the cluster is healthy between tests. + *

    + * + */ +public class TestTolerantUpdateProcessorRandomCloud extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String COLLECTION_NAME = "test_col"; + + /** A basic client for operations at the cloud level, default collection will be set */ + private static CloudSolrClient CLOUD_CLIENT; + /** one HttpSolrClient for each server */ + private static List NODE_CLIENTS; + + @BeforeClass + private static void createMiniSolrCloudCluster() throws Exception { + + final String configName = "solrCloudCollectionConfig"; + final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf"); + + final int numShards = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3); + final int repFactor = TestUtil.nextInt(random(), 2, TEST_NIGHTLY ? 5 : 3); + // at least one server won't have any replicas + final int numServers = 1 + (numShards * repFactor); + + log.info("Configuring cluster: servers={}, shards={}, repfactor={}", numServers, numShards, repFactor); + configureCluster(numServers) + .addConfig(configName, configDir.toPath()) + .configure(); + + TestTolerantUpdateProcessorCloud.assertSpinLoopAllJettyAreRunning(cluster); + + Map collectionProperties = new HashMap<>(); + collectionProperties.put("config", "solrconfig-distrib-update-processor-chains.xml"); + collectionProperties.put("schema", "schema15.xml"); // string id + + + assertNotNull(cluster.createCollection(COLLECTION_NAME, numShards, repFactor, + configName, null, null, collectionProperties)); + + CLOUD_CLIENT = cluster.getSolrClient(); + CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME); + + NODE_CLIENTS = new ArrayList(numServers); + + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + URL jettyURL = jetty.getBaseUrl(); + NODE_CLIENTS.add(new HttpSolrClient(jettyURL.toString() + "/" + COLLECTION_NAME + "/")); + } + assertEquals(numServers, NODE_CLIENTS.size()); + + ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader(); + AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330); + + } + + @Before + private void deleteAllDocs() throws Exception { + assertEquals(0, update(params("commit","true")).deleteByQuery("*:*").process(CLOUD_CLIENT).getStatus()); + assertEquals("index should be empty", 0L, countDocs(CLOUD_CLIENT)); + } + + public void testRandomUpdates() throws Exception { + final int maxDocId = atLeast(10000); + final BitSet expectedDocIds = new BitSet(maxDocId+1); + + final int numIters = atLeast(50); + for (int i = 0; i < numIters; i++) { + + log.info("BEGIN ITER #{}", i); + + final UpdateRequest req = update(params("maxErrors","-1", + "update.chain", "tolerant-chain-max-errors-10")); + final int numCmds = TestUtil.nextInt(random(), 1, 20); + final List expectedErrors = new ArrayList(numCmds); + int expectedErrorsCount = 0; + // it's ambigious/confusing which order mixed DELQ + ADD (or ADD and DELI for the same ID) + // in the same request wll be processed by various clients, so we keep things simple + // and ensure that no single doc Id is affected by more then one command in the same request + final BitSet docsAffectedThisRequest = new BitSet(maxDocId+1); + for (int cmdIter = 0; cmdIter < numCmds; cmdIter++) { + if ((maxDocId / 2) < docsAffectedThisRequest.cardinality()) { + // we're already mucking with more then half the docs in the index + break; + } + + final boolean causeError = random().nextBoolean(); + if (causeError) { + expectedErrorsCount++; + } + + if (random().nextBoolean()) { + // add a doc + String id = null; + SolrInputDocument doc = null; + if (causeError && (0 == TestUtil.nextInt(random(), 0, 21))) { + doc = doc(f("foo_s","no unique key")); + expectedErrors.add(addErr("(unknown)")); + } else { + final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId); + docsAffectedThisRequest.set(id_i); + id = "id_"+id_i; + if (causeError) { + expectedErrors.add(addErr(id)); + } else { + expectedDocIds.set(id_i); + } + final String val = causeError ? "bogus_val" : (""+TestUtil.nextInt(random(), 42, 666)); + doc = doc(f("id",id), + f("id_i", id_i), + f("foo_i", val)); + } + req.add(doc); + log.info("ADD: {} = {}", id, doc); + } else { + // delete something + if (random().nextBoolean()) { + // delete by id + final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId); + final String id = "id_"+id_i; + final boolean docExists = expectedDocIds.get(id_i); + docsAffectedThisRequest.set(id_i); + long versionConstraint = docExists ? 1 : -1; + if (causeError) { + versionConstraint = -1 * versionConstraint; + expectedErrors.add(delIErr(id)); + } else { + // if doc exists it will legitimately be deleted + expectedDocIds.clear(id_i); + } + req.deleteById(id, versionConstraint); + log.info("DEL: {} = {}", id, causeError ? "ERR" : "OK" ); + } else { + // delete by query + final String q; + if (causeError) { + // even though our DBQ is gibberish that's going to fail, record a docId as affected + // so that we don't generate the same random DBQ and get redundent errors + // (problematic because of how DUP forwarded DBQs have to have their errors deduped by TUP) + final int id_i = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId); + docsAffectedThisRequest.set(id_i); + q = "foo_i:["+id_i+" TO ....giberish"; + expectedErrors.add(delQErr(q)); + } else { + // ensure our DBQ is only over a range of docs not already affected + // by any other cmds in this request + final int rangeAxis = randomUnsetBit(random(), docsAffectedThisRequest, maxDocId); + final int loBound = docsAffectedThisRequest.previousSetBit(rangeAxis); + final int hiBound = docsAffectedThisRequest.nextSetBit(rangeAxis); + final int lo = TestUtil.nextInt(random(), loBound+1, rangeAxis); + final int hi = TestUtil.nextInt(random(), rangeAxis, + // bound might be negative if no set bits above axis + (hiBound < 0) ? maxDocId : hiBound-1); + + if (lo != hi) { + assert lo < hi : "lo="+lo+" hi="+hi; + // NOTE: clear & set are exclusive of hi, so we use "}" in range query accordingly + q = "id_i:[" + lo + " TO " + hi + "}"; + expectedDocIds.clear(lo, hi); + docsAffectedThisRequest.set(lo, hi); + } else { + // edge case: special case DBQ of one doc + assert (lo == rangeAxis && hi == rangeAxis) : "lo="+lo+" axis="+rangeAxis+" hi="+hi; + q = "id_i:[" + lo + " TO " + lo + "]"; // have to be inclusive of both ends + expectedDocIds.clear(lo); + docsAffectedThisRequest.set(lo); + } + } + req.deleteByQuery(q); + log.info("DEL: {}", q); + } + } + } + assertEquals("expected error count sanity check: " + req.toString(), + expectedErrorsCount, expectedErrors.size()); + + final SolrClient client = random().nextBoolean() ? CLOUD_CLIENT + : NODE_CLIENTS.get(TestUtil.nextInt(random(), 0, NODE_CLIENTS.size()-1)); + + final UpdateResponse rsp = req.process(client); + assertUpdateTolerantErrors(client.toString() + " => " + expectedErrors.toString(), rsp, + expectedErrors.toArray(new ExpectedErr[expectedErrors.size()])); + + log.info("END ITER #{}, expecting #docs: {}", i, expectedDocIds.cardinality()); + + assertEquals("post update commit failed?", 0, CLOUD_CLIENT.commit().getStatus()); + + for (int j = 0; j < 5; j++) { + if (expectedDocIds.cardinality() == countDocs(CLOUD_CLIENT)) { + break; + } + log.info("sleeping to give searchers a chance to re-open #" + j); + Thread.sleep(200); + } + + // check the index contents against our expecationts + final BitSet actualDocIds = allDocs(CLOUD_CLIENT, maxDocId); + if ( expectedDocIds.cardinality() != actualDocIds.cardinality() ) { + log.error("cardinality missmatch: expected {} BUT actual {}", + expectedDocIds.cardinality(), + actualDocIds.cardinality()); + } + final BitSet x = (BitSet) actualDocIds.clone(); + x.xor(expectedDocIds); + for (int b = x.nextSetBit(0); 0 <= b; b = x.nextSetBit(b+1)) { + final boolean expectedBit = expectedDocIds.get(b); + final boolean actualBit = actualDocIds.get(b); + log.error("bit #"+b+" mismatch: expected {} BUT actual {}", expectedBit, actualBit); + } + assertEquals(x.cardinality() + " mismatched bits", + expectedDocIds.cardinality(), actualDocIds.cardinality()); + } + } + + /** sanity check that randomUnsetBit works as expected + * @see #randomUnsetBit + */ + public void testSanityRandomUnsetBit() { + final int max = atLeast(100); + BitSet bits = new BitSet(max+1); + for (int i = 0; i <= max; i++) { + assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/max="+max, + bits.cardinality() == max+1); + final int nextBit = randomUnsetBit(random(), bits, max); + assertTrue("nextBit shouldn't be negative yet: " + nextBit, + 0 <= nextBit); + assertTrue("nextBit can't exceed max: " + nextBit, + nextBit <= max); + assertFalse("expect unset: " + nextBit, bits.get(nextBit)); + bits.set(nextBit); + } + + assertEquals("why isn't bitset full?", max+1, bits.cardinality()); + + final int firstClearBit = bits.nextClearBit(0); + assertTrue("why is there a clear bit? = " + firstClearBit, + max < firstClearBit); + assertEquals("why is a bit set above max?", + -1, bits.nextSetBit(max+1)); + + assertEquals("wrong nextBit at end of all iters", -1, + randomUnsetBit(random(), bits, max)); + assertEquals("wrong nextBit at redundent end of all iters", -1, + randomUnsetBit(random(), bits, max)); + } + + public static SolrInputDocument doc(SolrInputField... fields) { + // SolrTestCaseJ4 has same method name, prevents static import from working + return TestTolerantUpdateProcessorCloud.doc(fields); + } + + /** + * Given a BitSet, returns a random bit that is currently false, or -1 if all bits are true. + * NOTE: this method is not fair. + */ + public static final int randomUnsetBit(Random r, BitSet bits, final int max) { + // NOTE: don't forget, BitSet will grow automatically if not careful + if (bits.cardinality() == max+1) { + return -1; + } + final int candidate = TestUtil.nextInt(r, 0, max); + if (bits.get(candidate)) { + final int lo = bits.previousClearBit(candidate); + final int hi = bits.nextClearBit(candidate); + if (lo < 0 && max < hi) { + fail("how the hell did we not short circut out? card="+bits.cardinality()+"/size="+bits.size()); + } else if (lo < 0) { + return hi; + } else if (max < hi) { + return lo; + } // else... + return ((candidate - lo) < (hi - candidate)) ? lo : hi; + } + return candidate; + } + + /** returns the numFound from a *:* query */ + public static final long countDocs(SolrClient c) throws Exception { + return c.query(params("q","*:*","rows","0")).getResults().getNumFound(); + } + + /** uses a Cursor to iterate over every doc in the index, recording the 'id_i' value in a BitSet */ + private static final BitSet allDocs(final SolrClient c, final int maxDocIdExpected) throws Exception { + BitSet docs = new BitSet(maxDocIdExpected+1); + String cursorMark = CURSOR_MARK_START; + int docsOnThisPage = Integer.MAX_VALUE; + while (0 < docsOnThisPage) { + final SolrParams p = params("q","*:*", + "rows","100", + // note: not numeric, but we don't actual care about the order + "sort", "id asc", + CURSOR_MARK_PARAM, cursorMark); + QueryResponse rsp = c.query(p); + cursorMark = rsp.getNextCursorMark(); + docsOnThisPage = 0; + for (SolrDocument doc : rsp.getResults()) { + docsOnThisPage++; + int id_i = ((Integer)doc.get("id_i")).intValue(); + assertTrue("found id_i bigger then expected "+maxDocIdExpected+": " + id_i, + id_i <= maxDocIdExpected); + docs.set(id_i); + } + cursorMark = rsp.getNextCursorMark(); + } + return docs; + } +} diff --git a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java index 637c9751f7fd..31361befd5e8 100644 --- a/solr/core/src/test/org/apache/solr/core/TestBadConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestBadConfig.java @@ -96,4 +96,9 @@ public void testUnknownSchemaAttribute() throws Exception { assertConfigs("bad-solrconfig-unexpected-schema-attribute.xml", "schema-minimal.xml", "Unexpected arg(s): {bogusParam=bogusValue}"); } + + public void testTolerantUpdateProcessorNoUniqueKey() throws Exception { + assertConfigs("solrconfig-tolerant-update-minimal.xml", "schema-minimal.xml", + "requires a schema that includes a uniqueKey field"); + } } diff --git a/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java new file mode 100644 index 000000000000..9bbead8eb1e5 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/TolerantUpdateProcessorTest.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.xml.xpath.XPathExpressionException; + +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.ToleratedUpdateError; +import org.apache.solr.common.ToleratedUpdateError.CmdType; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.servlet.DirectSolrConnection; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.util.BaseTestHarness; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.xml.sax.SAXException; + +public class TolerantUpdateProcessorTest extends UpdateProcessorTestBase { + + /** + * List of valid + invalid documents + */ + private static List docs = null; + /** + * IDs of the invalid documents in docs + */ + private static String[] badIds = null; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-update-processor-chains.xml", "schema12.xml"); + } + + @AfterClass + public static void tearDownClass() { + docs = null; + badIds = null; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + //expected exception messages + ignoreException("Error adding field"); + ignoreException("Document is missing mandatory uniqueKey field"); + if (docs == null) { + docs = new ArrayList<>(20); + badIds = new String[10]; + for(int i = 0; i < 10;i++) { + // a valid document + docs.add(doc(field("id", 1f, String.valueOf(2*i)), field("weight", 1f, i))); + // ... and an invalid one + docs.add(doc(field("id", 1f, String.valueOf(2*i+1)), field("weight", 1f, "b"))); + badIds[i] = String.valueOf(2*i+1); + } + } + + } + + @Override + public void tearDown() throws Exception { + resetExceptionIgnores(); + assertU(delQ("*:*")); + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='0']"); + super.tearDown(); + } + + /** + * future proof TolerantUpdateProcessor against new default method impls being added to UpdateProcessor + * to ensure that every method involved in a processor chain life cycle is overridden with + * exception catching/tracking. + */ + public void testReflection() { + for (Method method : TolerantUpdateProcessor.class.getMethods()) { + if (method.getDeclaringClass().equals(Object.class)) { + continue; + } + assertEquals("base class(es) has changed, TolerantUpdateProcessor needs updated to ensure it " + + "overrides all solr update lifcycle methods with exception tracking: " + method.toString(), + TolerantUpdateProcessor.class, method.getDeclaringClass()); + } + } + + + @Test + public void testValidAdds() throws IOException { + SolrInputDocument validDoc = doc(field("id", 1f, "1"), field("text", 1f, "the quick brown fox")); + add("tolerant-chain-max-errors-10", null, validDoc); + + validDoc = doc(field("id", 1f, "2"), field("text", 1f, "the quick brown fox")); + add("tolerant-chain-max-errors-not-set", null, validDoc); + + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='2']"); + assertQ(req("q","id:1") + ,"//result[@numFound='1']"); + assertQ(req("q","id:2") + ,"//result[@numFound='1']"); + } + + @Test + public void testInvalidAdds() throws IOException { + SolrInputDocument invalidDoc = doc(field("text", 1f, "the quick brown fox")); //no id + try { + // This doc should fail without being tolerant + add("not-tolerant", null, invalidDoc); + fail("Expecting exception"); + } catch (Exception e) { + //expected + assertTrue(e.getMessage().contains("Document is missing mandatory uniqueKey field")); + } + assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc}), null, "(unknown)"); + + //a valid doc + SolrInputDocument validDoc = doc(field("id", 1f, "1"), field("text", 1f, "the quick brown fox")); + + try { + // This batch should fail without being tolerant + add("not-tolerant", null, Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc})); + fail("Expecting exception"); + } catch (Exception e) { + //expected + assertTrue(e.getMessage().contains("Document is missing mandatory uniqueKey field")); + } + + assertU(commit()); + assertQ(req("q","id:1") + ,"//result[@numFound='0']"); + + + assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc}), null, "(unknown)"); + assertU(commit()); + + // verify that the good document made it in. + assertQ(req("q","id:1") + ,"//result[@numFound='1']"); + + invalidDoc = doc(field("id", 1f, "2"), field("weight", 1f, "aaa")); + validDoc = doc(field("id", 1f, "3"), field("weight", 1f, "3")); + + try { + // This batch should fail without being tolerant + add("not-tolerant", null, Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc})); //no id + fail("Expecting exception"); + } catch (Exception e) { + //expected + assertTrue(e.getMessage().contains("Error adding field")); + } + + assertU(commit()); + assertQ(req("q","id:3") + ,"//result[@numFound='0']"); + + assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc}), null, "2"); + assertU(commit()); + + // The valid document was indexed + assertQ(req("q","id:3") + ,"//result[@numFound='1']"); + + // The invalid document was NOT indexed + assertQ(req("q","id:2") + ,"//result[@numFound='0']"); + + } + + @Test + public void testMaxErrorsDefault() throws IOException { + try { + // by default the TolerantUpdateProcessor accepts all errors, so this batch should succeed with 10 errors. + assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, null, badIds); + } catch(Exception e) { + fail("Shouldn't get an exception for this batch: " + e.getMessage()); + } + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='10']"); + } + + public void testMaxErrorsSucceed() throws IOException { + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.add("maxErrors", "10"); + // still OK + assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, requestParams, badIds); + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='10']"); + } + + @Test + public void testMaxErrorsThrowsException() throws IOException { + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.add("maxErrors", "5"); + try { + // should fail + assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, requestParams, badIds); + fail("Expecting exception"); + } catch (SolrException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("ERROR: [doc=1] Error adding field 'weight'='b' msg=For input string: \"b\"")); + } + //the first good documents made it to the index + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='6']"); + } + + @Test + public void testMaxErrorsInfinite() throws IOException { + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.add("maxErrors", "-1"); + try { + assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, null, badIds); + } catch(Exception e) { + fail("Shouldn't get an exception for this batch: " + e.getMessage()); + } + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='10']"); + } + + @Test + public void testMaxErrors0() throws IOException { + //make the TolerantUpdateProcessor intolerant + List smallBatch = docs.subList(0, 2); + ModifiableSolrParams requestParams = new ModifiableSolrParams(); + requestParams.add("maxErrors", "0"); + try { + // should fail + assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", smallBatch, requestParams, "1"); + fail("Expecting exception"); + } catch (SolrException e) { + assertTrue(e.getMessage().contains("ERROR: [doc=1] Error adding field 'weight'='b' msg=For input string: \"b\"")); + } + //the first good documents made it to the index + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='1']"); + } + + @Test + public void testInvalidDelete() throws XPathExpressionException, SAXException { + ignoreException("undefined field invalidfield"); + String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox")); + assertNull(BaseTestHarness.validateXPath(response, + "//int[@name='status']=0", + "//arr[@name='errors']", + "count(//arr[@name='errors']/lst)=0")); + + response = update("tolerant-chain-max-errors-10", delQ("invalidfield:1")); + assertNull(BaseTestHarness.validateXPath + (response, + "//int[@name='status']=0", + "count(//arr[@name='errors']/lst)=1", + "//arr[@name='errors']/lst/str[@name='type']/text()='DELQ'", + "//arr[@name='errors']/lst/str[@name='id']/text()='invalidfield:1'", + "//arr[@name='errors']/lst/str[@name='message']/text()='undefined field invalidfield'")); + } + + @Test + public void testValidDelete() throws XPathExpressionException, SAXException { + ignoreException("undefined field invalidfield"); + String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox")); + assertNull(BaseTestHarness.validateXPath(response, + "//int[@name='status']=0", + "//arr[@name='errors']", + "count(//arr[@name='errors']/lst)=0")); + + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='1']"); + + response = update("tolerant-chain-max-errors-10", delQ("id:1")); + assertNull(BaseTestHarness.validateXPath(response, + "//int[@name='status']=0", + "//arr[@name='errors']", + "count(//arr[@name='errors']/lst)=0")); + assertU(commit()); + assertQ(req("q","*:*") + ,"//result[@numFound='0']"); + } + + @Test + public void testResponse() throws SAXException, XPathExpressionException, IOException { + String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox")); + assertNull(BaseTestHarness.validateXPath(response, + "//int[@name='status']=0", + "//arr[@name='errors']", + "count(//arr[@name='errors']/lst)=0")); + response = update("tolerant-chain-max-errors-10", adoc("text", "the quick brown fox")); + assertNull(BaseTestHarness.validateXPath(response, "//int[@name='status']=0", + "//int[@name='maxErrors']/text()='10'", + "count(//arr[@name='errors']/lst)=1", + "//arr[@name='errors']/lst/str[@name='id']/text()='(unknown)'", + "//arr[@name='errors']/lst/str[@name='message']/text()='Document is missing mandatory uniqueKey field: id'")); + + response = update("tolerant-chain-max-errors-10", adoc("text", "the quick brown fox")); + StringWriter builder = new StringWriter(); + builder.append(""); + for (SolrInputDocument doc:docs) { + ClientUtils.writeXML(doc, builder); + } + builder.append(""); + response = update("tolerant-chain-max-errors-10", builder.toString()); + assertNull(BaseTestHarness.validateXPath(response, "//int[@name='status']=0", + "//int[@name='maxErrors']/text()='10'", + "count(//arr[@name='errors']/lst)=10", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='0')", + "//arr[@name='errors']/lst/str[@name='id']/text()='1'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='2')", + "//arr[@name='errors']/lst/str[@name='id']/text()='3'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='4')", + "//arr[@name='errors']/lst/str[@name='id']/text()='5'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='6')", + "//arr[@name='errors']/lst/str[@name='id']/text()='7'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='8')", + "//arr[@name='errors']/lst/str[@name='id']/text()='9'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='10')", + "//arr[@name='errors']/lst/str[@name='id']/text()='11'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='12')", + "//arr[@name='errors']/lst/str[@name='id']/text()='13'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='14')", + "//arr[@name='errors']/lst/str[@name='id']/text()='15'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='16')", + "//arr[@name='errors']/lst/str[@name='id']/text()='17'", + "not(//arr[@name='errors']/lst/str[@name='id']/text()='18')", + "//arr[@name='errors']/lst/str[@name='id']/text()='19'")); + + // spot check response when effective maxErrors is unlimited + response = update("tolerant-chain-max-errors-not-set", builder.toString()); + assertNull(BaseTestHarness.validateXPath(response, "//int[@name='maxErrors']/text()='-1'")); + + } + + + + public String update(String chain, String xml) { + DirectSolrConnection connection = new DirectSolrConnection(h.getCore()); + SolrRequestHandler handler = h.getCore().getRequestHandler("/update"); + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("update.chain", chain); + try { + return connection.request(handler, params, xml); + } catch (SolrException e) { + throw (SolrException)e; + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + private void assertAddsSucceedWithErrors(String chain, + final Collection docs, + SolrParams requestParams, + String... idsShouldFail) throws IOException { + + SolrQueryResponse response = add(chain, requestParams, docs); + + @SuppressWarnings("unchecked") + List> errors = (List>) + response.getResponseHeader().get("errors"); + assertNotNull(errors); + + assertEquals("number of errors", idsShouldFail.length, errors.size()); + + Set addErrorIdsExpected = new HashSet(Arrays.asList(idsShouldFail)); + + for (SimpleOrderedMap err : errors) { + assertEquals("this method only expects 'add' errors", "ADD", err.get("type")); + + String id = err.get("id"); + assertNotNull("null err id", id); + assertTrue("unexpected id", addErrorIdsExpected.contains(id)); + + } + } + + protected SolrQueryResponse add(final String chain, SolrParams requestParams, final SolrInputDocument doc) throws IOException { + return add(chain, requestParams, Arrays.asList(new SolrInputDocument[]{doc})); + } + + protected SolrQueryResponse add(final String chain, SolrParams requestParams, final Collection docs) throws IOException { + + SolrCore core = h.getCore(); + UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain); + assertNotNull("No Chain named: " + chain, pc); + + SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.add("responseHeader", new SimpleOrderedMap()); + + if(requestParams == null) { + requestParams = new ModifiableSolrParams(); + } + + SolrQueryRequest req = new LocalSolrQueryRequest(core, requestParams); + try { + UpdateRequestProcessor processor = pc.createProcessor(req, rsp); + for(SolrInputDocument doc:docs) { + AddUpdateCommand cmd = new AddUpdateCommand(req); + cmd.solrDoc = doc; + processor.processAdd(cmd); + } + processor.finish(); + + } finally { + req.close(); + } + return rsp; + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 59b37c5cce92..edfe1c3e1178 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -54,6 +54,7 @@ import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.ToleratedUpdateError; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -72,6 +73,7 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.Hash; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.KeeperException; @@ -726,6 +728,11 @@ public RouteResponse condenseResponse(NamedList response, int timeMillis) { int status = 0; Integer rf = null; Integer minRf = null; + + // TolerantUpdateProcessor + List> toleratedErrors = null; + int maxToleratedErrors = Integer.MAX_VALUE; + for(int i=0; i> shardTolerantErrors = + (List>) header.get("errors"); + if (null != shardTolerantErrors) { + Integer shardMaxToleratedErrors = (Integer) header.get("maxErrors"); + assert null != shardMaxToleratedErrors : "TolerantUpdateProcessor reported errors but not maxErrors"; + // if we get into some weird state where the nodes disagree about the effective maxErrors, + // assume the min value seen to decide if we should fail. + maxToleratedErrors = Math.min(maxToleratedErrors, + ToleratedUpdateError.getEffectiveMaxErrors(shardMaxToleratedErrors.intValue())); + + if (null == toleratedErrors) { + toleratedErrors = new ArrayList>(shardTolerantErrors.size()); + } + for (SimpleOrderedMap err : shardTolerantErrors) { + toleratedErrors.add(err); + } + } } NamedList cheader = new NamedList(); @@ -750,7 +775,31 @@ public RouteResponse condenseResponse(NamedList response, int timeMillis) { cheader.add(UpdateRequest.REPFACT, rf); if (minRf != null) cheader.add(UpdateRequest.MIN_REPFACT, minRf); - + if (null != toleratedErrors) { + cheader.add("maxErrors", ToleratedUpdateError.getUserFriendlyMaxErrors(maxToleratedErrors)); + cheader.add("errors", toleratedErrors); + if (maxToleratedErrors < toleratedErrors.size()) { + // cumulative errors are too high, we need to throw a client exception w/correct metadata + + // NOTE: it shouldn't be possible for 1 == toleratedErrors.size(), because if that were the case + // then at least one shard should have thrown a real error before this, so we don't worry + // about having a more "singular" exception msg for that situation + StringBuilder msgBuf = new StringBuilder() + .append(toleratedErrors.size()).append(" Async failures during distributed update: "); + + NamedList metadata = new NamedList(); + for (SimpleOrderedMap err : toleratedErrors) { + ToleratedUpdateError te = ToleratedUpdateError.parseMap(err); + metadata.add(te.getMetadataKey(), te.getMetadataValue()); + + msgBuf.append("\n").append(te.getMessage()); + } + + SolrException toThrow = new SolrException(ErrorCode.BAD_REQUEST, msgBuf.toString()); + toThrow.setMetadata(metadata); + throw toThrow; + } + } condensed.add("responseHeader", cheader); return condensed; } @@ -786,6 +835,22 @@ public RouteException(ErrorCode errorCode, NamedList throwables, Map< super(errorCode, throwables.getVal(0).getMessage(), throwables.getVal(0)); this.throwables = throwables; this.routes = routes; + + // create a merged copy of the metadata from all wrapped exceptions + NamedList metadata = new NamedList(); + for (int i = 0; i < throwables.size(); i++) { + Throwable t = throwables.getVal(i); + if (t instanceof SolrException) { + SolrException e = (SolrException) t; + NamedList eMeta = e.getMetadata(); + if (null != eMeta) { + metadata.addAll(eMeta); + } + } + } + if (0 < metadata.size()) { + this.setMetadata(metadata); + } } public NamedList getThrowables() { diff --git a/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java new file mode 100644 index 000000000000..fd8b8c79eea1 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/ToleratedUpdateError.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common; + +import java.util.ArrayList; +import java.util.List; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; + +/** + * Models the basic information related to a single "tolerated" error that occured during updates. + * This class is only useful when the ToleranteUpdateProcessorFactory is used in an update + * processor chain + */ +public final class ToleratedUpdateError { + + private final static String META_PRE = ToleratedUpdateError.class.getName() + "--"; + private final static int META_PRE_LEN = META_PRE.length(); + + /** + * Given a 'maxErrors' value such that-1 <= maxErrors <= {@link Integer#MAX_VALUE} + * this method returns the original input unless it is -1 in which case the effective value of + * {@link Integer#MAX_VALUE} is returned. + * Input of maxErrors < -1 will trip an assertion and otherwise have undefined behavior. + * @see #getUserFriendlyMaxErrors + */ + public static int getEffectiveMaxErrors(int maxErrors) { + assert -1 <= maxErrors; + return -1 == maxErrors ? Integer.MAX_VALUE : maxErrors; + } + + /** + * Given a 'maxErrors' value such that-1 <= maxErrors <= {@link Integer#MAX_VALUE} + * this method returns the original input unless it is {@link Integer#MAX_VALUE} in which case + * -1 is returned for user convinience. + * Input of maxErrors < -1 will trip an assertion and otherwise have undefined behavior. + * @see #getEffectiveMaxErrors + */ + public static int getUserFriendlyMaxErrors(int maxErrors) { + assert -1 <= maxErrors; + return Integer.MAX_VALUE == maxErrors ? -1 : maxErrors; + } + + /** + * returns a list of maps of simple objects suitable for putting in a SolrQueryResponse header + * @see #getSimpleMap + * @see #parseMap + */ + public static List> formatForResponseHeader(List errs) { + List> result = new ArrayList<>(errs.size()); + for (ToleratedUpdateError e : errs) { + result.add(e.getSimpleMap()); + } + return result; + } + + /** + * returns a ToleratedUpdateError instance from the data in this Map + * @see #getSimpleMap + */ + public static ToleratedUpdateError parseMap(SimpleOrderedMap data) { + final String id = data.get("id"); + final String message = data.get("message"); + final String t = data.get("type"); + if (null == t || null == id || null == message) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Map does not represent a ToleratedUpdateError, must contain 'type', 'id', and 'message'"); + } + try { + return new ToleratedUpdateError(CmdType.valueOf(t), id, message); + } catch (IllegalArgumentException iae) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid type for ToleratedUpdateError: " + t, iae); + } + } + + /** + * returns a ToleratedUpdateError instance if this metadataKey is one we care about, else null + * @see #getMetadataKey + * @see #getMetadataValue + */ + public static ToleratedUpdateError parseMetadataIfToleratedUpdateError(String metadataKey, + String metadataVal) { + if (! metadataKey.startsWith(META_PRE)) { + return null; // not a key we care about + } + final int typeEnd = metadataKey.indexOf(':', META_PRE_LEN); + if (typeEnd < 0) { + return null; // has our prefix, but not our format -- must not be a key we (actually) care about + } + return new ToleratedUpdateError(CmdType.valueOf(metadataKey.substring(META_PRE_LEN, typeEnd)), + metadataKey.substring(typeEnd+1), metadataVal); + } + + private final CmdType type; + private final String id; + private final String message; + + public ToleratedUpdateError(CmdType type, String id, String message) { + assert null != type; + this.type = type; + + assert null != id; + this.id = id; + + assert null != message; + this.message = message; + } + + public CmdType getType() { + return type; + } + public String getId() { + return id; + } + public String getMessage() { + return message; + } + + /** + * returns a string suitable for use as a key in {@link SolrException#setMetadata} + * + * @see #parseMetadataIfToleratedUpdateError + */ + public String getMetadataKey() { + return META_PRE + type + ":" + id; + } + + /** + * returns a string suitable for use as a value in {@link SolrException#setMetadata} + * + * @see #parseMetadataIfToleratedUpdateError + */ + public String getMetadataValue() { + return message.toString(); + } + + /** + * returns a map of simple objects suitable for putting in a SolrQueryResponse header + * @see #formatForResponseHeader + * @see #parseMap + */ + public SimpleOrderedMap getSimpleMap() { + SimpleOrderedMap entry = new SimpleOrderedMap(); + entry.add("type", type.toString()); + entry.add("id", id); + entry.add("message", message); + return entry; + } + + public String toString() { + return getMetadataKey() + "=>" + getMetadataValue(); + } + + public int hashCode() { + int h = this.getClass().hashCode(); + h = h * 31 + type.hashCode(); + h = h * 31 + id.hashCode(); + h = h * 31 + message.hashCode(); + return h; + } + + public boolean equals(Object o) { + if (o instanceof ToleratedUpdateError) { + ToleratedUpdateError that = (ToleratedUpdateError)o; + return that.type.equals(this.type) + && that.id.equals(this.id) + && that.message.equals(this.message); + } + return false; + } + + /** + * Helper class for dealing with SolrException metadata (String) keys + */ + public static enum CmdType { + ADD, DELID, DELQ; + + // if we add support for things like commit, parsing/toString/hashCode logic + // needs to be smarter to account for 'id' being null ... "usesId" should be a prop of enum instances + } +} + + diff --git a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java new file mode 100644 index 000000000000..aba07ae6cce7 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common; + +import java.util.EnumSet; +import org.apache.solr.common.ToleratedUpdateError; +import org.apache.solr.common.ToleratedUpdateError.CmdType; +import org.apache.solr.common.util.SimpleOrderedMap; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** Basic testing of the serialization/encapsulation code in ToleratedUpdateError */ +public class TestToleratedUpdateError extends LuceneTestCase { + + private final static CmdType[] ALL_TYPES = EnumSet.allOf(CmdType.class).toArray(new CmdType[0]); + + public void testBasics() { + + assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals + (new ToleratedUpdateError(CmdType.ADD, "doc2", "some error"))); + assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals + (new ToleratedUpdateError(CmdType.ADD, "doc1", "some errorxx"))); + assertFalse((new ToleratedUpdateError(CmdType.ADD, "doc1", "some error")).equals + (new ToleratedUpdateError(CmdType.DELID, "doc1", "some error"))); + } + + public void testParseMetadataErrorHandling() { + + assertNull(ToleratedUpdateError.parseMetadataIfToleratedUpdateError("some other key", "some value")); + + // see if someone tries to trick us into having an NPE... + ToleratedUpdateError valid = new ToleratedUpdateError(CmdType.ADD, "doc2", "some error"); + String badKey = valid.getMetadataKey().replace(":", "X"); + assertNull(ToleratedUpdateError.parseMetadataIfToleratedUpdateError(badKey, valid.getMetadataValue())); + } + + public void testParseMapErrorChecking() { + SimpleOrderedMap bogus = new SimpleOrderedMap(); + try { + ToleratedUpdateError.parseMap(bogus); + fail("map should not be parsable"); + } catch (SolrException e) { + assertTrue(e.toString(), e.getMessage().contains("Map does not represent a ToleratedUpdateError") ); + } + + bogus.add("id", "some id"); + bogus.add("message", "some message"); + try { + ToleratedUpdateError.parseMap(bogus); + fail("map should still not be parsable"); + } catch (SolrException e) { + assertTrue(e.toString(), e.getMessage().contains("Map does not represent a ToleratedUpdateError") ); + } + + bogus.add("type", "not a real type"); + try { + ToleratedUpdateError.parseMap(bogus); + fail("invalid type should not be parsable"); + } catch (SolrException e) { + assertTrue(e.toString(), e.getMessage().contains("Invalid type")); + } + } + + public void testParseMap() { + // trivial + SimpleOrderedMap valid = new SimpleOrderedMap(); + valid.add("type", CmdType.ADD.toString()); + valid.add("id", "some id"); + valid.add("message", "some message"); + + ToleratedUpdateError in = ToleratedUpdateError.parseMap(valid); + compare(in, MAP_COPPIER); + compare(in, METADATA_COPPIER); + + // randomized + int numIters = atLeast(5000); + for (int i = 0; i < numIters; i++) { + valid = new SimpleOrderedMap(); + valid.add("type", ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length-1)].toString()); + valid.add("id", TestUtil.randomUnicodeString(random())); + valid.add("message", TestUtil.randomUnicodeString(random())); + + in = ToleratedUpdateError.parseMap(valid); + compare(in, MAP_COPPIER); + compare(in, METADATA_COPPIER); + } + } + + public void checkRoundTripComparisons(Coppier coppier) { + + // some simple basics + for (ToleratedUpdateError in : new ToleratedUpdateError[] { + new ToleratedUpdateError(CmdType.ADD, "doc1", "some error"), + new ToleratedUpdateError(CmdType.DELID, "doc1", "some diff error"), + new ToleratedUpdateError(CmdType.DELQ, "-field:yakko other_field:wakko", "some other error"), + }) { + + compare(in, coppier); + } + + // randomized testing of non trivial keys/values + int numIters = atLeast(5000); + for (int i = 0; i < numIters; i++) { + ToleratedUpdateError in = new ToleratedUpdateError + (ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length-1)], + TestUtil.randomUnicodeString(random()), + TestUtil.randomUnicodeString(random())); + compare(in, coppier); + } + } + + public void testMetadataRoundTripComparisons(Coppier coppier) { + checkRoundTripComparisons(METADATA_COPPIER); + } + + public void testMapRoundTripComparisons() { + checkRoundTripComparisons(MAP_COPPIER); + } + + /** trivial sanity check */ + public void testMaxErrorsValueConversion() { + + assertEquals(-1, ToleratedUpdateError.getUserFriendlyMaxErrors(-1)); + assertEquals(-1, ToleratedUpdateError.getUserFriendlyMaxErrors(Integer.MAX_VALUE)); + + assertEquals(Integer.MAX_VALUE, ToleratedUpdateError.getEffectiveMaxErrors(Integer.MAX_VALUE)); + assertEquals(Integer.MAX_VALUE, ToleratedUpdateError.getEffectiveMaxErrors(-1)); + + for (int val : new int[] {0, 1, 10, 42, 600000 }) { + assertEquals(val, ToleratedUpdateError.getEffectiveMaxErrors(val)); + assertEquals(val, ToleratedUpdateError.getUserFriendlyMaxErrors(val)); + } + + } + + public void compare(ToleratedUpdateError in, Coppier coppier) { + ToleratedUpdateError out = coppier.copy(in); + assertNotNull(out); + compare(in, out); + } + + public void compare(ToleratedUpdateError in, ToleratedUpdateError out) { + assertEquals(out.getType(), in.getType()); + assertEquals(out.getId(), in.getId()); + assertEquals(out.getMessage(), in.getMessage()); + + assertEquals(out.hashCode(), in.hashCode()); + assertEquals(out.toString(), in.toString()); + + assertEquals(in.getMetadataKey(), out.getMetadataKey()); + assertEquals(in.getMetadataValue(), out.getMetadataValue()); + + assertEquals(out, in); + assertEquals(in, out); + } + + private static abstract class Coppier { + public abstract ToleratedUpdateError copy(ToleratedUpdateError in); + } + + private static final Coppier MAP_COPPIER = new Coppier() { + public ToleratedUpdateError copy(ToleratedUpdateError in) { + return ToleratedUpdateError.parseMap(in.getSimpleMap()); + } + }; + + private static final Coppier METADATA_COPPIER = new Coppier() { + public ToleratedUpdateError copy(ToleratedUpdateError in) { + return ToleratedUpdateError.parseMetadataIfToleratedUpdateError + (in.getMetadataKey(), in.getMetadataValue()); + } + }; + +} + + + + From e26c0b7125903e24e5865b825f0f9e993eb10178 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 25 Mar 2016 13:49:25 -0700 Subject: [PATCH 0218/1113] SOLR-8902: fix glob test (put back the fields.clear()) --- .../src/java/org/apache/solr/search/SolrReturnFields.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java index b667f0e2e46e..34ef79e674fb 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java @@ -130,6 +130,13 @@ private void parseFieldList(String[] fl, SolrQueryRequest req) { } augmenters.addTransformer( new RenameFieldTransformer( from, to, copy ) ); } + if( !_wantsAllFields && !globs.isEmpty() ) { + // TODO??? need to fill up the fields with matching field names in the index + // and add them to okFieldNames? + // maybe just get all fields? + // this would disable field selection optimization... i think thatis OK + fields.clear(); // this will get all fields, and use wantsField to limit + } if( augmenters.size() == 1 ) { transformer = augmenters.getTransformer(0); From 585683d6a781f54c6569c88ab1df2ca8d1dbc25a Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 25 Mar 2016 19:47:14 -0400 Subject: [PATCH 0219/1113] add missing javadocs --- .../apache/lucene/codecs/lucene60/Lucene60PointsWriter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java index 9098cfbb01a5..2b1e13dfedf4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java @@ -41,9 +41,13 @@ /** Writes dimensional values */ public class Lucene60PointsWriter extends PointsWriter implements Closeable { - + + /** Output used to write the BKD tree data file */ protected final IndexOutput dataOut; + + /** Maps field name to file pointer in the data file where the BKD index is located. */ protected final Map indexFPs = new HashMap<>(); + final SegmentWriteState writeState; final int maxPointsInLeafNode; final double maxMBSortInHeap; From ec60f5c43cea32cc7531e582e335c9ed826f4b94 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 25 Mar 2016 20:15:07 -0400 Subject: [PATCH 0220/1113] fix bug in exception's message --- .../org/apache/lucene/search/PointRangeQuery.java | 2 +- .../org/apache/lucene/search/TestPointQueries.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java index eb183e258407..4991096e7cf5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java @@ -72,7 +72,7 @@ protected PointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, in throw new IllegalArgumentException("lowerPoint is not a fixed multiple of numDims"); } if (lowerPoint.length != upperPoint.length) { - throw new IllegalArgumentException("lowerPoint has length=" + numDims + " but upperPoint has different length=" + upperPoint.length); + throw new IllegalArgumentException("lowerPoint has length=" + lowerPoint.length + " but upperPoint has different length=" + upperPoint.length); } this.numDims = numDims; this.bytesPerDim = lowerPoint.length / numDims; diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index c2136c4d6630..bc9ef759a189 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -2012,4 +2012,17 @@ public void testPointInSetEquals() { other[2] = (byte) 5; assertFalse(q1.equals(BinaryPoint.newSetQuery("a", new byte[][] {zeros, other}))); } + + public void testInvalidPointLength() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> { + new PointRangeQuery("field", new byte[4], new byte[8], 1) { + @Override + protected String toString(int dimension, byte[] value) { + return "foo"; + } + }; + }); + assertEquals("lowerPoint has length=4 but upperPoint has different length=8", e.getMessage()); + } } From c46d7686643e7503304cb35dfe546bce9c6684e7 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sat, 26 Mar 2016 03:47:06 -0400 Subject: [PATCH 0221/1113] LUCENE-7141: switch OfflineSorter's ByteSequencesReader to BytesRefIterator --- lucene/CHANGES.txt | 5 +++ .../lucene/analysis/hunspell/Dictionary.java | 13 +++--- .../org/apache/lucene/util/OfflineSorter.java | 40 +++++++++---------- .../org/apache/lucene/util/bkd/BKDWriter.java | 13 +++--- .../search/suggest/SortedInputIterator.java | 6 +-- .../suggest/analyzing/AnalyzingSuggester.java | 16 +++++--- .../search/suggest/fst/ExternalRefSorter.java | 18 +++------ .../suggest/fst/FSTCompletionLookup.java | 18 +++++---- .../suggest/fst/BytesRefSortersTest.java | 11 ++--- 9 files changed, 72 insertions(+), 68 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index fa9ab3165dbc..90fb82ff09ab 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -13,6 +13,11 @@ New Features * LUCENE-7140: Add PlanetModel.bisection to spatial3d (Karl Wright via Mike McCandless) +API Changes + +* LUCENE-7141: Switch OfflineSorter's ByteSequencesReader to + BytesRefIterator (Mike McCandless) + Optimizations * LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java index 562e5cba67d3..7885daab8d81 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java @@ -888,17 +888,20 @@ public int compare(BytesRef o1, BytesRef o2) { boolean success2 = false; try (ByteSequencesReader reader = new ByteSequencesReader(tempDir.openChecksumInput(sorted, IOContext.READONCE), sorted)) { - BytesRefBuilder scratchLine = new BytesRefBuilder(); // TODO: the flags themselves can be double-chars (long) or also numeric // either way the trick is to encode them as char... but they must be parsed differently String currentEntry = null; IntsRefBuilder currentOrds = new IntsRefBuilder(); - - String line; - while (reader.read(scratchLine)) { - line = scratchLine.get().utf8ToString(); + + while (true) { + BytesRef scratch = reader.next(); + if (scratch == null) { + break; + } + + String line = scratch.utf8ToString(); String entry; char wordForm[]; int end; diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java index 7549cbccfd5e..386cd0c24933 100644 --- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java @@ -346,7 +346,7 @@ void mergePartitions(Directory trackingDir, List segments) throws IOExce PriorityQueue queue = new PriorityQueue(segmentsToMerge.size()) { @Override protected boolean lessThan(FileAndTop a, FileAndTop b) { - return comparator.compare(a.current.get(), b.current.get()) < 0; + return comparator.compare(a.current, b.current) < 0; } }; @@ -361,15 +361,14 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { // Open streams and read the top for each file for (int i = 0; i < segmentsToMerge.size(); i++) { streams[i] = getReader(dir.openChecksumInput(segmentsToMerge.get(i), IOContext.READONCE), segmentsToMerge.get(i)); - BytesRefBuilder bytes = new BytesRefBuilder(); - boolean result = false; + BytesRef item = null; try { - result = streams[i].read(bytes); + item = streams[i].next(); } catch (Throwable t) { verifyChecksum(t, streams[i]); } - assert result; - queue.insertWithOverflow(new FileAndTop(i, bytes)); + assert item != null; + queue.insertWithOverflow(new FileAndTop(i, item)); } // Unix utility sort() uses ordered array of files to pick the next line from, updating @@ -378,15 +377,14 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { // so it shouldn't make much of a difference (didn't check). FileAndTop top; while ((top = queue.top()) != null) { - writer.write(top.current.bytes(), 0, top.current.length()); - boolean result = false; + writer.write(top.current); try { - result = streams[top.fd].read(top.current); + top.current = streams[top.fd].next(); } catch (Throwable t) { verifyChecksum(t, streams[top.fd]); } - if (result) { + if (top.current != null) { queue.updateTop(); } else { queue.pop(); @@ -416,18 +414,17 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { /** Read in a single partition of data */ int readPartition(ByteSequencesReader reader) throws IOException { long start = System.currentTimeMillis(); - final BytesRefBuilder scratch = new BytesRefBuilder(); while (true) { - boolean result = false; + BytesRef item = null; try { - result = reader.read(scratch); + item = reader.next(); } catch (Throwable t) { verifyChecksum(t, reader); } - if (result == false) { + if (item == null) { break; } - buffer.append(scratch.get()); + buffer.append(item); // Account for the created objects. // (buffer slots do not account to buffer size.) if (bufferBytesUsed.get() > ramBufferSize.bytes) { @@ -440,9 +437,9 @@ int readPartition(ByteSequencesReader reader) throws IOException { static class FileAndTop { final int fd; - final BytesRefBuilder current; + BytesRef current; - FileAndTop(int fd, BytesRefBuilder firstLine) { + FileAndTop(int fd, BytesRef firstLine) { this.fd = fd; this.current = firstLine; } @@ -518,10 +515,11 @@ public void close() throws IOException { * Utility class to read length-prefixed byte[] entries from an input. * Complementary to {@link ByteSequencesWriter}. */ - public static class ByteSequencesReader implements Closeable { + public static class ByteSequencesReader implements BytesRefIterator, Closeable { protected final String name; protected final ChecksumIndexInput in; protected final long end; + private final BytesRefBuilder ref = new BytesRefBuilder(); /** Constructs a ByteSequencesReader from the provided IndexInput */ public ByteSequencesReader(ChecksumIndexInput in, String name) { @@ -538,16 +536,16 @@ public ByteSequencesReader(ChecksumIndexInput in, String name) { * the header of the next sequence. Returns true otherwise. * @throws EOFException if the file ends before the full sequence is read. */ - public boolean read(BytesRefBuilder ref) throws IOException { + public BytesRef next() throws IOException { if (in.getFilePointer() >= end) { - return false; + return null; } short length = in.readShort(); ref.grow(length); ref.setLength(length); in.readBytes(ref.bytes(), 0, length); - return true; + return ref.get(); } /** diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 5002e50445b3..e075cedbfc29 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -33,7 +33,6 @@ import org.apache.lucene.store.TrackingDirectoryWrapper; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntroSorter; @@ -736,6 +735,8 @@ public int compare(BytesRef a, BytesRef b) { // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: IndexOutput[] lastWriter = new IndexOutput[1]; + final BytesRef scratch = new BytesRef(new byte[bytesPerDoc]); + OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @@ -756,14 +757,12 @@ public void write(byte[] bytes, int off, int len) throws IOException { protected ByteSequencesReader getReader(ChecksumIndexInput in, String name) throws IOException { return new ByteSequencesReader(in, name) { @Override - public boolean read(BytesRefBuilder ref) throws IOException { + public BytesRef next() throws IOException { if (in.getFilePointer() >= end) { - return false; + return null; } - ref.grow(bytesPerDoc); - in.readBytes(ref.bytes(), 0, bytesPerDoc); - ref.setLength(bytesPerDoc); - return true; + in.readBytes(scratch.bytes, 0, bytesPerDoc); + return scratch; } }; } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java index 38fbbab665f0..977df3713239 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java @@ -29,7 +29,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.OfflineSorter.ByteSequencesReader; import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter; @@ -53,7 +52,6 @@ public class SortedInputIterator implements InputIterator { private boolean done = false; private long weight; - private final BytesRefBuilder scratch = new BytesRefBuilder(); private BytesRef payload = new BytesRef(); private Set contexts = null; @@ -86,8 +84,8 @@ public BytesRef next() throws IOException { } try { ByteArrayDataInput input = new ByteArrayDataInput(); - if (reader.read(scratch)) { - final BytesRef bytes = scratch.get(); + BytesRef bytes = reader.next(); + if (bytes != null) { weight = decode(bytes, input); if (hasPayloads) { payload = decodePayload(bytes, input); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java index d7f17a00af7a..19982a5e5e6d 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java @@ -509,8 +509,12 @@ public void build(InputIterator iterator) throws IOException { Set seenSurfaceForms = new HashSet<>(); int dedup = 0; - while (reader.read(scratch)) { - input.reset(scratch.bytes(), 0, scratch.length()); + while (true) { + BytesRef bytes = reader.next(); + if (bytes == null) { + break; + } + input.reset(bytes.bytes, bytes.offset, bytes.length); short analyzedLength = input.readShort(); analyzed.grow(analyzedLength+2); input.readBytes(analyzed.bytes(), 0, analyzedLength); @@ -518,13 +522,13 @@ public void build(InputIterator iterator) throws IOException { long cost = input.readInt(); - surface.bytes = scratch.bytes(); + surface.bytes = bytes.bytes; if (hasPayloads) { surface.length = input.readShort(); surface.offset = input.getPosition(); } else { surface.offset = input.getPosition(); - surface.length = scratch.length() - surface.offset; + surface.length = bytes.length - surface.offset; } if (previousAnalyzed == null) { @@ -566,11 +570,11 @@ public void build(InputIterator iterator) throws IOException { builder.add(scratchInts.get(), outputs.newPair(cost, BytesRef.deepCopyOf(surface))); } else { int payloadOffset = input.getPosition() + surface.length; - int payloadLength = scratch.length() - payloadOffset; + int payloadLength = bytes.length - payloadOffset; BytesRef br = new BytesRef(surface.length + 1 + payloadLength); System.arraycopy(surface.bytes, surface.offset, br.bytes, 0, surface.length); br.bytes[surface.length] = PAYLOAD_SEP; - System.arraycopy(scratch.bytes(), payloadOffset, br.bytes, surface.length+1, payloadLength); + System.arraycopy(bytes.bytes, payloadOffset, br.bytes, surface.length+1, payloadLength); br.length = br.bytes.length; builder.add(scratchInts.get(), outputs.newPair(cost, br)); } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java index 9852eec170c4..fb876d2b4518 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java @@ -24,7 +24,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.OfflineSorter; @@ -105,9 +104,10 @@ public void close() throws IOException { /** * Iterate over byte refs in a file. */ + // TODO: this class is a bit silly ... sole purpose is to "remove" Closeable from what #iterator returns: class ByteSequenceIterator implements BytesRefIterator { private final OfflineSorter.ByteSequencesReader reader; - private BytesRefBuilder scratch = new BytesRefBuilder(); + private BytesRef scratch; public ByteSequenceIterator(OfflineSorter.ByteSequencesReader reader) { this.reader = reader; @@ -115,20 +115,14 @@ public ByteSequenceIterator(OfflineSorter.ByteSequencesReader reader) { @Override public BytesRef next() throws IOException { - if (scratch == null) { - return null; - } boolean success = false; try { - if (reader.read(scratch) == false) { - IOUtils.close(reader); - scratch = null; - } - success = true; + scratch = reader.next(); if (scratch == null) { - return null; + reader.close(); } - return scratch.get(); + success = true; + return scratch; } finally { if (!success) { IOUtils.closeWhileHandlingException(reader); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index df6b1c5d2b2b..7db97a8a1855 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -39,7 +39,6 @@ import org.apache.lucene.util.Accountables; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.OfflineSorter; @@ -215,10 +214,13 @@ public void build(InputIterator iterator) throws IOException { int previousBucket = 0; int previousScore = 0; ByteArrayDataInput input = new ByteArrayDataInput(); - BytesRefBuilder tmp1 = new BytesRefBuilder(); BytesRef tmp2 = new BytesRef(); - while (reader.read(tmp1)) { - input.reset(tmp1.bytes()); + while (true) { + BytesRef scratch = reader.next(); + if (scratch == null) { + break; + } + input.reset(scratch.bytes, scratch.offset, scratch.length); int currentScore = input.readInt(); int bucket; @@ -231,9 +233,9 @@ public void build(InputIterator iterator) throws IOException { previousBucket = bucket; // Only append the input, discard the weight. - tmp2.bytes = tmp1.bytes(); - tmp2.offset = input.getPosition(); - tmp2.length = tmp1.length() - input.getPosition(); + tmp2.bytes = scratch.bytes; + tmp2.offset = scratch.offset + input.getPosition(); + tmp2.length = scratch.length - input.getPosition(); builder.add(tmp2, bucket); line++; @@ -293,7 +295,7 @@ public Object get(CharSequence key) { @Override public synchronized boolean store(DataOutput output) throws IOException { output.writeVLong(count); - if (this.normalCompletion == null || normalCompletion.getFST() == null) { + if (normalCompletion == null || normalCompletion.getFST() == null) { return false; } normalCompletion.getFST().save(output); diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java index b64d283c4145..b26a2ad99ee8 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java @@ -57,12 +57,13 @@ private void check(BytesRefSorter sorter) throws Exception { sorter.add(new BytesRef(new byte [1])); }); - BytesRef spare1; - BytesRef spare2; - while ((spare1 = i1.next()) != null && (spare2 = i2.next()) != null) { + while (true) { + BytesRef spare1 = i1.next(); + BytesRef spare2 = i2.next(); assertEquals(spare1, spare2); + if (spare1 == null) { + break; + } } - assertNull(i1.next()); - assertNull(i2.next()); } } From 9f9c2f74b3adb380ca26ab054e3bbbb1161c3074 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sat, 26 Mar 2016 06:25:53 -0400 Subject: [PATCH 0222/1113] the scratch BytesRef must be private to each offline reader --- lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index e075cedbfc29..33f72e4e9e1d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -735,8 +735,6 @@ public int compare(BytesRef a, BytesRef b) { // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: IndexOutput[] lastWriter = new IndexOutput[1]; - final BytesRef scratch = new BytesRef(new byte[bytesPerDoc]); - OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @@ -756,6 +754,7 @@ public void write(byte[] bytes, int off, int len) throws IOException { @Override protected ByteSequencesReader getReader(ChecksumIndexInput in, String name) throws IOException { return new ByteSequencesReader(in, name) { + final BytesRef scratch = new BytesRef(new byte[bytesPerDoc]); @Override public BytesRef next() throws IOException { if (in.getFilePointer() >= end) { From 28a4335813df24e3db8269efe60347c2275e0b03 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 27 Mar 2016 05:52:00 -0400 Subject: [PATCH 0223/1113] LUCENE-7142: BKDWriter wasn't splitting correctly with long ords; improve tests so we sometimes long ords even for small number of points --- .../org/apache/lucene/util/bkd/BKDWriter.java | 27 +++++++--- .../lucene/util/bkd/OfflinePointReader.java | 11 ++-- .../lucene/util/bkd/Test2BBKDPoints.java | 6 ++- .../org/apache/lucene/util/bkd/TestBKD.java | 51 ++++++++++++++++++- .../org/apache/lucene/index/RandomCodec.java | 24 ++++++++- 5 files changed, 104 insertions(+), 15 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 33f72e4e9e1d..3cf32a9b0d30 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -128,21 +128,31 @@ public class BKDWriter implements Closeable { protected long pointCount; /** true if we have so many values that we must write ords using long (8 bytes) instead of int (4 bytes) */ - private final boolean longOrds; + protected final boolean longOrds; /** An upper bound on how many points the caller will add (includes deletions) */ private final long totalPointCount; /** True if every document has at most one value. We specialize this case by not bothering to store the ord since it's redundant with docID. */ - private final boolean singleValuePerDoc; + protected final boolean singleValuePerDoc; + + /** How much heap OfflineSorter is allowed to use */ + protected final OfflineSorter.BufferSize offlineSorterBufferMB; + + /** How much heap OfflineSorter is allowed to use */ + protected final int offlineSorterMaxTempFiles; private final int maxDoc; - public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, long totalPointCount, boolean singleValuePerDoc) throws IOException { - this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, DEFAULT_MAX_POINTS_IN_LEAF_NODE, DEFAULT_MAX_MB_SORT_IN_HEAP, totalPointCount, singleValuePerDoc); + public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, + int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException { + this(maxDoc, tempDir, tempFileNamePrefix, numDims, bytesPerDim, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount, singleValuePerDoc, + totalPointCount > Integer.MAX_VALUE, Math.max(1, (long) maxMBSortInHeap), OfflineSorter.MAX_TEMPFILES); } - public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, boolean singleValuePerDoc) throws IOException { + protected BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int numDims, int bytesPerDim, + int maxPointsInLeafNode, double maxMBSortInHeap, long totalPointCount, + boolean singleValuePerDoc, boolean longOrds, long offlineSorterBufferMB, int offlineSorterMaxTempFiles) throws IOException { verifyParams(numDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount); // We use tracking dir to deal with removing files on exception, so each place that // creates temp files doesn't need crazy try/finally/sucess logic: @@ -153,6 +163,8 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n this.bytesPerDim = bytesPerDim; this.totalPointCount = totalPointCount; this.maxDoc = maxDoc; + this.offlineSorterBufferMB = OfflineSorter.BufferSize.megabytes(offlineSorterBufferMB); + this.offlineSorterMaxTempFiles = offlineSorterMaxTempFiles; docsSeen = new FixedBitSet(maxDoc); packedBytesLength = numDims * bytesPerDim; @@ -166,7 +178,8 @@ public BKDWriter(int maxDoc, Directory tempDir, String tempFileNamePrefix, int n maxPackedValue = new byte[packedBytesLength]; // If we may have more than 1+Integer.MAX_VALUE values, then we must encode ords with long (8 bytes), else we can use int (4 bytes). - longOrds = totalPointCount > Integer.MAX_VALUE; + this.longOrds = longOrds; + this.singleValuePerDoc = singleValuePerDoc; // dimensional values (numDims * bytesPerDim) + ord (int or long) + docID (int) @@ -735,7 +748,7 @@ public int compare(BytesRef a, BytesRef b) { // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: IndexOutput[] lastWriter = new IndexOutput[1]; - OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, OfflineSorter.BufferSize.megabytes(Math.max(1, (long) maxMBSortInHeap)), OfflineSorter.MAX_TEMPFILES) { + OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, offlineSorterBufferMB, offlineSorterMaxTempFiles) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java index 20faa9e361dd..17758c02c934 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java @@ -198,12 +198,15 @@ public long split(long count, LongBitSet rightTree, PointWriter left, PointWrite in.readBytes(buffer, 0, buffer.length); long ord; - if (singleValuePerDoc == false) { - ord = readInt(buffer, packedBytesLength+Integer.BYTES); - } else if (longOrds) { + if (longOrds) { + // A long ord, after the docID: ord = readLong(buffer, packedBytesLength+Integer.BYTES); - } else { + } else if (singleValuePerDoc) { + // docID is the ord: ord = readInt(buffer, packedBytesLength); + } else { + // An int ord, after the docID: + ord = readInt(buffer, packedBytesLength+Integer.BYTES); } if (rightTree.get(ord)) { diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java index ffcbcf83cc59..af2e463102c4 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/Test2BBKDPoints.java @@ -40,7 +40,8 @@ public void test1D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, 26L * numDocs, false); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 1, Long.BYTES, + BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false); int counter = 0; byte[] packedBytes = new byte[Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { @@ -73,7 +74,8 @@ public void test2D() throws Exception { final int numDocs = (Integer.MAX_VALUE / 26) + 100; - BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, 26L * numDocs, false); + BKDWriter w = new BKDWriter(numDocs, dir, "_0", 2, Long.BYTES, + BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, 26L * numDocs, false); int counter = 0; byte[] packedBytes = new byte[2*Long.BYTES]; for (int docID = 0; docID < numDocs; docID++) { diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index 18fbacf38907..38b3fb5c87ce 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -891,7 +891,7 @@ private void assertCorruptionDetected(Throwable t) { public void testTieBreakOrder() throws Exception { try (Directory dir = newDirectory()) { int numDocs = 10000; - BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, 4, 2, 0.01f, numDocs, true); + BKDWriter w = new BKDWriter(numDocs+1, dir, "tmp", 1, Integer.BYTES, 2, 0.01f, numDocs, true); for(int i=0;i Integer.MAX_VALUE || (getRandomSingleValuePerDoc(singleValuePerDoc, randomSeed) == false && new Random(randomSeed).nextBoolean()); + } + + private static long getRandomOfflineSorterBufferMB(int randomSeed) { + return TestUtil.nextInt(new Random(randomSeed), 1, 8); + } + + private static int getRandomOfflineSorterMaxTempFiles(int randomSeed) { + return TestUtil.nextInt(new Random(randomSeed), 2, 20); + } + @Override protected int split(byte[] minPackedValue, byte[] maxPackedValue) { // BKD normally defaults by the widest dimension, to try to make as squarish cells as possible, but we just pick a random one ;) From bd74f42503c72d87a16474e03f5a26d3c250c6c2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 27 Mar 2016 15:27:17 -0400 Subject: [PATCH 0224/1113] slightly improve polygon testing: test something other than boxes --- .../spatial/util/BaseGeoPointTestCase.java | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index fd97207ef75a..c22b79cd6db4 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -695,18 +695,7 @@ static final boolean rectContainsPoint(GeoRect rect, double pointLat, double poi } } - static final boolean polyRectContainsPoint(GeoRect rect, double pointLat, double pointLon) { - // TODO write better random polygon tests - - // note: logic must be slightly different than rectContainsPoint, to satisfy - // insideness for cases exactly on boundaries. - - assert Double.isNaN(pointLat) == false; - assert rect.crossesDateline() == false; - double polyLats[] = new double[] { rect.minLat, rect.maxLat, rect.maxLat, rect.minLat, rect.minLat }; - double polyLons[] = new double[] { rect.minLon, rect.minLon, rect.maxLon, rect.maxLon, rect.minLon }; - - // TODO: separately test this method is 100% correct, here treat it like a black box (like haversin) + static final boolean polygonContainsPoint(double polyLats[], double polyLons[], double pointLat, double pointLon) { return GeoRelationUtils.pointInPolygon(polyLats, polyLons, pointLat, pointLon); } @@ -940,24 +929,42 @@ protected void describe(int docID, double pointLat, double pointLon) { final GeoRect bbox = randomRect(small, false); // Polygon - double[] polyLats = new double[5]; - double[] polyLons = new double[5]; - polyLats[0] = bbox.minLat; - polyLons[0] = bbox.minLon; - polyLats[1] = bbox.maxLat; - polyLons[1] = bbox.minLon; - polyLats[2] = bbox.maxLat; - polyLons[2] = bbox.maxLon; - polyLats[3] = bbox.minLat; - polyLons[3] = bbox.maxLon; - polyLats[4] = bbox.minLat; - polyLons[4] = bbox.minLon; + final double[] polyLats; + final double[] polyLons; + // TODO: factor this out, maybe if we add Polygon class? + if (random().nextBoolean()) { + // box + polyLats = new double[5]; + polyLons = new double[5]; + polyLats[0] = bbox.minLat; + polyLons[0] = bbox.minLon; + polyLats[1] = bbox.maxLat; + polyLons[1] = bbox.minLon; + polyLats[2] = bbox.maxLat; + polyLons[2] = bbox.maxLon; + polyLats[3] = bbox.minLat; + polyLons[3] = bbox.maxLon; + polyLats[4] = bbox.minLat; + polyLons[4] = bbox.minLon; + } else { + // right triangle + polyLats = new double[4]; + polyLons = new double[4]; + polyLats[0] = bbox.minLat; + polyLons[0] = bbox.minLon; + polyLats[1] = bbox.maxLat; + polyLons[1] = bbox.minLon; + polyLats[2] = bbox.maxLat; + polyLons[2] = bbox.maxLon; + polyLats[3] = bbox.minLat; + polyLons[3] = bbox.minLon; + } query = newPolygonQuery(FIELD_NAME, polyLats, polyLons); verifyHits = new VerifyHits() { @Override protected boolean shouldMatch(double pointLat, double pointLon) { - return polyRectContainsPoint(bbox, pointLat, pointLon); + return polygonContainsPoint(polyLats, polyLons, pointLat, pointLon); } @Override From 0c0a25d747de55d8c63dd9405c258b4ec8ee40b2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 27 Mar 2016 16:58:14 -0400 Subject: [PATCH 0225/1113] test polygon queries with mike's surprise polygon generator --- .../spatial/util/BaseGeoPointTestCase.java | 142 ++++++++++++++---- 1 file changed, 116 insertions(+), 26 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index c22b79cd6db4..c68dcbcf71a5 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashSet; @@ -636,6 +637,86 @@ public double randomLon(boolean small) { } return quantizeLon(result); } + + // for pole crossing: used by surpriseMePolygon() + private static double wrapLat(double lat) { + //System.out.println("wrapLat " + lat); + if (lat > 90) { + //System.out.println(" " + (180 - lat)); + return 180 - lat; + } else if (lat < -90) { + //System.out.println(" " + (-180 - lat)); + return -180 - lat; + } else { + //System.out.println(" " + lat); + return lat; + } + } + + // for dateline crossing: used by surpriseMePolygon() + // TODO: can we remove this? these should not cross dateline... + private static double wrapLon(double lon) { + //System.out.println("wrapLon " + lon); + if (lon > 180) { + //System.out.println(" " + (lon - 360)); + return lon - 360; + } else if (lon < -180) { + //System.out.println(" " + (lon + 360)); + return lon + 360; + } else { + //System.out.println(" " + lon); + return lon; + } + } + + /** Returns {polyLats, polyLons} double[] array */ + private double[][] surpriseMePolygon() { + // repeat until we get a poly that doesn't cross dateline: + newPoly: + while (true) { + //System.out.println("\nPOLY ITER"); + double centerLat = randomLat(false); + double centerLon = randomLon(false); + + double radius = 0.1 + 20 * random().nextDouble(); + double radiusDelta = random().nextDouble(); + + ArrayList lats = new ArrayList<>(); + ArrayList lons = new ArrayList<>(); + double angle = 0.0; + while (true) { + angle += random().nextDouble()*40.0; + //System.out.println(" angle " + angle); + if (angle > 360) { + break; + } + double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); + //System.out.println(" len=" + len); + double lat = wrapLat(centerLat + len * Math.cos(Math.toRadians(angle))); + double lon = centerLon + len * Math.sin(Math.toRadians(angle)); + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + lats.add(wrapLat(lat)); + lons.add(wrapLon(lon)); + + //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); + } + + // close it + lats.add(lats.get(0)); + lons.add(lons.get(0)); + + double[] latsArray = new double[lats.size()]; + double[] lonsArray = new double[lons.size()]; + for(int i=0;i Date: Sun, 27 Mar 2016 17:47:14 -0400 Subject: [PATCH 0226/1113] clean up random poly generation; simplify geo util methods for validating lat/lon --- .../apache/lucene/document/LatLonPoint.java | 12 ++--- .../document/LatLonPointDistanceQuery.java | 8 +-- .../document/LatLonPointInPolygonQuery.java | 8 +-- .../lucene/document/LatLonPointSortField.java | 8 +-- .../geopoint/document/GeoPointField.java | 9 +--- .../search/GeoPointDistanceQuery.java | 8 +-- .../geopoint/search/GeoPointInBBoxQuery.java | 16 ++---- .../search/GeoPointMultiTermQuery.java | 16 ++---- .../apache/lucene/spatial/util/GeoRect.java | 16 ++---- .../apache/lucene/spatial/util/GeoUtils.java | 21 ++++---- .../spatial/util/BaseGeoPointTestCase.java | 50 ++++++------------- 11 files changed, 52 insertions(+), 120 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 429f9792b97f..1278ea56aee7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -137,9 +137,7 @@ public Number numericValue() { * @throws IllegalArgumentException if latitude is out of bounds */ public static int encodeLatitude(double latitude) { - if (GeoUtils.isValidLat(latitude) == false) { - throw new IllegalArgumentException("invalid latitude: " + latitude + ", must be -90 to 90"); - } + GeoUtils.checkLatitude(latitude); // the maximum possible value cannot be encoded without overflow if (latitude == 90.0D) { latitude = Math.nextDown(latitude); @@ -154,9 +152,7 @@ public static int encodeLatitude(double latitude) { * @throws IllegalArgumentException if longitude is out of bounds */ public static int encodeLongitude(double longitude) { - if (GeoUtils.isValidLon(longitude) == false) { - throw new IllegalArgumentException("invalid longitude: " + longitude + ", must be -180 to 180"); - } + GeoUtils.checkLongitude(longitude); // the maximum possible value cannot be encoded without overflow if (longitude == 180.0D) { longitude = Math.nextDown(longitude); @@ -171,7 +167,7 @@ public static int encodeLongitude(double longitude) { */ public static double decodeLatitude(int encoded) { double result = encoded * LATITUDE_DECODE; - assert GeoUtils.isValidLat(result); + assert result >= GeoUtils.MIN_LAT_INCL && result <= GeoUtils.MAX_LAT_INCL; return result; } @@ -192,7 +188,7 @@ public static double decodeLatitude(byte[] src, int offset) { */ public static double decodeLongitude(int encoded) { double result = encoded * LONGITUDE_DECODE; - assert GeoUtils.isValidLon(result); + assert result >= GeoUtils.MIN_LON_INCL && result <= GeoUtils.MAX_LON_INCL; return result; } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 10d11c6f27b4..c283c7e6e9f3 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -61,12 +61,8 @@ public LatLonPointDistanceQuery(String field, double latitude, double longitude, if (Double.isFinite(radiusMeters) == false || radiusMeters < 0) { throw new IllegalArgumentException("radiusMeters: '" + radiusMeters + "' is invalid"); } - if (GeoUtils.isValidLat(latitude) == false) { - throw new IllegalArgumentException("latitude: '" + latitude + "' is invalid"); - } - if (GeoUtils.isValidLon(longitude) == false) { - throw new IllegalArgumentException("longitude: '" + longitude + "' is invalid"); - } + GeoUtils.checkLatitude(latitude); + GeoUtils.checkLongitude(longitude); this.field = field; this.latitude = latitude; this.longitude = longitude; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 494f42733c9f..c263ba340ae6 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -94,15 +94,11 @@ public LatLonPointInPolygonQuery(String field, double[] polyLats, double[] polyL double maxLat = Double.NEGATIVE_INFINITY; for(int i=0;i= MIN_LAT_INCL && lat <= MAX_LAT_INCL; + public static void checkLatitude(double latitude) { + if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) { + throw new IllegalArgumentException("invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL); + } } /** validates longitude value is within standard +/-180 coordinate bounds */ - public static boolean isValidLon(double lon) { - return Double.isNaN(lon) == false && lon >= MIN_LON_INCL && lon <= MAX_LON_INCL; + public static void checkLongitude(double longitude) { + if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) { + throw new IllegalArgumentException("invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL); + } } /** Compute Bounding Box for a circle using WGS-84 parameters */ @@ -115,12 +118,8 @@ public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { double maxLat = Double.NEGATIVE_INFINITY; for (int i=0;i 90) { - //System.out.println(" " + (180 - lat)); - return 180 - lat; - } else if (lat < -90) { - //System.out.println(" " + (-180 - lat)); - return -180 - lat; - } else { - //System.out.println(" " + lat); - return lat; - } - } - - // for dateline crossing: used by surpriseMePolygon() - // TODO: can we remove this? these should not cross dateline... - private static double wrapLon(double lon) { - //System.out.println("wrapLon " + lon); - if (lon > 180) { - //System.out.println(" " + (lon - 360)); - return lon - 360; - } else if (lon < -180) { - //System.out.println(" " + (lon + 360)); - return lon + 360; - } else { - //System.out.println(" " + lon); - return lon; - } - } - /** Returns {polyLats, polyLons} double[] array */ private double[][] surpriseMePolygon() { // repeat until we get a poly that doesn't cross dateline: @@ -692,14 +661,27 @@ private double[][] surpriseMePolygon() { } double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); //System.out.println(" len=" + len); - double lat = wrapLat(centerLat + len * Math.cos(Math.toRadians(angle))); + double lat = centerLat + len * Math.cos(Math.toRadians(angle)); double lon = centerLon + len * Math.sin(Math.toRadians(angle)); if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { // cannot cross dateline: try again! continue newPoly; } - lats.add(wrapLat(lat)); - lons.add(wrapLon(lon)); + if (lat > 90) { + // cross the north pole + lat = 180 - lat; + lon = 180 - lon; + } else if (lat < -90) { + // cross the south pole + lat = -180 - lat; + lon = 180 - lon; + } + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + lats.add(lat); + lons.add(lon); //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); } From 967f7d3492e17ef849892aed034de20ff6b01754 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 28 Mar 2016 10:06:49 +0200 Subject: [PATCH 0227/1113] LUCENE-6954 - keep info about relationship between fields and terms when retrieving terms in MLT (cherry picked from commit e8dac9b) --- .../lucene/queries/mlt/MoreLikeThis.java | 129 ++++++++++-------- .../lucene/queries/mlt/TestMoreLikeThis.java | 82 +++++++++++ 2 files changed, 155 insertions(+), 56 deletions(-) diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java index 161dddb7f60b..ea02af3f8f43 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java @@ -603,11 +603,11 @@ public Query like(Map> filteredDocument) throws IOExc * @return a query that will return docs like the passed Readers. */ public Query like(String fieldName, Reader... readers) throws IOException { - Map words = new HashMap<>(); + Map> perFieldTermFrequencies = new HashMap<>(); for (Reader r : readers) { - addTermFrequencies(r, words, fieldName); + addTermFrequencies(r, perFieldTermFrequencies, fieldName); } - return createQuery(createQueue(words)); + return createQuery(createQueue(perFieldTermFrequencies)); } /** @@ -642,58 +642,65 @@ private Query createQuery(PriorityQueue q) { /** * Create a PriorityQueue from a word->tf map. * - * @param words a map of words keyed on the word(String) with Int objects as the values. + * @param perFieldTermFrequencies a per field map of words keyed on the word(String) with Int objects as the values. */ - private PriorityQueue createQueue(Map words) throws IOException { + private PriorityQueue createQueue(Map> perFieldTermFrequencies) throws IOException { // have collected all words in doc and their freqs int numDocs = ir.numDocs(); - final int limit = Math.min(maxQueryTerms, words.size()); + final int limit = Math.min(maxQueryTerms, this.getTermsCount(perFieldTermFrequencies)); FreqQ queue = new FreqQ(limit); // will order words by score + for (Map.Entry> entry : perFieldTermFrequencies.entrySet()) { + Map perWordTermFrequencies = entry.getValue(); + String fieldName = entry.getKey(); + + for (Map.Entry tfEntry : perWordTermFrequencies.entrySet()) { // for every word + String word = tfEntry.getKey(); + int tf = tfEntry.getValue().x; // term freq in the source doc + if (minTermFreq > 0 && tf < minTermFreq) { + continue; // filter out words that don't occur enough times in the source + } - for (String word : words.keySet()) { // for every word - int tf = words.get(word).x; // term freq in the source doc - if (minTermFreq > 0 && tf < minTermFreq) { - continue; // filter out words that don't occur enough times in the source - } + int docFreq = ir.docFreq(new Term(fieldName, word)); - // go through all the fields and find the largest document frequency - String topField = fieldNames[0]; - int docFreq = 0; - for (String fieldName : fieldNames) { - int freq = ir.docFreq(new Term(fieldName, word)); - topField = (freq > docFreq) ? fieldName : topField; - docFreq = (freq > docFreq) ? freq : docFreq; - } - - if (minDocFreq > 0 && docFreq < minDocFreq) { - continue; // filter out words that don't occur in enough docs - } + if (minDocFreq > 0 && docFreq < minDocFreq) { + continue; // filter out words that don't occur in enough docs + } - if (docFreq > maxDocFreq) { - continue; // filter out words that occur in too many docs - } + if (docFreq > maxDocFreq) { + continue; // filter out words that occur in too many docs + } - if (docFreq == 0) { - continue; // index update problem? - } + if (docFreq == 0) { + continue; // index update problem? + } - float idf = similarity.idf(docFreq, numDocs); - float score = tf * idf; + float idf = similarity.idf(docFreq, numDocs); + float score = tf * idf; - if (queue.size() < limit) { - // there is still space in the queue - queue.add(new ScoreTerm(word, topField, score, idf, docFreq, tf)); - } else { - ScoreTerm term = queue.top(); - if (term.score < score) { // update the smallest in the queue in place and update the queue. - term.update(word, topField, score, idf, docFreq, tf); - queue.updateTop(); + if (queue.size() < limit) { + // there is still space in the queue + queue.add(new ScoreTerm(word, fieldName, score, idf, docFreq, tf)); + } else { + ScoreTerm term = queue.top(); + if (term.score < score) { // update the smallest in the queue in place and update the queue. + term.update(word, fieldName, score, idf, docFreq, tf); + queue.updateTop(); + } } } } return queue; } + private int getTermsCount(Map> perFieldTermFrequencies) { + int totalTermsCount = 0; + Collection> values = perFieldTermFrequencies.values(); + for (Map perWordTermFrequencies : values) { + totalTermsCount += perWordTermFrequencies.size(); + } + return totalTermsCount; + } + /** * Describe the parameters that control how the "more like this" query is formed. */ @@ -721,7 +728,7 @@ public String describeParams() { * @param docNum the id of the lucene document from which to find terms */ private PriorityQueue retrieveTerms(int docNum) throws IOException { - Map termFreqMap = new HashMap<>(); + Map> field2termFreqMap = new HashMap<>(); for (String fieldName : fieldNames) { final Fields vectors = ir.getTermVectors(docNum); final Terms vector; @@ -738,43 +745,48 @@ private PriorityQueue retrieveTerms(int docNum) throws IOException { for (IndexableField field : fields) { final String stringValue = field.stringValue(); if (stringValue != null) { - addTermFrequencies(new StringReader(stringValue), termFreqMap, fieldName); + addTermFrequencies(new StringReader(stringValue), field2termFreqMap, fieldName); } } } else { - addTermFrequencies(termFreqMap, vector); + addTermFrequencies(field2termFreqMap, vector, fieldName); } } - return createQueue(termFreqMap); + return createQueue(field2termFreqMap); } - private PriorityQueue retrieveTerms(Map> fields) throws + private PriorityQueue retrieveTerms(Map> field2fieldValues) throws IOException { - HashMap termFreqMap = new HashMap<>(); + Map> field2termFreqMap = new HashMap<>(); for (String fieldName : fieldNames) { - for (String field : fields.keySet()) { - Collection fieldValues = fields.get(field); + for (String field : field2fieldValues.keySet()) { + Collection fieldValues = field2fieldValues.get(field); if(fieldValues == null) continue; for(Object fieldValue:fieldValues) { if (fieldValue != null) { - addTermFrequencies(new StringReader(String.valueOf(fieldValue)), termFreqMap, + addTermFrequencies(new StringReader(String.valueOf(fieldValue)), field2termFreqMap, fieldName); } } } } - return createQueue(termFreqMap); + return createQueue(field2termFreqMap); } /** * Adds terms and frequencies found in vector into the Map termFreqMap * - * @param termFreqMap a Map of terms and their frequencies + * @param field2termFreqMap a Map of terms and their frequencies per field * @param vector List of terms and their frequencies for a doc/field */ - private void addTermFrequencies(Map termFreqMap, Terms vector) throws IOException { + private void addTermFrequencies(Map> field2termFreqMap, Terms vector, String fieldName) throws IOException { + Map termFreqMap = field2termFreqMap.get(fieldName); + if (termFreqMap == null) { + termFreqMap = new HashMap<>(); + field2termFreqMap.put(fieldName, termFreqMap); + } final TermsEnum termsEnum = vector.iterator(); final CharsRefBuilder spare = new CharsRefBuilder(); BytesRef text; @@ -802,15 +814,20 @@ private void addTermFrequencies(Map termFreqMap, Terms vector) thro * Adds term frequencies found by tokenizing text from reader into the Map words * * @param r a source of text to be tokenized - * @param termFreqMap a Map of terms and their frequencies + * @param perFieldTermFrequencies a Map of terms and their frequencies per field * @param fieldName Used by analyzer for any special per-field analysis */ - private void addTermFrequencies(Reader r, Map termFreqMap, String fieldName) + private void addTermFrequencies(Reader r, Map> perFieldTermFrequencies, String fieldName) throws IOException { if (analyzer == null) { throw new UnsupportedOperationException("To use MoreLikeThis without " + "term vectors, you must provide an Analyzer"); } + Map termFreqMap = perFieldTermFrequencies.get(fieldName); + if (termFreqMap == null) { + termFreqMap = new HashMap<>(); + perFieldTermFrequencies.put(fieldName, termFreqMap); + } try (TokenStream ts = analyzer.tokenStream(fieldName, r)) { int tokenCount = 0; // for every token @@ -880,9 +897,9 @@ private boolean isNoiseWord(String term) { * @see #retrieveInterestingTerms */ private PriorityQueue retrieveTerms(Reader r, String fieldName) throws IOException { - Map words = new HashMap<>(); - addTermFrequencies(r, words, fieldName); - return createQueue(words); + Map> field2termFreqMap = new HashMap<>(); + addTermFrequencies(r, field2termFreqMap, fieldName); + return createQueue(field2termFreqMap); } /** diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index 6eb42662c62e..5e6466fc0615 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -40,8 +41,14 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.junit.Test; public class TestMoreLikeThis extends LuceneTestCase { + + private static final String SHOP_TYPE = "type"; + private static final String FOR_SALE = "weSell"; + private static final String NOT_FOR_SALE = "weDontSell"; + private Directory directory; private IndexReader reader; private IndexSearcher searcher; @@ -246,5 +253,80 @@ private String[] generateStrSeq(int from, int size) { return generatedStrings; } + private int addShopDoc(RandomIndexWriter writer, String type, String[] weSell, String[] weDontSell) throws IOException { + Document doc = new Document(); + doc.add(newTextField(SHOP_TYPE, type, Field.Store.YES)); + for (String item : weSell) { + doc.add(newTextField(FOR_SALE, item, Field.Store.YES)); + } + for (String item : weDontSell) { + doc.add(newTextField(NOT_FOR_SALE, item, Field.Store.YES)); + } + writer.addDocument(doc); + return writer.numDocs() - 1; + } + + public void testMultiFieldShouldReturnPerFieldBooleanQuery() throws Exception { + IndexReader reader = null; + Directory dir = newDirectory(); + Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + try { + int maxQueryTerms = 25; + + String[] itShopItemForSale = new String[]{"watch", "ipod", "asrock", "imac", "macbookpro", "monitor", "keyboard", "mouse", "speakers"}; + String[] itShopItemNotForSale = new String[]{"tie", "trousers", "shoes", "skirt", "hat"}; + + String[] clothesShopItemForSale = new String[]{"tie", "trousers", "shoes", "skirt", "hat"}; + String[] clothesShopItemNotForSale = new String[]{"watch", "ipod", "asrock", "imac", "macbookpro", "monitor", "keyboard", "mouse", "speakers"}; + + // add series of shop docs + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + for (int i = 0; i < 100; i++) { + addShopDoc(writer, "it", itShopItemForSale, itShopItemNotForSale); + } + for (int i = 0; i < 10; i++) { + addShopDoc(writer, "clothes", clothesShopItemForSale, clothesShopItemNotForSale); + } + // Input Document is a clothes shop + int inputDocId = addShopDoc(writer, "clothes", clothesShopItemForSale, clothesShopItemNotForSale); + reader = writer.getReader(); + writer.close(); + + // setup MLT query + MoreLikeThis mlt = new MoreLikeThis(reader); + + mlt.setAnalyzer(analyzer); + mlt.setMaxQueryTerms(maxQueryTerms); + mlt.setMinDocFreq(1); + mlt.setMinTermFreq(1); + mlt.setMinWordLen(1); + mlt.setFieldNames(new String[]{FOR_SALE, NOT_FOR_SALE}); + + // perform MLT query + BooleanQuery query = (BooleanQuery) mlt.like(inputDocId); + Collection clauses = query.clauses(); + + Collection expectedClothesShopClauses = new ArrayList(); + for (String itemForSale : clothesShopItemForSale) { + BooleanClause booleanClause = new BooleanClause(new TermQuery(new Term(FOR_SALE, itemForSale)), BooleanClause.Occur.SHOULD); + expectedClothesShopClauses.add(booleanClause); + } + for (String itemNotForSale : clothesShopItemNotForSale) { + BooleanClause booleanClause = new BooleanClause(new TermQuery(new Term(NOT_FOR_SALE, itemNotForSale)), BooleanClause.Occur.SHOULD); + expectedClothesShopClauses.add(booleanClause); + } + + for (BooleanClause expectedClause : expectedClothesShopClauses) { + assertTrue(clauses.contains(expectedClause)); + } + } finally { + // clean up + if (reader != null) { + reader.close(); + } + dir.close(); + analyzer.close(); + } + } // TODO: add tests for the MoreLikeThisQuery } From d8dd06f422614961b029384c379e96a732fd7476 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 28 Mar 2016 07:46:08 -0400 Subject: [PATCH 0228/1113] LUCENE-7126: GeoPointDistanceRangeQuery not valid for multi-valued docs --- lucene/CHANGES.txt | 3 + .../lucene/search/TestLatLonPointQueries.java | 5 - .../search/GeoPointDistanceRangeQuery.java | 122 ------------------ .../geopoint/search/TestGeoPointQuery.java | 7 - .../search/TestLegacyGeoPointQuery.java | 7 - .../spatial/util/BaseGeoPointTestCase.java | 47 +------ 6 files changed, 8 insertions(+), 183 deletions(-) delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 90fb82ff09ab..ed1d83d3ec73 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -225,6 +225,9 @@ Bug Fixes * LUCENE-7112: WeightedSpanTermExtractor.extractUnknownQuery is only called on queries that could not be extracted. (Adrien Grand) +* LUCENE-7126: Remove GeoPointDistanceRangeQuery. This query was implemented + with boolean NOT, and incorrect for multi-valued documents. (Robert Muir) + Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 4f2c2d7a23c5..3f4da8ad1afb 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -37,11 +37,6 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo return LatLonPoint.newDistanceQuery(field, centerLat, centerLon, radiusMeters); } - @Override - protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - return null; - } - @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return LatLonPoint.newPolygonQuery(field, lats, lons); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java deleted file mode 100644 index 5cc778a5e7c7..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceRangeQuery.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.geopoint.search; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; - -/** Implements a point distance range query on a GeoPoint field. This is based on - * {@code org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery} and is implemented using a - * {@code org.apache.lucene.search.BooleanClause.MUST_NOT} clause to exclude any points that fall within - * minRadiusMeters from the provided point. - * - * @lucene.experimental - */ -public final class GeoPointDistanceRangeQuery extends GeoPointDistanceQuery { - /** minimum distance range (in meters) from lat, lon center location, maximum is inherited */ - protected final double minRadiusMeters; - - /** - * Constructs a query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types within a minimum / maximum - * distance (in meters) range from a given point - */ - public GeoPointDistanceRangeQuery(final String field, final double centerLat, final double centerLon, - final double minRadiusMeters, final double maxRadiusMeters) { - this(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, maxRadiusMeters); - } - - /** - * Constructs a query for all {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} types within a minimum / maximum - * distance (in meters) range from a given point. Accepts an optional - * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} - */ - public GeoPointDistanceRangeQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, - final double minRadiusMeters, final double maxRadius) { - super(field, termEncoding, centerLat, centerLon, maxRadius); - this.minRadiusMeters = minRadiusMeters; - } - - @Override - public Query rewrite(IndexReader reader) { - Query q = super.rewrite(reader); - if (minRadiusMeters == 0.0) { - return q; - } - - // add an exclusion query - BooleanQuery.Builder bqb = new BooleanQuery.Builder(); - - // create a new exclusion query - GeoPointDistanceQuery exclude = new GeoPointDistanceQuery(field, termEncoding, centerLat, centerLon, minRadiusMeters); - // full map search -// if (radiusMeters >= GeoProjectionUtils.SEMIMINOR_AXIS) { -// bqb.add(new BooleanClause(new GeoPointInBBoxQuery(this.field, -180.0, -90.0, 180.0, 90.0), BooleanClause.Occur.MUST)); -// } else { - bqb.add(new BooleanClause(q, BooleanClause.Occur.MUST)); -// } - bqb.add(new BooleanClause(exclude, BooleanClause.Occur.MUST_NOT)); - - return bqb.build(); - } - - @Override - public String toString(String field) { - final StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()); - sb.append(':'); - if (!this.field.equals(field)) { - sb.append(" field="); - sb.append(this.field); - sb.append(':'); - } - return sb.append( " Center: [") - .append(centerLat) - .append(',') - .append(centerLon) - .append(']') - .append(" From Distance: ") - .append(minRadiusMeters) - .append(" m") - .append(" To Distance: ") - .append(radiusMeters) - .append(" m") - .append(" Lower Left: [") - .append(minLat) - .append(',') - .append(minLon) - .append(']') - .append(" Upper Right: [") - .append(maxLat) - .append(',') - .append(maxLon) - .append("]") - .toString(); - } - - /** getter method for minimum distance */ - public double getMinRadiusMeters() { - return this.minRadiusMeters; - } - - /** getter method for maximum distance */ - public double getMaxRadiusMeters() { - return this.radiusMeters; - } -} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 1a739a3e3fdc..0f0eaeb555c6 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -55,13 +55,6 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo return new GeoPointDistanceQuery(field, TermEncoding.PREFIX, centerLat, centerLon, radiusMeters); } - @Override - protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - // LUCENE-7126: currently not valid for multi-valued documents, because it rewrites to a BooleanQuery! - // return new GeoPointDistanceRangeQuery(field, TermEncoding.PREFIX, centerLat, centerLon, minRadiusMeters, radiusMeters); - return null; - } - @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, lats, lons); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index c2f74f16deb6..e9b4c1273e81 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -55,13 +55,6 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo return new GeoPointDistanceQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, radiusMeters); } - @Override - protected Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters) { - // LUCENE-7126: currently not valid for multi-valued documents, because it rewrites to a BooleanQuery! - // return new GeoPointDistanceRangeQuery(field, TermEncoding.NUMERIC, centerLat, centerLon, minRadiusMeters, radiusMeters); - return null; - } - @Override protected Query newPolygonQuery(String field, double[] lats, double[] lons) { return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lats, lons); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 6ca3318b63c7..1c54f4e5222e 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -742,8 +742,6 @@ protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters); - protected abstract Query newDistanceRangeQuery(String field, double centerLat, double centerLon, double minRadiusMeters, double radiusMeters); - protected abstract Query newPolygonQuery(String field, double[] lats, double[] lons); static final boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { @@ -769,11 +767,6 @@ static final boolean circleContainsPoint(double centerLat, double centerLon, dou return result; } - static final boolean distanceRangeContainsPoint(double centerLat, double centerLon, double minRadiusMeters, double radiusMeters, double pointLat, double pointLon) { - final double d = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - return d >= minRadiusMeters && d <= radiusMeters; - } - private static abstract class VerifyHits { public void test(AtomicBoolean failed, boolean small, IndexSearcher s, NumericDocValues docIDToID, Set deleted, Query query, double[] lats, double[] lons) throws Exception { @@ -929,13 +922,10 @@ protected void describe(int docID, double lat, double lon) { } else if (random().nextBoolean()) { // Distance - final boolean rangeQuery = random().nextBoolean(); final double centerLat = randomLat(small); final double centerLon = randomLon(small); - double radiusMeters; - double minRadiusMeters; - + final double radiusMeters; if (small) { // Approx 3 degrees lon at the equator: radiusMeters = random().nextDouble() * 333000 + 1.0; @@ -944,36 +934,17 @@ protected void describe(int docID, double lat, double lon) { radiusMeters = random().nextDouble() * GeoUtils.SEMIMAJOR_AXIS * Math.PI / 2.0 + 1.0; } - // generate a random minimum radius between 1% and 95% the max radius - minRadiusMeters = (0.01 + 0.94 * random().nextDouble()) * radiusMeters; - if (VERBOSE) { final DecimalFormat df = new DecimalFormat("#,###.00", DecimalFormatSymbols.getInstance(Locale.ENGLISH)); - System.out.println(" radiusMeters = " + df.format(radiusMeters) - + ((rangeQuery == true) ? " minRadiusMeters = " + df.format(minRadiusMeters) : "")); + System.out.println(" radiusMeters = " + df.format(radiusMeters)); } - try { - if (rangeQuery == true) { - query = newDistanceRangeQuery(FIELD_NAME, centerLat, centerLon, minRadiusMeters, radiusMeters); - } else { - query = newDistanceQuery(FIELD_NAME, centerLat, centerLon, radiusMeters); - } - } catch (IllegalArgumentException e) { - if (e.getMessage().contains("exceeds maxRadius")) { - continue; - } - throw e; - } + query = newDistanceQuery(FIELD_NAME, centerLat, centerLon, radiusMeters); verifyHits = new VerifyHits() { @Override protected boolean shouldMatch(double pointLat, double pointLon) { - if (rangeQuery == false) { - return circleContainsPoint(centerLat, centerLon, radiusMeters, pointLat, pointLon); - } else { - return distanceRangeContainsPoint(centerLat, centerLon, minRadiusMeters, radiusMeters, pointLat, pointLon); - } + return circleContainsPoint(centerLat, centerLon, radiusMeters, pointLat, pointLon); } @Override @@ -981,7 +952,7 @@ protected void describe(int docID, double pointLat, double pointLon) { double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); System.out.println(" docID=" + docID + " centerLat=" + centerLat + " centerLon=" + centerLon + " pointLat=" + pointLat + " pointLon=" + pointLon + " distanceMeters=" + distanceMeters - + " vs" + ((rangeQuery == true) ? " minRadiusMeters=" + minRadiusMeters : "") + " radiusMeters=" + radiusMeters); + + " vs radiusMeters=" + radiusMeters); } }; @@ -1206,14 +1177,6 @@ public void testEquals() throws Exception { assertEquals(q1, q2); assertFalse(q1.equals(newDistanceQuery("field2", lat, lon, 10000.0))); - q1 = newDistanceRangeQuery("field", lat, lon, 10000.0, 100000.0); - if (q1 != null) { - // Not all subclasses can make distance range query! - q2 = newDistanceRangeQuery("field", lat, lon, 10000.0, 100000.0); - assertEquals(q1, q2); - assertFalse(q1.equals(newDistanceRangeQuery("field2", lat, lon, 10000.0, 100000.0))); - } - double[] lats = new double[5]; double[] lons = new double[5]; lats[0] = rect.minLat; From 51feadd185c4624294684f27f816faf65aba3bde Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 28 Mar 2016 09:42:29 -0400 Subject: [PATCH 0229/1113] LUCENE-7144: forceMerge before pulling reader --- .../apache/lucene/search/spans/TestFieldMaskingSpanQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java index b4435e7c1407..c2e2c7d6bccd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java @@ -115,8 +115,8 @@ public static void beforeClass() throws Exception { field("gender", "male"), field("first", "bubba"), field("last", "jones") })); - reader = writer.getReader(); writer.forceMerge(1); + reader = writer.getReader(); writer.close(); searcher = new IndexSearcher(getOnlyLeafReader(reader)); } From 81f32f272f0dc781daa518b71ed300dfef03ca08 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 28 Mar 2016 10:53:25 -0400 Subject: [PATCH 0230/1113] geo tests: simplify random generation --- .../spatial/util/BaseGeoPointTestCase.java | 50 ++------- .../lucene/spatial/util/GeoTestUtil.java | 104 ++++++++++++++++++ .../lucene/spatial/util/TestGeoUtils.java | 58 ++++------ 3 files changed, 134 insertions(+), 78 deletions(-) create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 1c54f4e5222e..ce908e70ef1d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -82,41 +82,11 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { private static double originLat; private static double originLon; - private static double lonRange; - private static double latRange; @BeforeClass public static void beforeClassBase() throws Exception { - // Between 1.0 and 3.0: - lonRange = 2 * (random().nextDouble() + 0.5); - latRange = 2 * (random().nextDouble() + 0.5); - - originLon = normalizeLon(GeoUtils.MIN_LON_INCL + lonRange + (GeoUtils.MAX_LON_INCL - GeoUtils.MIN_LON_INCL - 2 * lonRange) * random().nextDouble()); - originLat = normalizeLat(GeoUtils.MIN_LAT_INCL + latRange + (GeoUtils.MAX_LAT_INCL - GeoUtils.MIN_LAT_INCL - 2 * latRange) * random().nextDouble()); - } - - /** Puts longitude in range of -180 to +180. */ - public static double normalizeLon(double lon_deg) { - if (lon_deg >= -180 && lon_deg <= 180) { - return lon_deg; //common case, and avoids slight double precision shifting - } - double off = (lon_deg + 180) % 360; - if (off < 0) { - return 180 + off; - } else if (off == 0 && lon_deg > 0) { - return 180; - } else { - return -180 + off; - } - } - - /** Puts latitude in range of -90 to 90. */ - public static double normalizeLat(double lat_deg) { - if (lat_deg >= -90 && lat_deg <= 90) { - return lat_deg; //common case, and avoids slight double precision shifting - } - double off = Math.abs((lat_deg + 90) % 360); - return (off <= 180 ? off : 360-off) - 90; + originLon = GeoTestUtil.nextLongitude(); + originLat = GeoTestUtil.nextLatitude(); } /** Valid values that should not cause exception */ @@ -621,9 +591,9 @@ private void doTestRandom(int count) throws Exception { public double randomLat(boolean small) { double result; if (small) { - result = normalizeLat(originLat + latRange * (random().nextDouble() - 0.5)); + result = GeoTestUtil.nextLatitudeNear(originLat); } else { - result = -90 + 180.0 * random().nextDouble(); + result = GeoTestUtil.nextLatitude(); } return quantizeLat(result); } @@ -631,9 +601,9 @@ public double randomLat(boolean small) { public double randomLon(boolean small) { double result; if (small) { - result = normalizeLon(originLon + lonRange * (random().nextDouble() - 0.5)); + result = GeoTestUtil.nextLongitudeNear(originLon); } else { - result = -180 + 360.0 * random().nextDouble(); + result = GeoTestUtil.nextLongitude(); } return quantizeLon(result); } @@ -1107,8 +1077,8 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); for (int i = 0; i < numDocs; i++) { - double latRaw = -90 + 180.0 * random().nextDouble(); - double lonRaw = -180 + 360.0 * random().nextDouble(); + double latRaw = GeoTestUtil.nextLatitude(); + double lonRaw = GeoTestUtil.nextLongitude(); // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons double lat = quantizeLat(latRaw); double lon = quantizeLon(lonRaw); @@ -1122,8 +1092,8 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double radius = 50000000D * random().nextDouble(); BitSet expected = new BitSet(); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java new file mode 100644 index 000000000000..f28e623f5f11 --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.util; + +import java.util.Random; + +import com.carrotsearch.randomizedtesting.RandomizedContext; + +/** static methods for testing geo */ +public class GeoTestUtil { + + /** returns next pseudorandom latitude (anywhere) */ + public static double nextLatitude() { + return -90 + 180.0 * random().nextDouble(); + } + + + /** returns next pseudorandom longitude (anywhere) */ + public static double nextLongitude() { + return -180 + 360.0 * random().nextDouble(); + } + + /** returns next pseudorandom latitude, kinda close to {@code otherLatitude} */ + public static double nextLatitudeNear(double otherLatitude) { + GeoUtils.checkLatitude(otherLatitude); + return normalizeLatitude(otherLatitude + random().nextDouble() - 0.5); + } + + /** returns next pseudorandom longitude, kinda close to {@code otherLongitude} */ + public static double nextLongitudeNear(double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + return normalizeLongitude(otherLongitude + random().nextDouble() - 0.5); + } + + /** + * returns next pseudorandom latitude, kinda close to {@code minLatitude/maxLatitude} + * NOTE:minLatitude/maxLatitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLatitudeAround(double minLatitude, double maxLatitude) { + GeoUtils.checkLatitude(minLatitude); + GeoUtils.checkLatitude(maxLatitude); + return normalizeLatitude(randomRangeMaybeSlightlyOutside(minLatitude, maxLatitude)); + } + + /** + * returns next pseudorandom longitude, kinda close to {@code minLongitude/maxLongitude} + * NOTE:minLongitude/maxLongitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLongitudeAround(double minLongitude, double maxLongitude) { + GeoUtils.checkLongitude(minLongitude); + GeoUtils.checkLongitude(maxLongitude); + return normalizeLongitude(randomRangeMaybeSlightlyOutside(minLongitude, maxLongitude)); + } + + /** Returns random double min to max or up to 1% outside of that range */ + private static double randomRangeMaybeSlightlyOutside(double min, double max) { + return min + (random().nextDouble() + (0.5 - random().nextDouble()) * .02) * (max - min); + } + + /** Puts latitude in range of -90 to 90. */ + private static double normalizeLatitude(double latitude) { + if (latitude >= -90 && latitude <= 90) { + return latitude; //common case, and avoids slight double precision shifting + } + double off = Math.abs((latitude + 90) % 360); + return (off <= 180 ? off : 360-off) - 90; + } + + /** Puts longitude in range of -180 to +180. */ + private static double normalizeLongitude(double longitude) { + if (longitude >= -180 && longitude <= 180) { + return longitude; //common case, and avoids slight double precision shifting + } + double off = (longitude + 180) % 360; + if (off < 0) { + return 180 + off; + } else if (off == 0 && longitude > 0) { + return 180; + } else { + return -180 + off; + } + } + + /** Keep it simple, we don't need to take arbitrary Random for geo tests */ + private static Random random() { + return RandomizedContext.current().getRandom(); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 15fa29c0915e..10baf42ed392 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -34,24 +34,11 @@ public class TestGeoUtils extends LuceneTestCase { // Global bounding box we will "cover" in the random test; we have to make this "smallish" else the queries take very long: private static double originLat; private static double originLon; - // private static double range; - private static double lonRange; - private static double latRange; @BeforeClass public static void beforeClass() throws Exception { - // Between 1.0 and 3.0: - lonRange = 2 * (random().nextDouble() + 0.5); - latRange = 2 * (random().nextDouble() + 0.5); - - originLon = GeoUtils.MIN_LON_INCL + lonRange + (GeoUtils.MAX_LON_INCL - GeoUtils.MIN_LON_INCL - 2 * lonRange) * random().nextDouble(); - originLon = BaseGeoPointTestCase.normalizeLon(originLon); - originLat = GeoUtils.MIN_LAT_INCL + latRange + (GeoUtils.MAX_LAT_INCL - GeoUtils.MIN_LAT_INCL - 2 * latRange) * random().nextDouble(); - originLat = BaseGeoPointTestCase.normalizeLat(originLat); - - if (VERBOSE) { - System.out.println("TEST: originLon=" + originLon + " lonRange= " + lonRange + " originLat=" + originLat + " latRange=" + latRange); - } + originLon = GeoTestUtil.nextLongitude(); + originLat = GeoTestUtil.nextLatitude(); } public long scaleLon(final double val) { @@ -73,9 +60,9 @@ public double unscaleLat(final long val) { public double randomLat(boolean small) { double result; if (small) { - result = BaseGeoPointTestCase.normalizeLat(originLat + latRange * (random().nextDouble() - 0.5)); + result = GeoTestUtil.nextLatitudeNear(originLat); } else { - result = -90 + 180.0 * random().nextDouble(); + result = GeoTestUtil.nextLatitude(); } return result; } @@ -83,9 +70,9 @@ public double randomLat(boolean small) { public double randomLon(boolean small) { double result; if (small) { - result = BaseGeoPointTestCase.normalizeLon(originLon + lonRange * (random().nextDouble() - 0.5)); + result = GeoTestUtil.nextLongitudeNear(originLon); } else { - result = -180 + 360.0 * random().nextDouble(); + result = GeoTestUtil.nextLongitude(); } return result; } @@ -149,11 +136,6 @@ public void testScaleUnscaleIsStable() throws Exception { } } - /** Returns random double min to max or up to 1% outside of that range */ - private double randomRangeMaybeSlightlyOutside(double min, double max) { - return min + (random().nextDouble() + (0.5 - random().nextDouble()) * .02) * (max - min); - } - // We rely heavily on GeoUtils.circleToBBox so we test it here: public void testRandomCircleToBBox() throws Exception { int iters = atLeast(1000); @@ -188,15 +170,15 @@ public void testRandomCircleToBBox() throws Exception { lon = randomLon(useSmallRanges); } else { // pick a lat/lon within the bbox or "slightly" outside it to try to improve test efficiency - lat = BaseGeoPointTestCase.normalizeLat(randomRangeMaybeSlightlyOutside(bbox.minLat, bbox.maxLat)); + lat = GeoTestUtil.nextLatitudeAround(bbox.minLat, bbox.maxLat); if (bbox.crossesDateline()) { if (random().nextBoolean()) { - lon = BaseGeoPointTestCase.normalizeLon(randomRangeMaybeSlightlyOutside(bbox.maxLon, -180)); + lon = GeoTestUtil.nextLongitudeAround(bbox.maxLon, -180); } else { - lon = BaseGeoPointTestCase.normalizeLon(randomRangeMaybeSlightlyOutside(0, bbox.minLon)); + lon = GeoTestUtil.nextLongitudeAround(0, bbox.minLon); } } else { - lon = BaseGeoPointTestCase.normalizeLon(randomRangeMaybeSlightlyOutside(bbox.minLon, bbox.maxLon)); + lon = GeoTestUtil.nextLongitudeAround(bbox.minLon, bbox.maxLon); } } @@ -235,8 +217,8 @@ public void testRandomCircleToBBox() throws Exception { // similar to testRandomCircleToBBox, but different, less evil, maybe simpler public void testBoundingBoxOpto() { for (int i = 0; i < 1000; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double radius = 50000000 * random().nextDouble(); GeoRect box = GeoUtils.circleToBBox(lat, lon, radius); final GeoRect box1; @@ -250,8 +232,8 @@ public void testBoundingBoxOpto() { } for (int j = 0; j < 10000; j++) { - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); // if the point is within radius, then it should be in our bounding box if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { assertTrue(lat >= box.minLat && lat <= box.maxLat); @@ -264,8 +246,8 @@ public void testBoundingBoxOpto() { // test we can use haversinSortKey() for distance queries. public void testHaversinOpto() { for (int i = 0; i < 1000; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double radius = 50000000 * random().nextDouble(); GeoRect box = GeoUtils.circleToBBox(lat, lon, radius); @@ -274,8 +256,8 @@ public void testHaversinOpto() { SloppyMath.haversinSortKey(lat, lon, box.maxLat, lon)); for (int j = 0; j < 10000; j++) { - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); // if the point is within radius, then it should be <= our sort key if (SloppyMath.haversinMeters(lat, lon, lat2, lon2) <= radius) { assertTrue(SloppyMath.haversinSortKey(lat, lon, lat2, lon2) <= minPartialDistance); @@ -288,8 +270,8 @@ public void testHaversinOpto() { /** Test infinite radius covers whole earth */ public void testInfiniteRect() { for (int i = 0; i < 1000; i++) { - double centerLat = -90 + 180.0 * random().nextDouble(); - double centerLon = -180 + 360.0 * random().nextDouble(); + double centerLat = GeoTestUtil.nextLatitude(); + double centerLon = GeoTestUtil.nextLongitude(); GeoRect rect = GeoUtils.circleToBBox(centerLat, centerLon, Double.POSITIVE_INFINITY); assertEquals(-180.0, rect.minLon, 0.0D); assertEquals(180.0, rect.maxLon, 0.0D); From 9cccffad41858998ad9e9d0ef926c75d59438bb9 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 28 Mar 2016 11:08:53 -0400 Subject: [PATCH 0231/1113] simplify base geo test class --- .../spatial/util/BaseGeoPointTestCase.java | 468 ++++++++++++------ 1 file changed, 327 insertions(+), 141 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index ce908e70ef1d..c0633ebb2c15 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -45,6 +45,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReadState; @@ -57,6 +58,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; @@ -432,10 +434,9 @@ public void testMultiValued() throws Exception { IndexReader r = w.getReader(); w.close(); - // We can't wrap with "exotic" readers because the BKD query must see the BKDDVFormat: - IndexSearcher s = newSearcher(r, false); + IndexSearcher s = newSearcher(r); - int iters = atLeast(75); + int iters = atLeast(25); for (int iter=0;iter 0 && random().nextInt(100) == 42) { + int idToDelete = random().nextInt(id); + w.deleteDocuments(new Term("id", ""+idToDelete)); + deleted.add(idToDelete); + if (VERBOSE) { + System.out.println(" delete id=" + idToDelete); + } + } + } + + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); + + IndexSearcher s = newSearcher(r); + + int iters = atLeast(25); + + NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); + + Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader()); + int maxDoc = s.getIndexReader().maxDoc(); + + for (int iter=0;iter 100000) { + dir = newFSDirectory(createTempDir(getClass().getSimpleName())); + } else { + dir = newDirectory(); + } + + Set deleted = new HashSet<>(); + // RandomIndexWriter is too slow here: + IndexWriter w = new IndexWriter(dir, iwc); + for(int id=0;id 0 && random().nextInt(100) == 42) { + int idToDelete = random().nextInt(id); + w.deleteDocuments(new Term("id", ""+idToDelete)); + deleted.add(idToDelete); if (VERBOSE) { - final DecimalFormat df = new DecimalFormat("#,###.00", DecimalFormatSymbols.getInstance(Locale.ENGLISH)); - System.out.println(" radiusMeters = " + df.format(radiusMeters)); + System.out.println(" delete id=" + idToDelete); } + } + } - query = newDistanceQuery(FIELD_NAME, centerLat, centerLon, radiusMeters); + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); - verifyHits = new VerifyHits() { - @Override - protected boolean shouldMatch(double pointLat, double pointLon) { - return circleContainsPoint(centerLat, centerLon, radiusMeters, pointLat, pointLon); - } + // nocommit can we wrap again? + // We can't wrap with "exotic" readers because points needs to work: + IndexSearcher s = newSearcher(r, false); - @Override - protected void describe(int docID, double pointLat, double pointLon) { - double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, pointLat, pointLon); - System.out.println(" docID=" + docID + " centerLat=" + centerLat + " centerLon=" + centerLon - + " pointLat=" + pointLat + " pointLon=" + pointLon + " distanceMeters=" + distanceMeters - + " vs radiusMeters=" + radiusMeters); - } - }; + final int iters = atLeast(75); - // TODO: get poly query working with dateline crossing too (how?)! - } else { + NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); - // TODO: poly query can't handle dateline crossing yet: - final GeoRect bbox = randomRect(small, false); - - // Polygon - final double[] polyLats; - final double[] polyLons; - // TODO: factor this out, maybe if we add Polygon class? - switch (random().nextInt(3)) { - case 0: - // box - polyLats = new double[5]; - polyLons = new double[5]; - polyLats[0] = bbox.minLat; - polyLons[0] = bbox.minLon; - polyLats[1] = bbox.maxLat; - polyLons[1] = bbox.minLon; - polyLats[2] = bbox.maxLat; - polyLons[2] = bbox.maxLon; - polyLats[3] = bbox.minLat; - polyLons[3] = bbox.maxLon; - polyLats[4] = bbox.minLat; - polyLons[4] = bbox.minLon; - break; - case 1: - // right triangle - polyLats = new double[4]; - polyLons = new double[4]; - polyLats[0] = bbox.minLat; - polyLons[0] = bbox.minLon; - polyLats[1] = bbox.maxLat; - polyLons[1] = bbox.minLon; - polyLats[2] = bbox.maxLat; - polyLons[2] = bbox.maxLon; - polyLats[3] = bbox.minLat; - polyLons[3] = bbox.minLon; - break; - default: - // surprise me! - double[][] res = surpriseMePolygon(); - polyLats = res[0]; - polyLons = res[1]; - break; - } - query = newPolygonQuery(FIELD_NAME, polyLats, polyLons); + Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader()); + int maxDoc = s.getIndexReader().maxDoc(); + + for (int iter=0;iter Date: Mon, 28 Mar 2016 11:48:14 -0400 Subject: [PATCH 0232/1113] factor out polygon/box generation into GeoTestUtil. removes lots of leniency and unnecessary quantization! may cause failures!!!! --- .../spatial/util/BaseGeoPointTestCase.java | 165 +++-------------- .../lucene/spatial/util/GeoTestUtil.java | 167 +++++++++++++++++- 2 files changed, 192 insertions(+), 140 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index c0633ebb2c15..e25266a167ab 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -19,13 +19,11 @@ import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; -import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashSet; import java.util.Locale; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.FilterCodec; @@ -438,7 +436,7 @@ public void testMultiValued() throws Exception { int iters = atLeast(25); for (int iter=0;iter lats = new ArrayList<>(); - ArrayList lons = new ArrayList<>(); - double angle = 0.0; - while (true) { - angle += random().nextDouble()*40.0; - //System.out.println(" angle " + angle); - if (angle > 360) { - break; - } - double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); - //System.out.println(" len=" + len); - double lat = centerLat + len * Math.cos(Math.toRadians(angle)); - double lon = centerLon + len * Math.sin(Math.toRadians(angle)); - if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { - // cannot cross dateline: try again! - continue newPoly; - } - if (lat > 90) { - // cross the north pole - lat = 180 - lat; - lon = 180 - lon; - } else if (lat < -90) { - // cross the south pole - lat = -180 - lat; - lon = 180 - lon; - } - if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { - // cannot cross dateline: try again! - continue newPoly; - } - lats.add(lat); - lons.add(lon); - - //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); - } - - // close it - lats.add(lats.get(0)); - lons.add(lons.get(0)); - double[] latsArray = new double[lats.size()]; - double[] lonsArray = new double[lons.size()]; - for(int i=0;i lats = new ArrayList<>(); + ArrayList lons = new ArrayList<>(); + double angle = 0.0; + while (true) { + angle += random().nextDouble()*40.0; + //System.out.println(" angle " + angle); + if (angle > 360) { + break; + } + double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); + //System.out.println(" len=" + len); + double lat = centerLat + len * Math.cos(Math.toRadians(angle)); + double lon = centerLon + len * Math.sin(Math.toRadians(angle)); + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + if (lat > 90) { + // cross the north pole + lat = 180 - lat; + lon = 180 - lon; + } else if (lat < -90) { + // cross the south pole + lat = -180 - lat; + lon = 180 - lon; + } + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + lats.add(lat); + lons.add(lon); + + //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); + } + + // close it + lats.add(lats.get(0)); + lons.add(lons.get(0)); + + double[] latsArray = new double[lats.size()]; + double[] lonsArray = new double[lons.size()]; + for(int i=0;i Date: Mon, 28 Mar 2016 12:00:53 -0400 Subject: [PATCH 0233/1113] minor test cleanups --- .../document/TestLatLonPointDistanceSort.java | 9 +++++---- .../lucene/spatial/util/TestGeoUtils.java | 19 ------------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 0e5d898e41b9..cd38b0ebd9fd 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.spatial.util.GeoTestUtil; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -182,8 +183,8 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { doc.add(new StoredField("id", i)); doc.add(new NumericDocValuesField("id", i)); if (random().nextInt(10) > 7) { - double latRaw = -90 + 180.0 * random().nextDouble(); - double lonRaw = -180 + 360.0 * random().nextDouble(); + double latRaw = GeoTestUtil.nextLatitude(); + double lonRaw = GeoTestUtil.nextLongitude(); // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); @@ -198,8 +199,8 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { - double lat = -90 + 180.0 * random().nextDouble(); - double lon = -180 + 360.0 * random().nextDouble(); + double lat = GeoTestUtil.nextLatitude(); + double lon = GeoTestUtil.nextLongitude(); double missingValue = Double.POSITIVE_INFINITY; Result expected[] = new Result[reader.maxDoc()]; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 10baf42ed392..98867c65eb6d 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -28,9 +28,6 @@ */ public class TestGeoUtils extends LuceneTestCase { - private static final double LON_SCALE = (0x1L< Date: Mon, 28 Mar 2016 14:13:44 -0400 Subject: [PATCH 0234/1113] fix nocommit --- .../org/apache/lucene/spatial/util/BaseGeoPointTestCase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index e25266a167ab..8b712362d537 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -969,9 +969,8 @@ protected void verifyRandomPolygons(boolean small, double[] lats, double[] lons) final IndexReader r = DirectoryReader.open(w); w.close(); - // nocommit can we wrap again? // We can't wrap with "exotic" readers because points needs to work: - IndexSearcher s = newSearcher(r, false); + IndexSearcher s = newSearcher(r); final int iters = atLeast(75); From 70290d9c8e114ba8cf5d083d3c3c51db9bb87ce8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 28 Mar 2016 14:12:49 -0400 Subject: [PATCH 0235/1113] LUCENE-7145: consolidate polygon range checks, tests, box usage. make cannot -> must not consistent so we can test this stuff. --- .../DictionaryCompoundWordTokenFilter.java | 4 +- .../payloads/NumericPayloadTokenFilter.java | 2 +- .../apache/lucene/document/BinaryPoint.java | 16 ++-- .../apache/lucene/document/DoublePoint.java | 4 +- .../org/apache/lucene/document/Field.java | 26 +++---- .../org/apache/lucene/document/FieldType.java | 4 +- .../apache/lucene/document/FloatPoint.java | 4 +- .../org/apache/lucene/document/IntPoint.java | 4 +- .../org/apache/lucene/document/LongPoint.java | 4 +- .../lucene/index/DefaultIndexingChain.java | 2 +- .../lucene/index/DocValuesFieldUpdates.java | 2 +- .../org/apache/lucene/index/FieldInfo.java | 8 +- .../org/apache/lucene/index/FieldInfos.java | 2 +- .../apache/lucene/index/FilterLeafReader.java | 10 +-- .../org/apache/lucene/index/IndexWriter.java | 2 +- .../lucene/index/IndexWriterConfig.java | 6 +- .../lucene/index/PointValuesWriter.java | 2 +- .../org/apache/lucene/search/PrefixQuery.java | 2 +- .../lucene/search/ReferenceManager.java | 4 +- .../apache/lucene/store/BaseDirectory.java | 2 +- .../apache/lucene/util/CommandLineUtil.java | 2 +- .../org/apache/lucene/util/UnicodeUtil.java | 2 +- .../org/apache/lucene/facet/range/Range.java | 2 +- .../directory/DirectoryTaxonomyReader.java | 2 +- .../PostingsHighlighter.java | 4 +- .../apache/lucene/store/WindowsDirectory.cpp | 4 +- .../queryparser/classic/QueryParserBase.java | 4 +- .../core/config/AbstractQueryConfig.java | 8 +- .../flexible/core/config/FieldConfig.java | 4 +- .../standard/config/LegacyNumericConfig.java | 6 +- .../LegacyNumericFieldConfigListener.java | 2 +- .../standard/config/PointsConfig.java | 6 +- .../standard/config/PointsConfigListener.java | 2 +- .../nodes/LegacyNumericRangeQueryNode.java | 2 +- .../standard/nodes/PointRangeQueryNode.java | 2 +- .../lucene/replicator/RevisionFile.java | 2 +- .../lucene/document/BigIntegerPoint.java | 4 +- .../lucene/document/InetAddressPoint.java | 4 +- .../apache/lucene/document/LatLonPoint.java | 8 +- .../document/LatLonPointDistanceQuery.java | 2 +- .../document/LatLonPointInPolygonQuery.java | 49 +++--------- .../lucene/document/LatLonPointSortField.java | 2 +- .../lucene/search/DocValuesRangeQuery.java | 2 +- .../geopoint/search/GeoPointInBBoxQuery.java | 4 +- .../search/GeoPointInPolygonQuery.java | 18 +---- .../apache/lucene/spatial/util/GeoUtils.java | 42 +++++++--- .../spatial/util/BaseGeoPointTestCase.java | 78 ++++++++++++++++++- .../lucene/spatial/util/TestGeoUtils.java | 36 +++++++++ 48 files changed, 253 insertions(+), 160 deletions(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java index 3dfe3ab80fb0..b81a96cf0407 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java @@ -41,7 +41,7 @@ public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBa public DictionaryCompoundWordTokenFilter(TokenStream input, CharArraySet dictionary) { super(input, dictionary); if (dictionary == null) { - throw new IllegalArgumentException("dictionary cannot be null"); + throw new IllegalArgumentException("dictionary must not be null"); } } @@ -65,7 +65,7 @@ public DictionaryCompoundWordTokenFilter(TokenStream input, CharArraySet diction int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) { super(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch); if (dictionary == null) { - throw new IllegalArgumentException("dictionary cannot be null"); + throw new IllegalArgumentException("dictionary must not be null"); } } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java index 9f9eec62fdef..e5a4a4588fe2 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java @@ -41,7 +41,7 @@ public class NumericPayloadTokenFilter extends TokenFilter { public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) { super(input); if (typeMatch == null) { - throw new IllegalArgumentException("typeMatch cannot be null"); + throw new IllegalArgumentException("typeMatch must not be null"); } //Need to encode the payload thePayload = new BytesRef(PayloadHelper.encodeFloat(payload)); diff --git a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java index 60fd9d636e64..7132a5d8a3bf 100644 --- a/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/BinaryPoint.java @@ -47,19 +47,19 @@ public final class BinaryPoint extends Field { private static FieldType getType(byte[][] point) { if (point == null) { - throw new IllegalArgumentException("point cannot be null"); + throw new IllegalArgumentException("point must not be null"); } if (point.length == 0) { - throw new IllegalArgumentException("point cannot be 0 dimensions"); + throw new IllegalArgumentException("point must not be 0 dimensions"); } int bytesPerDim = -1; for(int i=0;i attributes, int pointDimensionCount, int pointNumBytes) { this.name = Objects.requireNonNull(name); this.number = number; - this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType cannot be null (field: \"" + name + "\")"); - this.indexOptions = Objects.requireNonNull(indexOptions, "IndexOptions cannot be null (field: \"" + name + "\")"); + this.docValuesType = Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")"); + this.indexOptions = Objects.requireNonNull(indexOptions, "IndexOptions must not be null (field: \"" + name + "\")"); if (indexOptions != IndexOptions.NONE) { this.storeTermVector = storeTermVector; this.storePayloads = storePayloads; @@ -130,7 +130,7 @@ public boolean checkConsistency() { void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, int dimensionCount, int dimensionNumBytes) { if (indexOptions == null) { - throw new NullPointerException("IndexOptions cannot be null (field: \"" + name + "\")"); + throw new NullPointerException("IndexOptions must not be null (field: \"" + name + "\")"); } //System.out.println("FI.update field=" + name + " indexed=" + indexed + " omitNorms=" + omitNorms + " this.omitNorms=" + this.omitNorms); if (this.indexOptions != indexOptions) { @@ -201,7 +201,7 @@ public int getPointNumBytes() { void setDocValuesType(DocValuesType type) { if (type == null) { - throw new NullPointerException("DocValuesType cannot be null (field: \"" + name + "\")"); + throw new NullPointerException("DocValuesType must not be null (field: \"" + name + "\")"); } if (docValuesType != DocValuesType.NONE && type != DocValuesType.NONE && docValuesType != type) { throw new IllegalArgumentException("cannot change DocValues type from " + docValuesType + " to " + type + " for field \"" + name + "\""); diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java index a49a9931dd15..c80fb854b49b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java @@ -394,7 +394,7 @@ private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, int dimensionCount, int dimensionNumBytes) { if (docValues == null) { - throw new NullPointerException("DocValuesType cannot be null"); + throw new NullPointerException("DocValuesType must not be null"); } FieldInfo fi = fieldInfo(name); if (fi == null) { diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java index 1d593c32a127..f273dba6fc99 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java @@ -67,7 +67,7 @@ public abstract static class FilterFields extends Fields { */ public FilterFields(Fields in) { if (in == null) { - throw new NullPointerException("incoming Fields cannot be null"); + throw new NullPointerException("incoming Fields must not be null"); } this.in = in; } @@ -103,7 +103,7 @@ public abstract static class FilterTerms extends Terms { */ public FilterTerms(Terms in) { if (in == null) { - throw new NullPointerException("incoming Terms cannot be null"); + throw new NullPointerException("incoming Terms must not be null"); } this.in = in; } @@ -170,7 +170,7 @@ public abstract static class FilterTermsEnum extends TermsEnum { */ public FilterTermsEnum(TermsEnum in) { if (in == null) { - throw new NullPointerException("incoming TermsEnum cannot be null"); + throw new NullPointerException("incoming TermsEnum must not be null"); } this.in = in; } @@ -233,7 +233,7 @@ public abstract static class FilterPostingsEnum extends PostingsEnum { */ public FilterPostingsEnum(PostingsEnum in) { if (in == null) { - throw new NullPointerException("incoming PostingsEnum cannot be null"); + throw new NullPointerException("incoming PostingsEnum must not be null"); } this.in = in; } @@ -300,7 +300,7 @@ public long cost() { public FilterLeafReader(LeafReader in) { super(); if (in == null) { - throw new NullPointerException("incoming LeafReader cannot be null"); + throw new NullPointerException("incoming LeafReader must not be null"); } this.in = in; in.registerParentReader(this); diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 66a0e73b601b..2b45b6b38abd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -1592,7 +1592,7 @@ public void updateDocValues(Term term, Field... updates) throws IOException { final Field f = updates[i]; final DocValuesType dvType = f.fieldType().docValuesType(); if (dvType == null) { - throw new NullPointerException("DocValuesType cannot be null (field: \"" + f.name() + "\")"); + throw new NullPointerException("DocValuesType must not be null (field: \"" + f.name() + "\")"); } if (dvType == DocValuesType.NONE) { throw new IllegalArgumentException("can only update NUMERIC or BINARY fields! field=" + f.name()); diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java index 4a7cf7158b71..a90d625e3050 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -160,7 +160,7 @@ public OpenMode getOpenMode() { * like NFS that do not support "delete on last close" semantics, which * Lucene's "point in time" search normally relies on. *

    - * NOTE: the deletion policy cannot be null. + * NOTE: the deletion policy must not be null. * *

    Only takes effect when IndexWriter is first created. */ @@ -197,7 +197,7 @@ public IndexCommit getIndexCommit() { /** * Expert: set the {@link Similarity} implementation used by this IndexWriter. *

    - * NOTE: the similarity cannot be null. + * NOTE: the similarity must not be null. * *

    Only takes effect when IndexWriter is first created. */ public IndexWriterConfig setSimilarity(Similarity similarity) { @@ -217,7 +217,7 @@ public Similarity getSimilarity() { * Expert: sets the merge scheduler used by this writer. The default is * {@link ConcurrentMergeScheduler}. *

    - * NOTE: the merge scheduler cannot be null. + * NOTE: the merge scheduler must not be null. * *

    Only takes effect when IndexWriter is first created. */ public IndexWriterConfig setMergeScheduler(MergeScheduler mergeScheduler) { diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java index f259c5b99626..e3d1c1d8659e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java @@ -48,7 +48,7 @@ public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo // TODO: if exactly the same value is added to exactly the same doc, should we dedup? public void addPackedValue(int docID, BytesRef value) { if (value == null) { - throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value cannot be null"); + throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value must not be null"); } if (value.length != fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes())); diff --git a/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java b/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java index 207a1434488d..f4b5e5026cf6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PrefixQuery.java @@ -35,7 +35,7 @@ public PrefixQuery(Term prefix) { // It's OK to pass unlimited maxDeterminizedStates: the automaton is born small and determinized: super(prefix, toAutomaton(prefix.bytes()), Integer.MAX_VALUE, true); if (prefix == null) { - throw new NullPointerException("prefix cannot be null"); + throw new NullPointerException("prefix must not be null"); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java b/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java index 6ecf81bd9444..c3487ad0fffd 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java @@ -291,7 +291,7 @@ private void notifyRefreshListenersRefreshed(boolean didRefresh) throws IOExcept */ public void addListener(RefreshListener listener) { if (listener == null) { - throw new NullPointerException("Listener cannot be null"); + throw new NullPointerException("Listener must not be null"); } refreshListeners.add(listener); } @@ -301,7 +301,7 @@ public void addListener(RefreshListener listener) { */ public void removeListener(RefreshListener listener) { if (listener == null) { - throw new NullPointerException("Listener cannot be null"); + throw new NullPointerException("Listener must not be null"); } refreshListeners.remove(listener); } diff --git a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java index a5fff241b491..709eb2cc2e18 100644 --- a/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/BaseDirectory.java @@ -35,7 +35,7 @@ public abstract class BaseDirectory extends Directory { protected BaseDirectory(LockFactory lockFactory) { super(); if (lockFactory == null) { - throw new NullPointerException("LockFactory cannot be null, use an explicit instance!"); + throw new NullPointerException("LockFactory must not be null, use an explicit instance!"); } this.lockFactory = lockFactory; } diff --git a/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java b/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java index c4b22776c92b..952e2c7787d1 100644 --- a/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java @@ -96,7 +96,7 @@ public static Class loadFSDirectoryClass(String clazzName private static String adjustDirectoryClassName(String clazzName) { if (clazzName == null || clazzName.trim().length() == 0) { throw new IllegalArgumentException("The " + FSDirectory.class.getSimpleName() - + " implementation cannot be null or empty"); + + " implementation must not be null or empty"); } if (clazzName.indexOf(".") == -1) {// if not fully qualified, assume .store diff --git a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java index 1a8022d8d99e..a21281f8a0a4 100644 --- a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java +++ b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java @@ -449,7 +449,7 @@ public static int codePointCount(BytesRef utf8) { * content is prematurely truncated. */ public static int UTF8toUTF32(final BytesRef utf8, final int[] ints) { - // TODO: ints cannot be null, should be an assert + // TODO: ints must not be null, should be an assert int utf32Count = 0; int utf8Upto = utf8.offset; final byte[] bytes = utf8.bytes; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java b/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java index 227383165fb0..5f6de98f1a88 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java @@ -31,7 +31,7 @@ public abstract class Range { /** Sole constructor. */ protected Range(String label) { if (label == null) { - throw new NullPointerException("label cannot be null"); + throw new NullPointerException("label must not be null"); } this.label = label; } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java index dee967e7b9ad..c72e60bbdc7a 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java @@ -178,7 +178,7 @@ protected DirectoryTaxonomyReader doOpenIfChanged() throws IOException { recreated = true; } } else if (!t1.equals(t2)) { - // t1 != null and t2 cannot be null b/c DirTaxoWriter always puts the commit data. + // t1 != null and t2 must not be null b/c DirTaxoWriter always puts the commit data. // it's ok to use String.equals because we require the two epoch values to be the same. recreated = true; } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java index 3c0a42814f07..e4d366793aa3 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java @@ -465,7 +465,7 @@ private Map highlightField(String field, String contents[], Brea PassageFormatter fieldFormatter = getFormatter(field); if (fieldFormatter == null) { - throw new NullPointerException("PassageFormatter cannot be null"); + throw new NullPointerException("PassageFormatter must not be null"); } // check if we should do any multiterm processing @@ -550,7 +550,7 @@ private Passage[] highlightDoc(String field, BytesRef terms[], int contentLength TermsEnum termsEnum, PostingsEnum[] postings, int n) throws IOException { PassageScorer scorer = getScorer(field); if (scorer == null) { - throw new NullPointerException("PassageScorer cannot be null"); + throw new NullPointerException("PassageScorer must not be null"); } PriorityQueue pq = new PriorityQueue<>(); float weights[] = new float[terms.length]; diff --git a/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp b/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp index e6063b936f8f..416aed90fb87 100644 --- a/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp +++ b/lucene/misc/src/java/org/apache/lucene/store/WindowsDirectory.cpp @@ -67,7 +67,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open HANDLE handle; if (filename == NULL) { - throwException(env, "java/lang/NullPointerException", "filename cannot be null"); + throwException(env, "java/lang/NullPointerException", "filename must not be null"); return -1; } @@ -109,7 +109,7 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read io.OffsetHigh = (DWORD) ((pos >> 0x20) & 0x7FFFFFFF); if (bytes == NULL) { - throwException(env, "java/lang/NullPointerException", "bytes cannot be null"); + throwException(env, "java/lang/NullPointerException", "bytes must not be null"); return -1; } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java index d0a371c03582..c00d88eecffc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java @@ -342,7 +342,7 @@ public void setDateResolution(DateTools.Resolution dateResolution) { */ public void setDateResolution(String fieldName, DateTools.Resolution dateResolution) { if (fieldName == null) { - throw new IllegalArgumentException("Field cannot be null."); + throw new IllegalArgumentException("Field must not be null."); } if (fieldToDateResolution == null) { @@ -361,7 +361,7 @@ public void setDateResolution(String fieldName, DateTools.Resolution dateResolut */ public DateTools.Resolution getDateResolution(String fieldName) { if (fieldName == null) { - throw new IllegalArgumentException("Field cannot be null."); + throw new IllegalArgumentException("Field must not be null."); } if (fieldToDateResolution == null) { diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java index e60f931c7bb8..bfbde4d4fc12 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/AbstractQueryConfig.java @@ -51,7 +51,7 @@ public abstract class AbstractQueryConfig { public T get(ConfigurationKey key) { if (key == null) { - throw new IllegalArgumentException("key cannot be null!"); + throw new IllegalArgumentException("key must not be null!"); } return (T) this.configMap.get(key); @@ -68,7 +68,7 @@ public T get(ConfigurationKey key) { public boolean has(ConfigurationKey key) { if (key == null) { - throw new IllegalArgumentException("key cannot be null!"); + throw new IllegalArgumentException("key must not be null!"); } return this.configMap.containsKey(key); @@ -85,7 +85,7 @@ public boolean has(ConfigurationKey key) { public void set(ConfigurationKey key, T value) { if (key == null) { - throw new IllegalArgumentException("key cannot be null!"); + throw new IllegalArgumentException("key must not be null!"); } if (value == null) { @@ -107,7 +107,7 @@ public void set(ConfigurationKey key, T value) { public boolean unset(ConfigurationKey key) { if (key == null) { - throw new IllegalArgumentException("key cannot be null!"); + throw new IllegalArgumentException("key must not be null!"); } return this.configMap.remove(key) != null; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java index c8604d1cca38..fae6f9fc2ec9 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/config/FieldConfig.java @@ -26,13 +26,13 @@ public class FieldConfig extends AbstractQueryConfig { /** * Constructs a {@link FieldConfig} * - * @param fieldName the field name, it cannot be null + * @param fieldName the field name, it must not be null * @throws IllegalArgumentException if the field name is null */ public FieldConfig(String fieldName) { if (fieldName == null) { - throw new IllegalArgumentException("field name should not be null!"); + throw new IllegalArgumentException("field name must not be null!"); } this.fieldName = fieldName; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java index 985f55a616fd..6cd3c490e578 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericConfig.java @@ -113,7 +113,7 @@ public LegacyNumericType getType() { public void setType(LegacyNumericType type) { if (type == null) { - throw new IllegalArgumentException("type cannot be null!"); + throw new IllegalArgumentException("type must not be null!"); } this.type = type; @@ -126,12 +126,12 @@ public void setType(LegacyNumericType type) { * * @param format * the {@link NumberFormat} used to parse a {@link String} to - * {@link Number}, cannot be null + * {@link Number}, must not be null */ public void setNumberFormat(NumberFormat format) { if (format == null) { - throw new IllegalArgumentException("format cannot be null!"); + throw new IllegalArgumentException("format must not be null!"); } this.format = format; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java index f2d312489f61..e5ba9daa9b53 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/LegacyNumericFieldConfigListener.java @@ -48,7 +48,7 @@ public class LegacyNumericFieldConfigListener implements FieldConfigListener { public LegacyNumericFieldConfigListener(QueryConfigHandler config) { if (config == null) { - throw new IllegalArgumentException("config cannot be null!"); + throw new IllegalArgumentException("config must not be null!"); } this.config = config; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java index db59b48123a4..41239826da21 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfig.java @@ -76,7 +76,7 @@ public Class getType() { */ public void setType(Class type) { if (type == null) { - throw new IllegalArgumentException("type cannot be null!"); + throw new IllegalArgumentException("type must not be null!"); } if (Integer.class.equals(type) == false && Long.class.equals(type) == false && @@ -93,11 +93,11 @@ public void setType(Class type) { * * @param format * the {@link NumberFormat} used to parse a {@link String} to - * {@link Number}, cannot be null + * {@link Number}, must not be null */ public void setNumberFormat(NumberFormat format) { if (format == null) { - throw new IllegalArgumentException("format cannot be null!"); + throw new IllegalArgumentException("format must not be null!"); } this.format = format; } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java index 9efbbb7e5a01..e7b5185229fb 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/config/PointsConfigListener.java @@ -45,7 +45,7 @@ public class PointsConfigListener implements FieldConfigListener { */ public PointsConfigListener(QueryConfigHandler config) { if (config == null) { - throw new IllegalArgumentException("config cannot be null!"); + throw new IllegalArgumentException("config must not be null!"); } this.config = config; } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java index 088ab98e175a..27c285eb34cc 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/LegacyNumericRangeQueryNode.java @@ -88,7 +88,7 @@ public void setBounds(LegacyNumericQueryNode lower, LegacyNumericQueryNode upper boolean lowerInclusive, boolean upperInclusive, LegacyNumericConfig numericConfig) throws QueryNodeException { if (numericConfig == null) { - throw new IllegalArgumentException("numericConfig cannot be null!"); + throw new IllegalArgumentException("numericConfig must not be null!"); } LegacyNumericType lowerNumberType, upperNumberType; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java index cb838fc7d6cc..3611ebfe979c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/PointRangeQueryNode.java @@ -62,7 +62,7 @@ public void setBounds(PointQueryNode lower, PointQueryNode upper, boolean lowerInclusive, boolean upperInclusive, PointsConfig pointsConfig) throws QueryNodeException { if (pointsConfig == null) { - throw new IllegalArgumentException("pointsConfig cannot be null!"); + throw new IllegalArgumentException("pointsConfig must not be null!"); } Class lowerNumberType, upperNumberType; diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java b/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java index 7d3d4b7660cf..2ca52057a4d1 100644 --- a/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java +++ b/lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java @@ -34,7 +34,7 @@ public class RevisionFile { /** Constructor with the given file name. */ public RevisionFile(String fileName) { if (fileName == null || fileName.isEmpty()) { - throw new IllegalArgumentException("fileName cannot be null or empty"); + throw new IllegalArgumentException("fileName must not be null or empty"); } this.fileName = fileName; } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java index efbb37c0c867..4dc50d110f28 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/BigIntegerPoint.java @@ -85,10 +85,10 @@ public Number numericValue() { private static BytesRef pack(BigInteger... point) { if (point == null) { - throw new IllegalArgumentException("point cannot be null"); + throw new IllegalArgumentException("point must not be null"); } if (point.length == 0) { - throw new IllegalArgumentException("point cannot be 0 dimensions"); + throw new IllegalArgumentException("point must not be 0 dimensions"); } byte[] packed = new byte[point.length * BYTES]; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java index c0739a40e9ea..7ebabab02eb0 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/InetAddressPoint.java @@ -68,7 +68,7 @@ public class InetAddressPoint extends Field { /** Change the values of this field */ public void setInetAddressValue(InetAddress value) { if (value == null) { - throw new IllegalArgumentException("point cannot be null"); + throw new IllegalArgumentException("point must not be null"); } fieldsData = new BytesRef(encode(value)); } @@ -165,7 +165,7 @@ public static Query newExactQuery(String field, InetAddress value) { */ public static Query newPrefixQuery(String field, InetAddress value, int prefixLength) { if (value == null) { - throw new IllegalArgumentException("InetAddress cannot be null"); + throw new IllegalArgumentException("InetAddress must not be null"); } if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) { throw new IllegalArgumentException("illegal prefixLength '" + prefixLength + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges"); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 1278ea56aee7..60e83000083f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -236,7 +236,7 @@ static void checkCompatible(FieldInfo fieldInfo) { * Create a query for matching a bounding box. *

    * The box may cross over the dateline. - * @param field field name. cannot be null. + * @param field field name. must not be null. * @param minLatitude latitude lower bound: must be within standard +/-90 coordinate bounds. * @param maxLatitude latitude upper bound: must be within standard +/-90 coordinate bounds. * @param minLongitude longitude lower bound: must be within standard +/-180 coordinate bounds. @@ -288,7 +288,7 @@ protected String toString(int dimension, byte[] value) { /** * Create a query for matching points within the specified distance of the supplied location. - * @param field field name. cannot be null. + * @param field field name. must not be null. * @param latitude latitude at the center: must be within standard +/-90 coordinate bounds. * @param longitude longitude at the center: must be within standard +/-180 coordinate bounds. * @param radiusMeters maximum distance from the center in meters: must be non-negative and finite. @@ -303,7 +303,7 @@ public static Query newDistanceQuery(String field, double latitude, double longi * Create a query for matching a polygon. *

    * The supplied {@code polyLats}/{@code polyLons} must be clockwise or counter-clockwise. - * @param field field name. cannot be null. + * @param field field name. must not be null. * @param polyLats latitude values for points of the polygon: must be within standard +/-90 coordinate bounds. * @param polyLons longitude values for points of the polygon: must be within standard +/-180 coordinate bounds. * @return query matching points within this polygon @@ -327,7 +327,7 @@ public static Query newPolygonQuery(String field, double[] polyLats, double[] po *

    * If a document contains multiple values for the field, the closest distance to the location is used. * - * @param field field name. cannot be null. + * @param field field name. must not be null. * @param latitude latitude at the center: must be within standard +/-90 coordinate bounds. * @param longitude longitude at the center: must be within standard +/-180 coordinate bounds. * @return SortField ordering documents by distance diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index c283c7e6e9f3..1a9a11aff0f6 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -56,7 +56,7 @@ final class LatLonPointDistanceQuery extends Query { public LatLonPointDistanceQuery(String field, double latitude, double longitude, double radiusMeters) { if (field == null) { - throw new IllegalArgumentException("field cannot be null"); + throw new IllegalArgumentException("field must not be null"); } if (Double.isFinite(radiusMeters) == false || radiusMeters < 0) { throw new IllegalArgumentException("radiusMeters: '" + radiusMeters + "' is invalid"); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index c263ba340ae6..454a3ddf68b0 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.spatial.util.GeoRelationUtils; import org.apache.lucene.spatial.util.GeoUtils; @@ -60,52 +61,20 @@ final class LatLonPointInPolygonQuery extends Query { /** The lats/lons must be clockwise or counter-clockwise. */ public LatLonPointInPolygonQuery(String field, double[] polyLats, double[] polyLons) { - this.field = field; if (field == null) { - throw new IllegalArgumentException("field cannot be null"); - } - if (polyLats == null) { - throw new IllegalArgumentException("polyLats cannot be null"); - } - if (polyLons == null) { - throw new IllegalArgumentException("polyLons cannot be null"); - } - if (polyLats.length != polyLons.length) { - throw new IllegalArgumentException("polyLats and polyLons must be equal length"); - } - if (polyLats.length < 4) { - throw new IllegalArgumentException("at least 4 polygon points required"); - } - if (polyLats[0] != polyLats[polyLats.length-1]) { - throw new IllegalArgumentException("first and last points of the polygon must be the same (it must close itself): polyLats[0]=" + polyLats[0] + " polyLats[" + (polyLats.length-1) + "]=" + polyLats[polyLats.length-1]); + throw new IllegalArgumentException("field must not be null"); } - if (polyLons[0] != polyLons[polyLons.length-1]) { - throw new IllegalArgumentException("first and last points of the polygon must be the same (it must close itself): polyLons[0]=" + polyLons[0] + " polyLons[" + (polyLons.length-1) + "]=" + polyLons[polyLons.length-1]); - } - + GeoUtils.checkPolygon(polyLats, polyLons); + this.field = field; this.polyLats = polyLats; this.polyLons = polyLons; // TODO: we could also compute the maximal inner bounding box, to make relations faster to compute? - - double minLon = Double.POSITIVE_INFINITY; - double minLat = Double.POSITIVE_INFINITY; - double maxLon = Double.NEGATIVE_INFINITY; - double maxLat = Double.NEGATIVE_INFINITY; - for(int i=0;i { + newRectQuery(null, 18, 19, -66, -65); + }); + assertTrue(expected.getMessage().contains("field must not be null")); + } // box should not accept invalid lat/lon public void testBoxInvalidCoordinates() throws Exception { @@ -245,12 +252,21 @@ public void testDistanceBasics() throws Exception { dir.close(); } + /** null field name not allowed */ + public void testDistanceNull() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newDistanceQuery(null, 18, -65, 50_000); + }); + assertTrue(expected.getMessage().contains("field must not be null")); + } + /** distance query should not accept invalid lat/lon as origin */ public void testDistanceIllegal() throws Exception { expectThrows(Exception.class, () -> { newDistanceQuery("field", 92.0, 181.0, 120000); }); } + /** negative distance queries are not allowed */ public void testDistanceNegative() { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { @@ -307,6 +323,66 @@ public void testPolygonBasics() throws Exception { writer.close(); dir.close(); } + + /** null field name not allowed */ + public void testPolygonNullField() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery(null, + new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }); + }); + assertTrue(expected.getMessage().contains("field must not be null")); + } + + /** null polyLats not allowed */ + public void testPolygonNullPolyLats() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery("test", + null, + new double[] { -66, -65, -65, -66, -66 }); + }); + assertTrue(expected.getMessage().contains("polyLats must not be null")); + } + + /** null polyLons not allowed */ + public void testPolygonNullPolyLons() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery("test", + new double[] { 18, 18, 19, 19, 18 }, + null); + }); + assertTrue(expected.getMessage().contains("polyLons must not be null")); + } + + /** polygon needs at least 3 vertices */ + public void testPolygonLine() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery("test", + new double[] { 18, 18, 18 }, + new double[] { -66, -65, -66 }); + }); + assertTrue(expected.getMessage().contains("at least 4 polygon points required")); + } + + /** polygon needs same number of latitudes as longitudes */ + public void testPolygonBogus() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery("test", + new double[] { 18, 18, 19, 19 }, + new double[] { -66, -65, -65, -66, -66 }); + }); + assertTrue(expected.getMessage().contains("must be equal length")); + } + + /** polygon must be closed */ + public void testPolygonNotClosed() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + newPolygonQuery("test", + new double[] { 18, 18, 19, 19, 19 }, + new double[] { -66, -65, -65, -66, -67 }); + }); + assertTrue(expected.getMessage(), expected.getMessage().contains("it must close itself")); + } // A particularly tricky adversary for BKD tree: public void testSamePointManyTimes() throws Exception { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 15fa29c0915e..5378debf6a16 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -321,4 +321,40 @@ public void testPacManPolyQuery() throws Exception { assertFalse(GeoRelationUtils.rectCrossesPolyApprox(0, 5, -5, 0.000001, py, px, yMin, yMax, xMin, xMax)); assertTrue(GeoRelationUtils.rectWithinPolyApprox(0, 5, -5, -2, py, px, yMin, yMax, xMin, xMax)); } + + public void testPolyToBBox() throws Exception { + for (int i = 0; i < 1000; i++) { + double[][] polygon = GeoTestUtil.nextPolygon(); + GeoRect box = GeoUtils.polyToBBox(polygon[0], polygon[1]); + assertFalse(box.crossesDateline()); + + for (int j = 0; j < 1000; j++) { + double latitude = GeoTestUtil.nextLatitude(); + double longitude = GeoTestUtil.nextLongitude(); + // if the point is within poly, then it should be in our bounding box + if (GeoRelationUtils.pointInPolygon(polygon[0], polygon[1], latitude, longitude)) { + assertTrue(latitude >= box.minLat && latitude <= box.maxLat); + assertTrue(longitude >= box.minLon && longitude <= box.maxLon); + } + } + } + } + + public void testPolyToBBoxEdgeCases() throws Exception { + for (int i = 0; i < 1000; i++) { + double[][] polygon = GeoTestUtil.nextPolygon(); + GeoRect box = GeoUtils.polyToBBox(polygon[0], polygon[1]); + assertFalse(box.crossesDateline()); + + for (int j = 0; j < 1000; j++) { + double latitude = GeoTestUtil.nextLatitudeAround(box.minLat, box.maxLat); + double longitude = GeoTestUtil.nextLongitudeAround(box.minLon, box.maxLon); + // if the point is within poly, then it should be in our bounding box + if (GeoRelationUtils.pointInPolygon(polygon[0], polygon[1], latitude, longitude)) { + assertTrue(latitude >= box.minLat && latitude <= box.maxLat); + assertTrue(longitude >= box.minLon && longitude <= box.maxLon); + } + } + } + } } From 1ebb10c649ccc8da65e106d8b03a63bc454b50ce Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 28 Mar 2016 14:39:01 -0400 Subject: [PATCH 0236/1113] don't quantize ALL generated points in this test, just the ones we index (except for some leniency: see the TODO). This may cause more test failures! --- .../spatial/util/BaseGeoPointTestCase.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 595891046b83..021d638e759f 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -485,12 +485,12 @@ public void testMultiValued() throws Exception { for (int id=0;id Date: Mon, 28 Mar 2016 16:02:02 -0400 Subject: [PATCH 0237/1113] this test uses NaN to mean missing. don't try to quantize it. --- .../apache/lucene/spatial/util/BaseGeoPointTestCase.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 021d638e759f..0d6b5bdd630e 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -726,11 +726,16 @@ static final boolean rectContainsPoint(GeoRect rect, double pointLat, double poi private void verify(boolean small, double[] lats, double[] lons) throws Exception { // quantize each value the same way the index does + // NaN means missing for the doc!!!!! for (int i = 0; i < lats.length; i++) { - lats[i] = quantizeLat(lats[i]); + if (!Double.isNaN(lats[i])) { + lats[i] = quantizeLat(lats[i]); + } } for (int i = 0; i < lons.length; i++) { - lons[i] = quantizeLon(lons[i]); + if (!Double.isNaN(lons[i])) { + lons[i] = quantizeLon(lons[i]); + } } verifyRandomRectangles(small, lats, lons); verifyRandomDistances(small, lats, lons); From c769199c201f4989789bd0ce710a2120977fe4cf Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 29 Mar 2016 00:16:23 -0400 Subject: [PATCH 0238/1113] quantize random rects in test too (for now) until we can really remove this leniency... --- .../org/apache/lucene/spatial/util/BaseGeoPointTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 0d6b5bdd630e..dd0e09be96d6 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -799,6 +799,8 @@ protected void verifyRandomRectangles(boolean small, double[] lats, double[] lon } GeoRect rect = randomRect(small); + // TODO: why does this test need this quantization leniency? something is not right + rect = new GeoRect(quantizeLat(rect.minLat), quantizeLat(rect.maxLat), quantizeLon(rect.minLon), quantizeLon(rect.maxLon)); Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); From 5958bcb68e5326a2db8ffceb6fdf0e185fbd3195 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 29 Mar 2016 09:51:09 -0400 Subject: [PATCH 0239/1113] disable test method for deprecated encoding: its too memory-hungry --- .../spatial/geopoint/search/TestLegacyGeoPointQuery.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index e9b4c1273e81..00cc279a00a7 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -70,4 +70,9 @@ public void testRandomDistance() throws Exception { public void testRandomDistanceHuge() throws Exception { assumeTrue("legacy encoding is too slow/hangs on this test", false); } + + @Override + public void testSamePointManyTimes() throws Exception { + assumeTrue("legacy encoding goes OOM on this test", false); + } } From 0cf26bf368f9e08346df80f025ad021d5b3dbfe1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Mar 2016 12:14:15 -0700 Subject: [PATCH 0240/1113] LUCENE-7147: Improve disjoint check for geo distance query traversal --- lucene/CHANGES.txt | 3 + .../document/LatLonPointDistanceQuery.java | 18 +- .../search/GeoPointDistanceQueryImpl.java | 16 +- .../apache/lucene/spatial/util/GeoUtils.java | 50 +++- .../lucene/spatial/util/GeoTestUtil.java | 231 ++++++++++++++++++ .../lucene/spatial/util/TestGeoUtils.java | 134 ++++++++++ 6 files changed, 447 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index ed1d83d3ec73..b80b577e1f4d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -37,6 +37,9 @@ Optimizations * LUCENE-7115: Speed up FieldCache.CacheEntry toString by setting initial StringBuilder capacity (Gregory Chanan) +* LUCENE-7147: Improve disjoint check for geo distance query traversal + (Ryan Ernst, Robert Muir, Mike McCandless) + Bug Fixes * LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 1a9a11aff0f6..04d37233c956 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -108,6 +108,8 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I maxPartialDistance = Double.POSITIVE_INFINITY; } + final double axisLat = GeoUtils.axisLat(latitude, radiusMeters); + return new ConstantScoreWeight(this) { @Override @@ -171,8 +173,9 @@ public void visit(int docID, byte[] packedValue) { // algorithm: we create a bounding box (two bounding boxes if we cross the dateline). // 1. check our bounding box(es) first. if the subtree is entirely outside of those, bail. - // 2. see if the subtree is fully contained. if the subtree is enormous along the x axis, wrapping half way around the world, etc: then this can't work, just go to step 3. - // 3. recurse naively. + // 2. check if the subtree is disjoint. it may cross the bounding box but not intersect with circle + // 3. see if the subtree is fully contained. if the subtree is enormous along the x axis, wrapping half way around the world, etc: then this can't work, just go to step 3. + // 4. recurse naively (subtrees crossing over circle edge) @Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 || @@ -193,6 +196,17 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); + if ((longitude < lonMin || longitude > lonMax) && (axisLat+GeoUtils.AXISLAT_ERROR < latMin || axisLat-GeoUtils.AXISLAT_ERROR > latMax)) { + // circle not fully inside / crossing axis + if (SloppyMath.haversinMeters(latitude, longitude, latMin, lonMin) > radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMin, lonMax) > radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMax, lonMin) > radiusMeters && + SloppyMath.haversinMeters(latitude, longitude, latMax, lonMax) > radiusMeters) { + // no points inside + return Relation.CELL_OUTSIDE_QUERY; + } + } + if (lonMax - longitude < 90 && longitude - lonMin < 90 && SloppyMath.haversinMeters(latitude, longitude, latMin, lonMin) <= radiusMeters && SloppyMath.haversinMeters(latitude, longitude, latMin, lonMax) <= radiusMeters && diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index 243f33e26ddc..dea2be80f81b 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.SloppyMath; /** Package private implementation for the public facing GeoPointDistanceQuery delegate class. @@ -32,6 +33,9 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { // optimization, maximum partial haversin needed to be a candidate private final double maxPartialDistance; + // optimization, used for detecting axis cross + final double axisLat; + GeoPointDistanceQueryImpl(final String field, final TermEncoding termEncoding, final GeoPointDistanceQuery q, final double centerLonUnwrapped, final GeoRect bbox) { super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); @@ -46,6 +50,7 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { } else { maxPartialDistance = Double.POSITIVE_INFINITY; } + axisLat = GeoUtils.axisLat(distanceQuery.centerLat, distanceQuery.radiusMeters); } @Override @@ -65,14 +70,21 @@ private final class GeoPointRadiusCellComparator extends CellComparator { @Override protected boolean cellCrosses(final double minLat, final double maxLat, final double minLon, final double maxLon) { + // bounding box check if (maxLat < GeoPointDistanceQueryImpl.this.minLat || maxLon < GeoPointDistanceQueryImpl.this.minLon || minLat > GeoPointDistanceQueryImpl.this.maxLat || minLon > GeoPointDistanceQueryImpl.this.maxLon) { return false; - } else { - return true; + } else if ((centerLon < minLon || centerLon > maxLon) && (axisLat+GeoUtils.AXISLAT_ERROR < minLat || axisLat-GeoUtils.AXISLAT_ERROR > maxLat)) { + if (SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, minLon) > distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, maxLon) > distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, minLon) > distanceQuery.radiusMeters && + SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, maxLon) > distanceQuery.radiusMeters) { + return false; + } } + return true; } @Override diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index ecb50460edc4..52e94052798f 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -109,7 +109,8 @@ public static void checkPolygon(double[] polyLats, double[] polyLons) { public static GeoRect circleToBBox(final double centerLat, final double centerLon, final double radiusMeters) { final double radLat = TO_RADIANS * centerLat; final double radLon = TO_RADIANS * centerLon; - double radDistance = radiusMeters / SEMIMAJOR_AXIS; + // LUCENE-7143 + double radDistance = (radiusMeters + 7E-2) / SEMIMAJOR_AXIS; double minLat = radLat - radDistance; double maxLat = radLat + radDistance; double minLon; @@ -176,4 +177,51 @@ private static double sloppySin(double a) { return cos(a - PIO2); } + /** maximum error from {@link #axisLat(double, double)}. logic must be prepared to handle this */ + public static final double AXISLAT_ERROR = 0.1D / SEMIMAJOR_AXIS * TO_DEGREES; + + /** + * Calculate the latitude of a circle's intersections with its bbox meridians. + *

    + * NOTE: the returned value will be +/- {@link #AXISLAT_ERROR} of the actual value. + * @param centerLat The latitude of the circle center + * @param radiusMeters The radius of the circle in meters + * @return A latitude + */ + public static double axisLat(double centerLat, double radiusMeters) { + // A spherical triangle with: + // r is the radius of the circle in radians + // l1 is the latitude of the circle center + // l2 is the latitude of the point at which the circle intersect's its bbox longitudes + // We know r is tangent to the bbox meridians at l2, therefore it is a right angle. + // So from the law of cosines, with the angle of l1 being 90, we have: + // cos(l1) = cos(r) * cos(l2) + sin(r) * sin(l2) * cos(90) + // The second part cancels out because cos(90) == 0, so we have: + // cos(l1) = cos(r) * cos(l2) + // Solving for l2, we get: + // l2 = acos( cos(l1) / cos(r) ) + // We ensure r is in the range (0, PI/2) and l1 in the range (0, PI/2]. This means we + // cannot divide by 0, and we will always get a positive value in the range [0, 1) as + // the argument to arc cosine, resulting in a range (0, PI/2]. + + double l1 = TO_RADIANS * centerLat; + double r = (radiusMeters + 7E-2) / SEMIMAJOR_AXIS; + + // if we are within radius range of a pole, the lat is the pole itself + if (Math.abs(l1) + r >= MAX_LAT_RADIANS) { + return centerLat >= 0 ? MAX_LAT_INCL : MIN_LAT_INCL; + } + + // adjust l1 as distance from closest pole, to form a right triangle with bbox meridians + // and ensure it is in the range (0, PI/2] + l1 = centerLat >= 0 ? PIO2 - l1 : l1 + PIO2; + + double l2 = Math.acos(Math.cos(l1) / Math.cos(r)); + assert !Double.isNaN(l2); + + // now adjust back to range [-pi/2, pi/2], ie latitude in radians + l2 = centerLat >= 0 ? PIO2 - l2 : l2 - PIO2; + + return TO_DEGREES * l2; + } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java index 0059888a63f8..e52c8a587d9e 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java @@ -17,8 +17,11 @@ package org.apache.lucene.spatial.util; import java.util.ArrayList; +import java.util.List; import java.util.Random; +import org.apache.lucene.util.SloppyMath; + import com.carrotsearch.randomizedtesting.RandomizedContext; /** static methods for testing geo */ @@ -266,4 +269,232 @@ private static double normalizeLongitude(double longitude) { private static Random random() { return RandomizedContext.current().getRandom(); } + + // craziness for plotting stuff :) + + private static double wrapLat(double lat) { + //System.out.println("wrapLat " + lat); + if (lat > 90) { + //System.out.println(" " + (180 - lat)); + return 180 - lat; + } else if (lat < -90) { + //System.out.println(" " + (-180 - lat)); + return -180 - lat; + } else { + //System.out.println(" " + lat); + return lat; + } + } + + private static double wrapLon(double lon) { + //System.out.println("wrapLon " + lon); + if (lon > 180) { + //System.out.println(" " + (lon - 360)); + return lon - 360; + } else if (lon < -180) { + //System.out.println(" " + (lon + 360)); + return lon + 360; + } else { + //System.out.println(" " + lon); + return lon; + } + } + + private static void drawRectApproximatelyOnEarthSurface(String name, String color, double minLat, double maxLat, double minLon, double maxLon) { + int steps = 20; + System.out.println(" var " + name + " = WE.polygon(["); + System.out.println(" // min -> max lat, min lon"); + for(int i=0;i max lon"); + for(int i=0;i min lat, max lon"); + for(int i=0;i min lon"); + for(int i=0;i=minLon;lon -= (maxLon-minLon)/36) { + System.out.println(" [" + lat + ", " + lon + "],"); + } + System.out.println(" ], {color: \"" + color + "\", fillColor: \"#ffffff\", opacity: " + (color.equals("#ffffff") ? "0.3" : "1") + ", fillOpacity: 0.0001});"); + System.out.println(" " + name + ".addTo(earth);"); + } + + private static void plotLonApproximatelyOnEarthSurface(String name, String color, double lon, double minLat, double maxLat) { + System.out.println(" var " + name + " = WE.polygon(["); + double lat; + for(lat = minLat;lat<=maxLat;lat += (maxLat-minLat)/36) { + System.out.println(" [" + lat + ", " + lon + "],"); + } + System.out.println(" [" + maxLat + ", " + lon + "],"); + lat -= (maxLat-minLat)/36; + for(;lat>=minLat;lat -= (maxLat-minLat)/36) { + System.out.println(" [" + lat + ", " + lon + "],"); + } + System.out.println(" ], {color: \"" + color + "\", fillColor: \"#ffffff\", opacity: " + (color.equals("#ffffff") ? "0.3" : "1") + ", fillOpacity: 0.0001});"); + System.out.println(" " + name + ".addTo(earth);"); + } + + // http://www.webglearth.org has API details: + public static void polysToWebGLEarth(List polys) { + System.out.println(""); + System.out.println(""); + System.out.println(" "); + System.out.println(" "); + System.out.println(" "); + System.out.println(" "); + System.out.println(" WebGL Earth API: Hello World"); + System.out.println(" "); + System.out.println(" "); + System.out.println("

    "); + System.out.println(" "); + System.out.println(""); + } + + // http://www.webglearth.org has API details: + public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, + double rectMinLongitude, double rectMaxLongitude, + double centerLatitude, double centerLongitude, + double radiusMeters) { + GeoRect box = GeoUtils.circleToBBox(centerLatitude, centerLongitude, radiusMeters); + System.out.println(""); + System.out.println(""); + System.out.println(" "); + System.out.println(" "); + System.out.println(" "); + System.out.println(" "); + System.out.println(" WebGL Earth API: Hello World"); + System.out.println(" "); + System.out.println(" "); + System.out.println("
    "); + System.out.println(" "); + System.out.println(""); + } + + private static void inverseHaversin(StringBuilder b, double centerLat, double centerLon, double radiusMeters) { + double angle = 0; + int steps = 100; + + newAngle: + while (angle < 360) { + double x = Math.cos(Math.toRadians(angle)); + double y = Math.sin(Math.toRadians(angle)); + double factor = 2.0; + double step = 1.0; + int last = 0; + double lastDistanceMeters = 0.0; + //System.out.println("angle " + angle + " slope=" + slope); + while (true) { + double lat = wrapLat(centerLat + y * factor); + double lon = wrapLon(centerLon + x * factor); + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, lat, lon); + + if (last == 1 && distanceMeters < lastDistanceMeters) { + // For large enough circles, some angles are not possible: + //System.out.println(" done: give up on angle " + angle); + angle += 360./steps; + continue newAngle; + } + if (last == -1 && distanceMeters > lastDistanceMeters) { + // For large enough circles, some angles are not possible: + //System.out.println(" done: give up on angle " + angle); + angle += 360./steps; + continue newAngle; + } + lastDistanceMeters = distanceMeters; + + //System.out.println(" iter lat=" + lat + " lon=" + lon + " distance=" + distanceMeters + " vs " + radiusMeters); + if (Math.abs(distanceMeters - radiusMeters) < 0.1) { + b.append(" [" + lat + ", " + lon + "],\n"); + break; + } + if (distanceMeters > radiusMeters) { + // too big + //System.out.println(" smaller"); + factor -= step; + if (last == 1) { + //System.out.println(" half-step"); + step /= 2.0; + } + last = -1; + } else if (distanceMeters < radiusMeters) { + // too small + //System.out.println(" bigger"); + factor += step; + if (last == -1) { + //System.out.println(" half-step"); + step /= 2.0; + } + last = 1; + } + } + angle += 360./steps; + } + } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index f6fb139a7bdf..9d6549a010c5 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -16,6 +16,8 @@ */ package org.apache.lucene.spatial.util; +import java.util.Locale; + import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -320,4 +322,136 @@ public void testPolyToBBoxEdgeCases() throws Exception { } } } + + public void testAxisLat() { + double earthCircumference = 2D * Math.PI * GeoUtils.SEMIMAJOR_AXIS; + assertEquals(90, GeoUtils.axisLat(0, earthCircumference / 4), 0.0D); + + for (int i = 0; i < 100; ++i) { + boolean reallyBig = random().nextInt(10) == 0; + final double maxRadius = reallyBig ? 1.1 * earthCircumference : earthCircumference / 8; + final double radius = maxRadius * random().nextDouble(); + double prevAxisLat = GeoUtils.axisLat(0.0D, radius); + for (double lat = 0.1D; lat < 90D; lat += 0.1D) { + double nextAxisLat = GeoUtils.axisLat(lat, radius); + GeoRect bbox = GeoUtils.circleToBBox(lat, 180D, radius); + double dist = SloppyMath.haversinMeters(lat, 180D, nextAxisLat, bbox.maxLon); + if (nextAxisLat < GeoUtils.MAX_LAT_INCL) { + assertEquals("lat = " + lat, dist, radius, 0.1D); + } + assertTrue("lat = " + lat, prevAxisLat <= nextAxisLat); + prevAxisLat = nextAxisLat; + } + + prevAxisLat = GeoUtils.axisLat(-0.0D, radius); + for (double lat = -0.1D; lat > -90D; lat -= 0.1D) { + double nextAxisLat = GeoUtils.axisLat(lat, radius); + GeoRect bbox = GeoUtils.circleToBBox(lat, 180D, radius); + double dist = SloppyMath.haversinMeters(lat, 180D, nextAxisLat, bbox.maxLon); + if (nextAxisLat > GeoUtils.MIN_LAT_INCL) { + assertEquals("lat = " + lat, dist, radius, 0.1D); + } + assertTrue("lat = " + lat, prevAxisLat >= nextAxisLat); + prevAxisLat = nextAxisLat; + } + } + } + + // TODO: does not really belong here, but we test it like this for now + // we can make a fake IndexReader to send boxes directly to Point visitors instead? + public void testCircleOpto() throws Exception { + for (int i = 0; i < 50; i++) { + // circle + final double centerLat = -90 + 180.0 * random().nextDouble(); + final double centerLon = -180 + 360.0 * random().nextDouble(); + final double radius = 50_000_000D * random().nextDouble(); + final GeoRect box = GeoUtils.circleToBBox(centerLat, centerLon, radius); + // TODO: remove this leniency! + if (box.crossesDateline()) { + --i; // try again... + continue; + } + final double axisLat = GeoUtils.axisLat(centerLat, radius); + + for (int k = 0; k < 1000; ++k) { + + double[] latBounds = {-90, box.minLat, axisLat, box.maxLat, 90}; + double[] lonBounds = {-180, box.minLon, centerLon, box.maxLon, 180}; + // first choose an upper left corner + int maxLatRow = random().nextInt(4); + double latMax = randomInRange(latBounds[maxLatRow], latBounds[maxLatRow + 1]); + int minLonCol = random().nextInt(4); + double lonMin = randomInRange(lonBounds[minLonCol], lonBounds[minLonCol + 1]); + // now choose a lower right corner + int minLatMaxRow = maxLatRow == 3 ? 3 : maxLatRow + 1; // make sure it will at least cross into the bbox + int minLatRow = random().nextInt(minLatMaxRow); + double latMin = randomInRange(latBounds[minLatRow], Math.min(latBounds[minLatRow + 1], latMax)); + int maxLonMinCol = Math.max(minLonCol, 1); // make sure it will at least cross into the bbox + int maxLonCol = maxLonMinCol + random().nextInt(4 - maxLonMinCol); + double lonMax = randomInRange(Math.max(lonBounds[maxLonCol], lonMin), lonBounds[maxLonCol + 1]); + + assert latMax >= latMin; + assert lonMax >= lonMin; + + if (isDisjoint(centerLat, centerLon, radius, axisLat, latMin, latMax, lonMin, lonMax)) { + // intersects says false: test a ton of points + for (int j = 0; j < 200; j++) { + double lat = latMin + (latMax - latMin) * random().nextDouble(); + double lon = lonMin + (lonMax - lonMin) * random().nextDouble(); + + if (random().nextBoolean()) { + // explicitly test an edge + int edge = random().nextInt(4); + if (edge == 0) { + lat = latMin; + } else if (edge == 1) { + lat = latMax; + } else if (edge == 2) { + lon = lonMin; + } else if (edge == 3) { + lon = lonMax; + } + } + double distance = SloppyMath.haversinMeters(centerLat, centerLon, lat, lon); + try { + assertTrue(String.format(Locale.ROOT, "\nisDisjoint(\n" + + "centerLat=%s\n" + + "centerLon=%s\n" + + "radius=%s\n" + + "latMin=%s\n" + + "latMax=%s\n" + + "lonMin=%s\n" + + "lonMax=%s) == false BUT\n" + + "haversin(%s, %s, %s, %s) = %s\nbbox=%s", + centerLat, centerLon, radius, latMin, latMax, lonMin, lonMax, + centerLat, centerLon, lat, lon, distance, GeoUtils.circleToBBox(centerLat, centerLon, radius)), + distance > radius); + } catch (AssertionError e) { + GeoTestUtil.toWebGLEarth(latMin, latMax, lonMin, lonMax, centerLat, centerLon, radius); + throw e; + } + } + } + } + } + } + + static double randomInRange(double min, double max) { + return min + (max - min) * random().nextDouble(); + } + + static boolean isDisjoint(double centerLat, double centerLon, double radius, double axisLat, double latMin, double latMax, double lonMin, double lonMax) { + if ((centerLon < lonMin || centerLon > lonMax) && (axisLat+GeoUtils.AXISLAT_ERROR < latMin || axisLat-GeoUtils.AXISLAT_ERROR > latMax)) { + // circle not fully inside / crossing axis + if (SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMin) > radius && + SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMax) > radius && + SloppyMath.haversinMeters(centerLat, centerLon, latMax, lonMin) > radius && + SloppyMath.haversinMeters(centerLat, centerLon, latMax, lonMax) > radius) { + // no points inside + return true; + } + } + + return false; + } } From e2d58ccb3711d838df55d99ea63fb4d075fcc9c8 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 30 Mar 2016 05:52:05 -0400 Subject: [PATCH 0241/1113] LUCENE-7122: handle fixed length byte[] values more efficiently in OfflineSorter --- .../org/apache/lucene/util/BytesRefArray.java | 15 +- .../lucene/util/FixedLengthBytesRefArray.java | 189 ++++++++++++++++++ .../org/apache/lucene/util/OfflineSorter.java | 75 ++++--- .../lucene/util/SortableBytesRefArray.java | 31 +++ .../org/apache/lucene/util/bkd/BKDWriter.java | 17 +- .../lucene/util/bkd/OfflinePointWriter.java | 15 +- .../util/TestFixedLengthBytesRefArray.java | 84 ++++++++ .../apache/lucene/util/TestOfflineSorter.java | 68 ++++++- 8 files changed, 439 insertions(+), 55 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/util/FixedLengthBytesRefArray.java create mode 100644 lucene/core/src/java/org/apache/lucene/util/SortableBytesRefArray.java create mode 100644 lucene/core/src/test/org/apache/lucene/util/TestFixedLengthBytesRefArray.java diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java b/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java index a19b7da5abed..bab3d04097a2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java +++ b/lucene/core/src/java/org/apache/lucene/util/BytesRefArray.java @@ -19,14 +19,6 @@ import java.util.Arrays; import java.util.Comparator; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.ByteBlockPool; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.Counter; -import org.apache.lucene.util.IntroSorter; -import org.apache.lucene.util.RamUsageEstimator; - /** * A simple append only random-access {@link BytesRef} array that stores full * copies of the appended bytes in a {@link ByteBlockPool}. @@ -37,7 +29,7 @@ * @lucene.internal * @lucene.experimental */ -public final class BytesRefArray { +public final class BytesRefArray implements SortableBytesRefArray { private final ByteBlockPool pool; private int[] offsets = new int[1]; private int lastElement = 0; @@ -58,9 +50,11 @@ public BytesRefArray(Counter bytesUsed) { /** * Clears this {@link BytesRefArray} */ + @Override public void clear() { lastElement = 0; currentOffset = 0; + // TODO: it's trappy that this does not return storage held by int[] offsets array! Arrays.fill(offsets, 0); pool.reset(false, true); // no need to 0 fill the buffers we control the allocator } @@ -70,6 +64,7 @@ public void clear() { * @param bytes the bytes to append * @return the index of the appended bytes */ + @Override public int append(BytesRef bytes) { if (lastElement >= offsets.length) { int oldLen = offsets.length; @@ -86,6 +81,7 @@ public int append(BytesRef bytes) { * Returns the current size of this {@link BytesRefArray} * @return the current size of this {@link BytesRefArray} */ + @Override public int size() { return lastElement; } @@ -192,6 +188,7 @@ public BytesRefIterator iterator() { * This is a non-destructive operation. *

    */ + @Override public BytesRefIterator iterator(final Comparator comp) { final BytesRefBuilder spare = new BytesRefBuilder(); final BytesRef result = new BytesRef(); diff --git a/lucene/core/src/java/org/apache/lucene/util/FixedLengthBytesRefArray.java b/lucene/core/src/java/org/apache/lucene/util/FixedLengthBytesRefArray.java new file mode 100644 index 000000000000..346b908bc6fc --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/FixedLengthBytesRefArray.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Comparator; + +/** + * Just like {@link BytesRefArray} except all values have the same length. + * + * Note: This class is not Thread-Safe! + * + * @lucene.internal + * @lucene.experimental + */ +final class FixedLengthBytesRefArray implements SortableBytesRefArray { + private final int valueLength; + private final int valuesPerBlock; + + /** How many values have been appended */ + private int size; + + /** How many blocks are used */ + private int currentBlock = -1; + private int nextEntry; + + private byte[][] blocks; + + /** + * Creates a new {@link BytesRefArray} with a counter to track allocated bytes + */ + public FixedLengthBytesRefArray(int valueLength) { + this.valueLength = valueLength; + + // ~32K per page, unless each value is > 32K: + valuesPerBlock = Math.max(1, 32768/valueLength); + nextEntry = valuesPerBlock; + blocks = new byte[0][]; + } + + /** + * Clears this {@link BytesRefArray} + */ + @Override + public void clear() { + size = 0; + blocks = new byte[0][]; + currentBlock = -1; + nextEntry = valuesPerBlock; + } + + /** + * Appends a copy of the given {@link BytesRef} to this {@link BytesRefArray}. + * @param bytes the bytes to append + * @return the index of the appended bytes + */ + @Override + public int append(BytesRef bytes) { + if (bytes.length != valueLength) { + throw new IllegalArgumentException("value length is " + bytes.length + " but is supposed to always be " + valueLength); + } + if (nextEntry == valuesPerBlock) { + currentBlock++; + if (currentBlock == blocks.length) { + int size = ArrayUtil.oversize(currentBlock+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF); + byte[][] next = new byte[size][]; + System.arraycopy(blocks, 0, next, 0, blocks.length); + blocks = next; + } + blocks[currentBlock] = new byte[valuesPerBlock * valueLength]; + nextEntry = 0; + } + + System.arraycopy(bytes.bytes, bytes.offset, blocks[currentBlock], nextEntry*valueLength, valueLength); + nextEntry++; + + return size++; + } + + /** + * Returns the current size of this {@link FixedLengthBytesRefArray} + * @return the current size of this {@link FixedLengthBytesRefArray} + */ + @Override + public int size() { + return size; + } + + private int[] sort(final Comparator comp) { + final int[] orderedEntries = new int[size()]; + for (int i = 0; i < orderedEntries.length; i++) { + orderedEntries[i] = i; + } + + final BytesRef pivot = new BytesRef(); + final BytesRef scratch1 = new BytesRef(); + final BytesRef scratch2 = new BytesRef(); + pivot.length = valueLength; + scratch1.length = valueLength; + scratch2.length = valueLength; + + new IntroSorter() { + + @Override + protected void swap(int i, int j) { + int o = orderedEntries[i]; + orderedEntries[i] = orderedEntries[j]; + orderedEntries[j] = o; + } + + @Override + protected int compare(int i, int j) { + int index1 = orderedEntries[i]; + scratch1.bytes = blocks[index1 / valuesPerBlock]; + scratch1.offset = (index1 % valuesPerBlock) * valueLength; + + int index2 = orderedEntries[j]; + scratch2.bytes = blocks[index2 / valuesPerBlock]; + scratch2.offset = (index2 % valuesPerBlock) * valueLength; + + return comp.compare(scratch1, scratch2); + } + + @Override + protected void setPivot(int i) { + int index = orderedEntries[i]; + pivot.bytes = blocks[index / valuesPerBlock]; + pivot.offset = (index % valuesPerBlock) * valueLength; + } + + @Override + protected int comparePivot(int j) { + final int index = orderedEntries[j]; + scratch2.bytes = blocks[index / valuesPerBlock]; + scratch2.offset = (index % valuesPerBlock) * valueLength; + return comp.compare(pivot, scratch2); + } + }.sort(0, size()); + return orderedEntries; + } + + /** + *

    + * Returns a {@link BytesRefIterator} with point in time semantics. The + * iterator provides access to all so far appended {@link BytesRef} instances. + *

    + *

    + * The iterator will iterate the byte values in the order specified by the comparator. + *

    + *

    + * This is a non-destructive operation. + *

    + */ + @Override + public BytesRefIterator iterator(final Comparator comp) { + final BytesRef result = new BytesRef(); + result.length = valueLength; + final int size = size(); + final int[] indices = sort(comp); + return new BytesRefIterator() { + int pos = 0; + + @Override + public BytesRef next() { + if (pos < size) { + int index = indices[pos]; + pos++; + result.bytes = blocks[index / valuesPerBlock]; + result.offset = (index % valuesPerBlock) * valueLength; + return result; + } + return null; + } + }; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java index 386cd0c24933..fa223205a7ed 100644 --- a/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java @@ -69,7 +69,7 @@ public class OfflineSorter { public final static int MAX_TEMPFILES = 10; private final Directory dir; - + private final int valueLength; private final String tempFileNamePrefix; /** @@ -170,7 +170,7 @@ public String toString() { private final BufferSize ramBufferSize; private final Counter bufferBytesUsed = Counter.newCounter(); - private final BytesRefArray buffer = new BytesRefArray(bufferBytesUsed); + private final SortableBytesRefArray buffer; SortInfo sortInfo; private int maxTempFiles; private final Comparator comparator; @@ -184,7 +184,7 @@ public String toString() { * @see BufferSize#automatic() */ public OfflineSorter(Directory dir, String tempFileNamePrefix) throws IOException { - this(dir, tempFileNamePrefix, DEFAULT_COMPARATOR, BufferSize.automatic(), MAX_TEMPFILES); + this(dir, tempFileNamePrefix, DEFAULT_COMPARATOR, BufferSize.automatic(), MAX_TEMPFILES, -1); } /** @@ -193,13 +193,14 @@ public OfflineSorter(Directory dir, String tempFileNamePrefix) throws IOExceptio * @see BufferSize#automatic() */ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator comparator) throws IOException { - this(dir, tempFileNamePrefix, comparator, BufferSize.automatic(), MAX_TEMPFILES); + this(dir, tempFileNamePrefix, comparator, BufferSize.automatic(), MAX_TEMPFILES, -1); } /** - * All-details constructor. + * All-details constructor. If {@code valueLength} is -1 (the default), the length of each value differs; otherwise, + * all values have the specified length. */ - public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator comparator, BufferSize ramBufferSize, int maxTempfiles) { + public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator comparator, BufferSize ramBufferSize, int maxTempfiles, int valueLength) { if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) { throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + ramBufferSize.bytes); } @@ -207,7 +208,15 @@ public OfflineSorter(Directory dir, String tempFileNamePrefix, Comparator= 2"); } - + if (valueLength == -1) { + buffer = new BytesRefArray(bufferBytesUsed); + } else { + if (valueLength == 0 || valueLength > Short.MAX_VALUE) { + throw new IllegalArgumentException("valueLength must be 1 .. " + Short.MAX_VALUE + "; got: " + valueLength); + } + buffer = new FixedLengthBytesRefArray(valueLength); + } + this.valueLength = valueLength; this.ramBufferSize = ramBufferSize; this.maxTempFiles = maxTempfiles; this.comparator = comparator; @@ -283,7 +292,7 @@ public String sort(String inputFileName) throws IOException { // We should be explicitly removing all intermediate files ourselves unless there is an exception: assert trackingDir.getCreatedFiles().size() == 1 && trackingDir.getCreatedFiles().contains(result); - sortInfo.totalTime = (System.currentTimeMillis() - sortInfo.totalTime); + sortInfo.totalTime = System.currentTimeMillis() - sortInfo.totalTime; CodecUtil.checkFooter(is.in); @@ -308,7 +317,7 @@ protected String sortPartition(TrackingDirectoryWrapper trackingDir) throws IOEx long start = System.currentTimeMillis(); BytesRefIterator iter = buffer.iterator(comparator); - sortInfo.sortTime += (System.currentTimeMillis() - start); + sortInfo.sortTime += System.currentTimeMillis() - start; while ((spare = iter.next()) != null) { assert spare.length <= Short.MAX_VALUE; @@ -414,24 +423,40 @@ protected boolean lessThan(FileAndTop a, FileAndTop b) { /** Read in a single partition of data */ int readPartition(ByteSequencesReader reader) throws IOException { long start = System.currentTimeMillis(); - while (true) { - BytesRef item = null; - try { - item = reader.next(); - } catch (Throwable t) { - verifyChecksum(t, reader); - } - if (item == null) { - break; + if (valueLength != -1) { + int limit = ramBufferSize.bytes / valueLength; + for(int i=0;i ramBufferSize.bytes) { - break; + } else { + while (true) { + BytesRef item = null; + try { + item = reader.next(); + } catch (Throwable t) { + verifyChecksum(t, reader); + } + if (item == null) { + break; + } + buffer.append(item); + // Account for the created objects. + // (buffer slots do not account to buffer size.) + if (bufferBytesUsed.get() > ramBufferSize.bytes) { + break; + } } } - sortInfo.readTime += (System.currentTimeMillis() - start); + sortInfo.readTime += System.currentTimeMillis() - start; return buffer.size(); } @@ -463,6 +488,8 @@ protected ByteSequencesReader getReader(ChecksumIndexInput in, String name) thro public static class ByteSequencesWriter implements Closeable { protected final IndexOutput out; + // TODO: this should optimize the fixed width case as well + /** Constructs a ByteSequencesWriter to the provided DataOutput */ public ByteSequencesWriter(IndexOutput out) { this.out = out; diff --git a/lucene/core/src/java/org/apache/lucene/util/SortableBytesRefArray.java b/lucene/core/src/java/org/apache/lucene/util/SortableBytesRefArray.java new file mode 100644 index 000000000000..086fcbb4535f --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/SortableBytesRefArray.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util; + +import java.util.Comparator; + +interface SortableBytesRefArray { + /** Append a new value */ + int append(BytesRef bytes); + /** Clear all previously stored values */ + void clear(); + /** Returns the number of values appended so far */ + int size(); + /** Sort all values by the provided comparator and return an iterator over the sorted values */ + BytesRefIterator iterator(final Comparator comp); +} diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 3cf32a9b0d30..288ece4c51dd 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -722,6 +722,8 @@ private PointWriter sort(int dim) throws IOException { // Offline sort: assert tempInput != null; + final int offset = bytesPerDim * dim; + Comparator cmp = new Comparator() { final ByteArrayDataInput reader = new ByteArrayDataInput(); @@ -729,7 +731,7 @@ private PointWriter sort(int dim) throws IOException { @Override public int compare(BytesRef a, BytesRef b) { // First compare by the requested dimension we are sorting by: - int cmp = StringHelper.compare(bytesPerDim, a.bytes, a.offset + bytesPerDim*dim, b.bytes, b.offset + bytesPerDim*dim); + int cmp = StringHelper.compare(bytesPerDim, a.bytes, a.offset + offset, b.bytes, b.offset + offset); if (cmp != 0) { return cmp; @@ -745,15 +747,11 @@ public int compare(BytesRef a, BytesRef b) { } }; - // TODO: this is sort of sneaky way to get the final OfflinePointWriter from OfflineSorter: - IndexOutput[] lastWriter = new IndexOutput[1]; - - OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, offlineSorterBufferMB, offlineSorterMaxTempFiles) { + OfflineSorter sorter = new OfflineSorter(tempDir, tempFileNamePrefix + "_bkd" + dim, cmp, offlineSorterBufferMB, offlineSorterMaxTempFiles, bytesPerDoc) { /** We write/read fixed-byte-width file that {@link OfflinePointReader} can read. */ @Override protected ByteSequencesWriter getWriter(IndexOutput out) { - lastWriter[0] = out; return new ByteSequencesWriter(out) { @Override public void write(byte[] bytes, int off, int len) throws IOException { @@ -779,11 +777,10 @@ public BytesRef next() throws IOException { }; } }; - sorter.sort(tempInput.getName()); - assert lastWriter[0] != null; + String name = sorter.sort(tempInput.getName()); - return new OfflinePointWriter(tempDir, lastWriter[0], packedBytesLength, pointCount, longOrds, singleValuePerDoc); + return new OfflinePointWriter(tempDir, name, packedBytesLength, pointCount, longOrds, singleValuePerDoc); } } @@ -994,7 +991,7 @@ private void verifyChecksum(Throwable priorException, PointWriter writer) throws // and would mean leaving readers (IndexInputs) open for longer: if (writer instanceof OfflinePointWriter) { // We are reading from a temp file; go verify the checksum: - String tempFileName = ((OfflinePointWriter) writer).out.getName(); + String tempFileName = ((OfflinePointWriter) writer).name; try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName, IOContext.READONCE)) { CodecUtil.checkFooter(in, priorException); } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java index 2a5952e5c260..87637ae27ba7 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java @@ -31,6 +31,7 @@ final class OfflinePointWriter implements PointWriter { final Directory tempDir; final IndexOutput out; + final String name; final int packedBytesLength; final boolean singleValuePerDoc; long count; @@ -45,6 +46,7 @@ final class OfflinePointWriter implements PointWriter { public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int packedBytesLength, boolean longOrds, String desc, long expectedCount, boolean singleValuePerDoc) throws IOException { this.out = tempDir.createTempOutput(tempFileNamePrefix, "bkd_" + desc, IOContext.DEFAULT); + this.name = out.getName(); this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.longOrds = longOrds; @@ -53,8 +55,9 @@ public OfflinePointWriter(Directory tempDir, String tempFileNamePrefix, int pack } /** Initializes on an already written/closed file, just so consumers can use {@link #getReader} to read the file. */ - public OfflinePointWriter(Directory tempDir, IndexOutput out, int packedBytesLength, long count, boolean longOrds, boolean singleValuePerDoc) { - this.out = out; + public OfflinePointWriter(Directory tempDir, String name, int packedBytesLength, long count, boolean longOrds, boolean singleValuePerDoc) { + this.out = null; + this.name = name; this.tempDir = tempDir; this.packedBytesLength = packedBytesLength; this.count = count; @@ -86,7 +89,7 @@ public PointReader getReader(long start, long length) throws IOException { assert closed; assert start + length <= count: "start=" + start + " length=" + length + " count=" + count; assert expectedCount == 0 || count == expectedCount; - return new OfflinePointReader(tempDir, out.getName(), packedBytesLength, start, length, longOrds, singleValuePerDoc); + return new OfflinePointReader(tempDir, name, packedBytesLength, start, length, longOrds, singleValuePerDoc); } @Override @@ -94,7 +97,7 @@ public PointReader getSharedReader(long start, long length, List toCl if (sharedReader == null) { assert start == 0; assert length <= count; - sharedReader = new OfflinePointReader(tempDir, out.getName(), packedBytesLength, 0, count, longOrds, singleValuePerDoc); + sharedReader = new OfflinePointReader(tempDir, name, packedBytesLength, 0, count, longOrds, singleValuePerDoc); toCloseHeroically.add(sharedReader); // Make sure the OfflinePointReader intends to verify its checksum: assert sharedReader.in instanceof ChecksumIndexInput; @@ -126,11 +129,11 @@ public void destroy() throws IOException { sharedReader.close(); sharedReader = null; } - tempDir.deleteFile(out.getName()); + tempDir.deleteFile(name); } @Override public String toString() { - return "OfflinePointWriter(count=" + count + " tempFileName=" + out.getName() + ")"; + return "OfflinePointWriter(count=" + count + " tempFileName=" + name + ")"; } } diff --git a/lucene/core/src/test/org/apache/lucene/util/TestFixedLengthBytesRefArray.java b/lucene/core/src/test/org/apache/lucene/util/TestFixedLengthBytesRefArray.java new file mode 100644 index 000000000000..586d7d3c0354 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestFixedLengthBytesRefArray.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util; + +import java.util.Arrays; +import java.util.Comparator; + +public class TestFixedLengthBytesRefArray extends LuceneTestCase { + + public void testBasic() throws Exception { + FixedLengthBytesRefArray a = new FixedLengthBytesRefArray(Integer.BYTES); + int numValues = 100; + for(int i=0;i() { + @Override + public int compare(BytesRef a, BytesRef b) { + return a.compareTo(b); + } + }); + + BytesRef last = null; + + int count = 0; + while (true) { + BytesRef bytes = iterator.next(); + if (bytes == null) { + break; + } + if (last != null) { + assertTrue("count=" + count + " last=" + last + " bytes=" + bytes, last.compareTo(bytes) < 0); + } + last = BytesRef.deepCopyOf(bytes); + count++; + } + + assertEquals(numValues, count); + } + + public void testRandom() throws Exception { + int length = TestUtil.nextInt(random(), 4, 10); + int count = atLeast(10000); + BytesRef[] values = new BytesRef[count]; + + FixedLengthBytesRefArray a = new FixedLengthBytesRefArray(length); + for(int i=0;i() { + @Override + public int compare(BytesRef a, BytesRef b) { + return a.compareTo(b); + } + }); + for(int i=0;i 10); } } @@ -85,7 +85,7 @@ public void testIntermediateMerges() throws Exception { public void testSmallRandom() throws Exception { // Sort 20 mb worth of data with 1mb buffer. try (Directory dir = newDirectory()) { - SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES), + SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES, -1), generateRandom((int)OfflineSorter.MB * 20)); assertEquals(3, sortInfo.mergeRounds); } @@ -95,7 +95,7 @@ public void testSmallRandom() throws Exception { public void testLargerRandom() throws Exception { // Sort 100MB worth of data with 15mb buffer. try (Directory dir = newFSDirectory(createTempDir())) { - checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.MAX_TEMPFILES), + checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.MAX_TEMPFILES, -1), generateRandom((int)OfflineSorter.MB * 100)); } } @@ -367,7 +367,7 @@ public IndexOutput createTempOutput(String prefix, String suffix, IOContext cont writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3))); CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> { - new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10).sort(unsorted.getName()); + new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1).sort(unsorted.getName()); }); assertTrue(e.getMessage().contains("checksum failed (hardware problem?)")); } @@ -420,11 +420,67 @@ protected void corruptFile() throws IOException { writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3))); EOFException e = expectThrows(EOFException.class, () -> { - new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10).sort(unsorted.getName()); + new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1).sort(unsorted.getName()); }); assertEquals(1, e.getSuppressed().length); assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException); assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)")); } } + + public void testFixedLengthHeap() throws Exception { + // Make sure the RAM accounting is correct, i.e. if we are sorting fixed width + // ints (4 bytes) then the heap used is really only 4 bytes per value: + Directory dir = newDirectory(); + IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT); + try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) { + byte[] bytes = new byte[Integer.BYTES]; + for (int i=0;i<1024*1024;i++) { + random().nextBytes(bytes); + w.write(bytes); + } + CodecUtil.writeFooter(out); + } + + OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Integer.BYTES); + sorter.sort(out.getName()); + // 1 MB of ints with 4 MH heap allowed should have been sorted in a single heap partition: + assertEquals(0, sorter.sortInfo.mergeRounds); + dir.close(); + } + + public void testFixedLengthLiesLiesLies() throws Exception { + // Make sure OfflineSorter catches me if I lie about the fixed value length: + Directory dir = newDirectory(); + IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT); + try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) { + byte[] bytes = new byte[Integer.BYTES]; + random().nextBytes(bytes); + w.write(bytes); + CodecUtil.writeFooter(out); + } + + OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Long.BYTES); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + sorter.sort(out.getName()); + }); + assertEquals("value length is 4 but is supposed to always be 8", e.getMessage()); + dir.close(); + } + + public void testInvalidFixedLength() throws Exception { + IllegalArgumentException e; + e = expectThrows(IllegalArgumentException.class, + () -> { + new OfflineSorter(null, "foo", OfflineSorter.DEFAULT_COMPARATOR, + BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES, 0); + }); + assertEquals("valueLength must be 1 .. 32767; got: 0", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, + () -> { + new OfflineSorter(null, "foo", OfflineSorter.DEFAULT_COMPARATOR, + BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES, Integer.MAX_VALUE); + }); + assertEquals("valueLength must be 1 .. 32767; got: 2147483647", e.getMessage()); + } } From f9a4deb7be623e31b6da39955483bb2ca494aae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 29 Mar 2016 00:31:40 +0200 Subject: [PATCH 0242/1113] SOLR-8870: AngularJS support for qt style handlers, and fix slash encoding bug to support Query panel through proxy (cherry picked from commit aec11eb) --- solr/CHANGES.txt | 3 +++ solr/webapp/web/js/angular/controllers/query.js | 9 +++++++-- solr/webapp/web/js/angular/services.js | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 48cacf2410d3..11e2e9ef5bbb 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -537,6 +537,9 @@ Bug Fixes hasn't refreshed yet). In this case the reported size of the file is -1. (Shai Erera, Alexey Serba, Richard Coggins) +* SOLR-8870: AngularJS Query tab no longer URL-encodes the /select part of the request, fixing possible 404 issue + when Solr is behind a proxy. Also, now supports old-style &qt param when handler not prefixed with "/" (janhoy) + ======================= 5.5.0 ======================= Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/webapp/web/js/angular/controllers/query.js b/solr/webapp/web/js/angular/controllers/query.js index 3f0fb27b47a9..8e89778acd12 100644 --- a/solr/webapp/web/js/angular/controllers/query.js +++ b/solr/webapp/web/js/angular/controllers/query.js @@ -67,14 +67,19 @@ solrAdminApp.controller('QueryController', } } - var qt = $scope.qt ? $scope.qt : "select"; + var qt = $scope.qt ? $scope.qt : "/select"; for (var filter in $scope.filters) { copy(params, $scope.filters[filter]); } params.core = $routeParams.core; - params.handler = qt; + if (qt[0] == '/') { + params.handler = qt.substring(1); + } else { // Support legacy style handleSelect=true configs + params.handler = "select"; + set("qt", qt); + } var url = Query.url(params); Query.query(params, function(data) { $scope.lang = $scope.query.wt; diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js index 13d01f7204cd..014d36ba525d 100644 --- a/solr/webapp/web/js/angular/services.js +++ b/solr/webapp/web/js/angular/services.js @@ -212,7 +212,7 @@ solrAdminServices.factory('System', }]) .factory('Query', ['$resource', function($resource) { - var resource = $resource('/solr/:core:handler', {core: '@core', handler: '@handler', '_':Date.now()}, { + var resource = $resource('/solr/:core/:handler', {core: '@core', handler: '@handler', '_':Date.now()}, { "query": { method: "GET", transformResponse: function (data) { @@ -230,7 +230,7 @@ solrAdminServices.factory('System', } } } - return "/solr/" + params.core + params.handler + "?" + qs.sort().join("&"); + return "/solr/" + params.core + "/" + params.handler + "?" + qs.sort().join("&"); } return resource; }]) From 28a51cef56dfc43a935590e57c7cbdc8f84a30ef Mon Sep 17 00:00:00 2001 From: markrmiller Date: Wed, 30 Mar 2016 12:33:14 -0400 Subject: [PATCH 0243/1113] SOLR-8864: TestTestInjection needs to cleanup after itself. --- solr/core/src/test/org/apache/solr/util/TestTestInjection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/util/TestTestInjection.java b/solr/core/src/test/org/apache/solr/util/TestTestInjection.java index a2893969130c..418b4a48ee76 100644 --- a/solr/core/src/test/org/apache/solr/util/TestTestInjection.java +++ b/solr/core/src/test/org/apache/solr/util/TestTestInjection.java @@ -31,7 +31,7 @@ public static void beforeClass() { @AfterClass public static void cleanup() { - + TestInjection.reset(); } public void testBasics() { From 72f5eac2c5e7fb743f166fb3c1b25e73078ebdbe Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Mar 2016 13:42:42 -0400 Subject: [PATCH 0244/1113] SOLR-8904: switch from SimpleDateFormat to Instant.parse and format. [value] and ms() and contrib/analytics now call DateMathParser to parse. DateFormatUtil is now removed. (cherry picked from commit 94c0423) (cherry picked from commit 39932f5) --- solr/CHANGES.txt | 8 + .../accumulator/BasicAccumulator.java | 6 +- .../accumulator/FacetingAccumulator.java | 8 +- .../expression/ExpressionFactory.java | 9 +- .../StatsCollectorSupplierFactory.java | 21 +- .../solr/analytics/util/AnalyticsParsers.java | 7 +- .../util/RangeEndpointCalculator.java | 5 +- .../util/valuesource/ConstDateSource.java | 9 +- .../util/valuesource/DateFieldSource.java | 6 +- .../util/valuesource/FilterFieldSource.java | 3 +- .../analytics/expression/ExpressionTest.java | 27 +- .../apache/solr/core/SolrDeletionPolicy.java | 16 +- .../solr/handler/CdcrReplicatorState.java | 21 +- .../solr/handler/PingRequestHandler.java | 6 +- .../handler/component/PivotFacetValue.java | 3 +- .../handler/component/RangeFacetRequest.java | 50 +--- .../solr/response/CSVResponseWriter.java | 10 +- .../solr/response/TextResponseWriter.java | 3 +- .../transform/ValueAugmenterFactory.java | 4 +- .../org/apache/solr/rest/ManagedResource.java | 24 +- .../apache/solr/schema/DateRangeField.java | 15 +- .../org/apache/solr/schema/TrieDateField.java | 5 +- .../org/apache/solr/schema/TrieField.java | 32 ++- .../apache/solr/search/ValueSourceParser.java | 37 ++- .../apache/solr/search/facet/FacetRange.java | 5 +- .../DocExpirationUpdateProcessorFactory.java | 42 ++- .../org/apache/solr/util/DateFormatUtil.java | 245 ------------------ .../org/apache/solr/util/DateMathParser.java | 88 ++++++- .../apache/solr/BasicFunctionalityTest.java | 29 ++- .../src/test/org/apache/solr/TestTrie.java | 13 +- .../cloud/SegmentTerminateEarlyTestState.java | 6 +- .../solr/cloud/TestCloudPivotFacet.java | 23 +- .../DistributedFacetPivotLargeTest.java | 3 +- .../apache/solr/request/SimpleFacetsTest.java | 9 +- .../solr/response/TestCSVResponseWriter.java | 14 +- .../org/apache/solr/schema/DateFieldTest.java | 148 +---------- .../solr/schema/TestUseDocValuesAsStored.java | 9 +- .../update/processor/AtomicUpdatesTest.java | 48 ++-- .../apache/solr/util/DateMathParserTest.java | 125 ++++++++- .../apache/solr/client/solrj/SolrQuery.java | 21 +- .../client/solrj/impl/XMLResponseParser.java | 27 +- .../solr/client/solrj/util/ClientUtils.java | 20 +- .../solr/client/solrj/SolrQueryTest.java | 18 +- .../solrj/response/QueryResponseTest.java | 7 +- .../solr/BaseDistributedSearchTestCase.java | 54 ++-- .../java/org/apache/solr/SolrTestCaseJ4.java | 16 +- 46 files changed, 516 insertions(+), 789 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/util/DateFormatUtil.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 11e2e9ef5bbb..0397e7aa5910 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -164,6 +164,10 @@ Upgrading from Solr 5.x to prevent future optizations from using the column-stored values over the row-stored values when fields have both stored="true" and docValues="true". +* Formatted date-times from Solr have some differences. If the year is more than 4 digits, there is a leading '+'. + When there is a non-zero number of milliseconds, it is padded with zeros to 3 digits. Negative year (BC) dates are + now possible. Parsing: It is now an error to supply a portion of the date out of its, range, like 67 seconds. + Detailed Change List ---------------------- @@ -517,6 +521,10 @@ Other Changes * SOLR-8810: Implement Connection.setReadOnly, Statement.set/getFetchSize, ResultSet.getType (Kevin Risden) +* SOLR-8904: All dates are formatted via Instant.toString() (ISO-8601); see Solr upgrade notes for differences. Will + now parse (and format) dates with a leading '+' or '-' (BC dates or dates > 4 digit year. + [value] and ms() and contrib/analytics now parse with date math. (David Smiley) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java index 53037c1ad570..bd21209e55d2 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; +import com.google.common.base.Supplier; import org.apache.lucene.index.LeafReaderContext; import org.apache.solr.analytics.expression.Expression; import org.apache.solr.analytics.expression.ExpressionFactory; @@ -36,12 +37,9 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.search.DocSet; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.util.DateFormatUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Supplier; - /** * A BasicAccumulator manages the ValueCounters and Expressions without regard to Facets. */ @@ -143,7 +141,7 @@ public String getResult(String expressionName) { if (expressionName.equals(expressionNames[count])) { Comparable value = expressions[count].getValue(); if (value.getClass().equals(Date.class)) { - return DateFormatUtil.formatExternal((Date)value); + return ((Date)value).toInstant().toString(); } else { return value.toString(); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java index 7cb0c144ea25..03392f0ef1a7 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.TreeMap; +import com.google.common.collect.Iterables; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -60,9 +61,6 @@ import org.apache.solr.search.QParser; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SyntaxError; -import org.apache.solr.util.DateFormatUtil; - -import com.google.common.collect.Iterables; /** * A FacetingAccumulator manages the StatsCollectors and Expressions for facets. @@ -376,7 +374,7 @@ public String getResult(String expressionName, String fieldFacet, String facetVa if (expressionName.equals(expressionNames[count])) { Comparable value = facetExpressions[count].getValue(); if (value.getClass().equals(Date.class)) { - return DateFormatUtil.formatExternal((Date)value); + return ((Date)value).toInstant().toString(); } else { return value.toString(); } @@ -430,7 +428,7 @@ public String getQueryResult(String currentFacet, String expressionName, String if (expressionName.equals(expressionNames[count])) { Comparable value = facetExpressions[count].getValue(); if (value.getClass().equals(Date.class)) { - return DateFormatUtil.formatExternal((Date)value); + return ((Date)value).toInstant().toString(); } else { return value.toString(); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/ExpressionFactory.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/ExpressionFactory.java index 8ad4070d1804..1f2d0e04b252 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/ExpressionFactory.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/ExpressionFactory.java @@ -16,7 +16,6 @@ */ package org.apache.solr.analytics.expression; -import java.text.ParseException; import java.util.ArrayList; import java.util.List; @@ -24,7 +23,7 @@ import org.apache.solr.analytics.util.AnalyticsParams; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.util.DateMathParser; public class ExpressionFactory { @@ -81,11 +80,7 @@ public static Expression create(String expression, StatsCollector[] statsCollect throw new SolrException(ErrorCode.BAD_REQUEST, "The constant "+operands+" cannot be converted into a number.",e); } } else if (topOperation.equals(AnalyticsParams.CONSTANT_DATE)) { - try { - return new ConstantDateExpression(DateFormatUtil.parseDate(operands)); - } catch (ParseException e) { - throw new SolrException(ErrorCode.BAD_REQUEST, "The constant "+operands+" cannot be converted into a date.",e); - } + return new ConstantDateExpression(DateMathParser.parseMath(null, operands)); } else if (topOperation.equals(AnalyticsParams.CONSTANT_STRING)) { operands = expression.substring(paren+1, expression.lastIndexOf(')')); return new ConstantStringExpression(operands); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java index bee7431a69d2..bd9f65d24270 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java @@ -17,16 +17,14 @@ package org.apache.solr.analytics.statistics; import java.lang.invoke.MethodHandles; -import java.text.ParseException; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import com.google.common.base.Supplier; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource; import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; @@ -36,7 +34,6 @@ import org.apache.solr.analytics.expression.ExpressionFactory; import org.apache.solr.analytics.request.ExpressionRequest; import org.apache.solr.analytics.util.AnalyticsParams; -import org.apache.solr.analytics.util.AnalyticsParsers; import org.apache.solr.analytics.util.valuesource.AbsoluteValueDoubleFunction; import org.apache.solr.analytics.util.valuesource.AddDoubleFunction; import org.apache.solr.analytics.util.valuesource.ConcatStringFunction; @@ -67,12 +64,10 @@ import org.apache.solr.schema.TrieFloatField; import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.util.DateMathParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Supplier; - public class StatsCollectorSupplierFactory { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -443,11 +438,7 @@ private static ValueSource buildFilterSource(IndexSchema schema, String expressi throw new SolrException(ErrorCode.BAD_REQUEST,"The filter value "+arguments[1]+" cannot be converted into an integer.",e); } } else if ( src instanceof DateFieldSource || src instanceof MultiDateFunction) { - try { - defaultObject = DateFormatUtil.parseDate(arguments[1]); - } catch (ParseException e) { - throw new SolrException(ErrorCode.BAD_REQUEST,"The filter value "+arguments[1]+" cannot be converted into a date.",e); - } + defaultObject = DateMathParser.parseMath(null, arguments[1]); } else if ( src instanceof LongFieldSource ) { try { defaultObject = new Long(arguments[1]); @@ -578,11 +569,7 @@ private static ValueSource buildDateSource(IndexSchema schema, String expression if (arguments.length!=1) { throw new SolrException(ErrorCode.BAD_REQUEST,"The constant date declaration ["+expressionString+"] does not have exactly 1 argument."); } - try { - return new ConstDateSource(DateFormatUtil.parseDate(operands)); - } catch (ParseException e) { - throw new SolrException(ErrorCode.BAD_REQUEST,"The constant "+operands+" cannot be converted into a date.",e); - } + return new ConstDateSource(DateMathParser.parseMath(null, operands)); } else if (operation.equals(AnalyticsParams.FILTER)) { return buildFilterSource(schema, operands, DATE_TYPE); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java index dad4a5b28d4c..7a7e697d059c 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java @@ -17,8 +17,8 @@ package org.apache.solr.analytics.util; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; -import java.util.Date; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LegacyNumericUtils; @@ -29,7 +29,6 @@ import org.apache.solr.schema.TrieFloatField; import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; -import org.apache.solr.util.DateFormatUtil; /** * Class to hold the parsers used for Solr Analytics. @@ -149,7 +148,7 @@ public String parseNum(long l) { @SuppressWarnings("deprecation") public String parse(BytesRef bytes) throws IOException { try { - return DateFormatUtil.formatExternal(new Date(LegacyNumericUtils.prefixCodedToLong(bytes))); + return Instant.ofEpochMilli(LegacyNumericUtils.prefixCodedToLong(bytes)).toString(); } catch (NumberFormatException e) { throw new IOException("The byte array "+Arrays.toString(bytes.bytes)+" cannot be converted to a date."); } @@ -157,7 +156,7 @@ public String parse(BytesRef bytes) throws IOException { @SuppressWarnings("deprecation") @Override public String parseNum(long l) { - return ""+DateFormatUtil.formatExternal(new Date(l)); + return Instant.ofEpochMilli(l).toString(); } }; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/RangeEndpointCalculator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/RangeEndpointCalculator.java index 6d899e0b9cd0..fa9686d6785f 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/RangeEndpointCalculator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/RangeEndpointCalculator.java @@ -31,7 +31,6 @@ import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; public abstract class RangeEndpointCalculator> { @@ -332,12 +331,12 @@ public DateRangeEndpointCalculator(final RangeFacetRequest request, final Date n @Override public String formatValue(Date val) { - return DateFormatUtil.formatExternal(val); + return val.toInstant().toString(); } @Override protected Date parseVal(String rawval) { - return DateFormatUtil.parseMath(now, rawval); + return DateMathParser.parseMath(now, rawval); } @Override diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConstDateSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConstDateSource.java index dc2e02ca037c..3ce608f69871 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConstDateSource.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/ConstDateSource.java @@ -17,7 +17,7 @@ package org.apache.solr.analytics.util.valuesource; import java.io.IOException; -import java.text.ParseException; +import java.time.Instant; import java.util.Date; import java.util.Map; @@ -27,7 +27,6 @@ import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueDate; import org.apache.solr.analytics.util.AnalyticsParams; -import org.apache.solr.util.DateFormatUtil; /** * ConstDateSource returns a constant date for all documents @@ -35,7 +34,7 @@ public class ConstDateSource extends ConstDoubleSource { public final static String NAME = AnalyticsParams.CONSTANT_DATE; - public ConstDateSource(Date constant) throws ParseException { + public ConstDateSource(Date constant) { super(constant.getTime()); } @@ -46,7 +45,7 @@ public ConstDateSource(Long constant) { @SuppressWarnings("deprecation") @Override public String description() { - return name()+"(" + DateFormatUtil.formatExternal(new Date(getLong())) + ")"; + return name()+"(" + Instant.ofEpochMilli(getLong()) + ")"; } protected String name() { @@ -83,7 +82,7 @@ public Object objectVal(int doc) { @SuppressWarnings("deprecation") @Override public String strVal(int doc) { - return DateFormatUtil.formatExternal(new Date(longVal(doc))); + return Instant.ofEpochMilli(longVal(doc)).toString(); } @Override public boolean boolVal(int doc) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java index 631aca0b63ec..4d66e0025bcc 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/DateFieldSource.java @@ -17,11 +17,12 @@ package org.apache.solr.analytics.util.valuesource; import java.io.IOException; +import java.time.Instant; import java.util.Date; import java.util.Map; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; @@ -31,7 +32,6 @@ import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueDate; -import org.apache.solr.util.DateFormatUtil; /** * Extends {@link LongFieldSource} to have a field source that takes in @@ -53,7 +53,7 @@ public Object longToObject(long val) { @SuppressWarnings("deprecation") public String longToString(long val) { - return DateFormatUtil.formatExternal((Date)longToObject(val)); + return Instant.ofEpochMilli(val).toString(); } @Override diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java index 8eb35d144c36..22f5a5720766 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/util/valuesource/FilterFieldSource.java @@ -25,7 +25,6 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.util.mutable.MutableValue; import org.apache.solr.analytics.util.AnalyticsParams; -import org.apache.solr.util.DateFormatUtil; /** * DefaultIsMissingFieldSource wraps a field source to return missing values @@ -49,7 +48,7 @@ protected String name() { @Override public String description() { if (missValue.getClass().equals(Date.class)) { - return name()+"("+source.description()+","+DateFormatUtil.formatExternal((Date)missValue)+")"; + return name()+"("+source.description()+","+ ((Date)missValue).toInstant() +")"; } else { return name()+"("+source.description()+","+missValue.toString()+")"; } diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java index 25193588f6ac..4a3276b9ce24 100644 --- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/expression/ExpressionTest.java @@ -16,22 +16,22 @@ */ package org.apache.solr.analytics.expression; -import com.google.common.collect.ObjectArrays; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.Scanner; +import com.google.common.collect.ObjectArrays; import org.apache.lucene.util.IOUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.analytics.AbstractAnalyticsStatsTest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; import org.junit.BeforeClass; import org.junit.Test; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Scanner; - public class ExpressionTest extends AbstractAnalyticsStatsTest { private static final String fileName = "/analytics/requestFiles/expressions.txt"; @@ -156,20 +156,19 @@ public void constantNumberTest() throws Exception { assertEquals(getRawResponse(), 10, result, 0.0); } - @SuppressWarnings("deprecation") @Test public void dateMathTest() throws Exception { String math = (String) getStatResult("dmr", "cme", VAL_TYPE.STRING); - DateMathParser date = new DateMathParser(); - date.setNow(DateFormatUtil.parseDate((String) getStatResult("dmr", "median", VAL_TYPE.DATE))); + DateMathParser dateMathParser = new DateMathParser(); + dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "median", VAL_TYPE.DATE)).toEpochMilli())); String dateMath = (String) getStatResult("dmr", "dmme", VAL_TYPE.DATE); - assertEquals(getRawResponse(), DateFormatUtil.parseDate(dateMath), date.parseMath(math)); + assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math)); math = (String) getStatResult("dmr", "cma", VAL_TYPE.STRING); - date = new DateMathParser(); - date.setNow(DateFormatUtil.parseDate((String) getStatResult("dmr", "max", VAL_TYPE.DATE))); + dateMathParser = new DateMathParser(); + dateMathParser.setNow(new Date(Instant.parse((String) getStatResult("dmr", "max", VAL_TYPE.DATE)).toEpochMilli())); dateMath = (String) getStatResult("dmr", "dmma", VAL_TYPE.DATE); - assertEquals(getRawResponse(), DateFormatUtil.parseDate(dateMath), date.parseMath(math)); + assertEquals(getRawResponse(), new Date(Instant.parse(dateMath).toEpochMilli()), dateMathParser.parseMath(math)); } @Test diff --git a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java index 24b0e610d30c..c7a668908caf 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java +++ b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java @@ -15,23 +15,23 @@ * limitations under the License. */ package org.apache.solr.core; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Locale; + import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.Locale; - /** * Standard Solr deletion policy that allows reserving index commit points @@ -174,7 +174,7 @@ private void updateCommits(List commits) { try { if (maxCommitAge != null) { if (maxCommitAgeTimeStamp==-1) { - DateMathParser dmp = new DateMathParser(DateFormatUtil.UTC, Locale.ROOT); + DateMathParser dmp = new DateMathParser(DateMathParser.UTC, Locale.ROOT); maxCommitAgeTimeStamp = dmp.parseMath(maxCommitAge).getTime(); } if (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) < maxCommitAgeTimeStamp) { diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java index d0b1c4a1152d..9e01f117b672 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java @@ -16,17 +16,24 @@ */ package org.apache.solr.handler; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.update.CdcrUpdateLog; import org.apache.solr.update.UpdateLog; -import org.apache.solr.util.DateFormatUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; - /** * The state of the replication with a target cluster. */ @@ -133,7 +140,7 @@ List getLastErrors() { Iterator it = errorsQueue.iterator(); while (it.hasNext()) { ErrorQueueEntry entry = it.next(); - lastErrors.add(new String[]{DateFormatUtil.formatExternal(entry.timestamp), entry.type.toLower()}); + lastErrors.add(new String[]{entry.timestamp.toInstant().toString(), entry.type.toLower()}); } } return lastErrors; @@ -145,7 +152,7 @@ List getLastErrors() { String getTimestampOfLastProcessedOperation() { if (logReader != null && logReader.getLastVersion() != -1) { // Shift back to the right by 20 bits the version number - See VersionInfo#getNewClock - return DateFormatUtil.formatExternal(new Date(logReader.getLastVersion() >> 20)); + return Instant.ofEpochMilli(logReader.getLastVersion() >> 20).toString(); } return ""; } diff --git a/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java index 5df46f5a1a5c..4b72e0f5d6e9 100644 --- a/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.file.Files; -import java.util.Date; +import java.time.Instant; import java.util.Locale; import org.apache.commons.io.FileUtils; @@ -34,7 +34,6 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -308,8 +307,7 @@ protected void handleEnable(boolean enable) throws SolrException { if ( enable ) { try { // write out when the file was created - FileUtils.write(healthcheck, - DateFormatUtil.formatExternal(new Date()), "UTF-8"); + FileUtils.write(healthcheck, Instant.now().toString(), "UTF-8"); } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to write healthcheck flag file", e); diff --git a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java index a1eed07f6798..3280c6c90be1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java +++ b/solr/core/src/java/org/apache/solr/handler/component/PivotFacetValue.java @@ -26,7 +26,6 @@ import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.PivotListEntry; /** @@ -92,7 +91,7 @@ public List getValuePath() { if (null == value) { out.add(null); } else if (value instanceof Date) { - out.add(DateFormatUtil.formatExternal((Date) value)); + out.add(((Date) value).toInstant().toString()); } else { out.add(value.toString()); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java index 92b118a8da01..8c0c3812b451 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RangeFacetRequest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.component; import java.lang.invoke.MethodHandles; - import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; @@ -39,8 +38,6 @@ import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -165,7 +162,7 @@ private RangeEndpointCalculator> createCalculator() { "Unable to range facet on tried field of unexpected type:" + this.facetOn); } } else if (ft instanceof DateRangeField) { - calc = new DateRangeFieldEndpointCalculator(this, null); + calc = new DateRangeEndpointCalculator(this, null); } else { throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, @@ -703,56 +700,19 @@ public DateRangeEndpointCalculator(final RangeFacetRequest rangeFacetRequest, final Date now) { super(rangeFacetRequest); this.now = now; - if (!(field.getType() instanceof TrieDateField)) { - throw new IllegalArgumentException - (TYPE_ERR_MSG); - } - } - - @Override - public String formatValue(Date val) { - return DateFormatUtil.formatExternal(val); - } - - @Override - protected Date parseVal(String rawval) { - return DateFormatUtil.parseMath(now, rawval); - } - - @Override - protected Object parseGap(final String rawval) { - return rawval; - } - - @Override - public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException { - final DateMathParser dmp = new DateMathParser(); - dmp.setNow(value); - return dmp.parseMath(gap); - } - } - - private static class DateRangeFieldEndpointCalculator - extends RangeEndpointCalculator { - private final Date now; - - public DateRangeFieldEndpointCalculator(final RangeFacetRequest rangeFacetRequest, - final Date now) { - super(rangeFacetRequest); - this.now = now; - if (!(field.getType() instanceof DateRangeField)) { - throw new IllegalArgumentException(DateRangeEndpointCalculator.TYPE_ERR_MSG); + if (!(field.getType() instanceof TrieDateField) && !(field.getType() instanceof DateRangeField)) { + throw new IllegalArgumentException(TYPE_ERR_MSG); } } @Override public String formatValue(Date val) { - return DateFormatUtil.formatExternal(val); + return val.toInstant().toString(); } @Override protected Date parseVal(String rawval) { - return ((DateRangeField) field.getType()).parseMath(now, rawval); + return DateMathParser.parseMath(now, rawval); } @Override diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java index ba77821badb3..326220c3dac4 100644 --- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java @@ -29,12 +29,13 @@ import java.util.List; import java.util.Map; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.apache.lucene.index.IndexableField; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.internal.csv.CSVPrinter; import org.apache.solr.internal.csv.CSVStrategy; @@ -46,9 +47,6 @@ import org.apache.solr.search.ReturnFields; import org.apache.solr.util.FastWriter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; - /** * */ @@ -497,9 +495,7 @@ public void writeDouble(String name, String val) throws IOException { @Override public void writeDate(String name, Date val) throws IOException { - StringBuilder sb = new StringBuilder(25); - cal = DateUtil.formatDate(val, cal, sb); - writeDate(name, sb.toString()); + writeDate(name, val.toInstant().toString()); } @Override diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java index 3f9e01071263..bde5759957ea 100644 --- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java @@ -40,7 +40,6 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.FastWriter; /** Base class for text-oriented response writers. @@ -335,7 +334,7 @@ public void writeDouble(String name, double val) throws IOException { public void writeDate(String name, Date val) throws IOException { - writeDate(name, DateFormatUtil.formatExternal(val)); + writeDate(name, val.toInstant().toString()); } diff --git a/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java index 8279780155c0..e4edbdfdb7fb 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java @@ -20,9 +20,9 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.DateMathParser; /** * @@ -48,7 +48,7 @@ public static Object getObjectFrom( String val, String type ) if( "int".equals( type ) ) return Integer.valueOf( val ); if( "double".equals( type ) ) return Double.valueOf( val ); if( "float".equals( type ) ) return Float.valueOf( val ); - if( "date".equals( type ) ) return DateUtil.parseDate(val); + if( "date".equals( type ) ) return DateMathParser.parseMath(null, val ); } catch( Exception ex ) { throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/rest/ManagedResource.java b/solr/core/src/java/org/apache/solr/rest/ManagedResource.java index 6b023643b873..8668c9cd9048 100644 --- a/solr/core/src/java/org/apache/solr/rest/ManagedResource.java +++ b/solr/core/src/java/org/apache/solr/rest/ManagedResource.java @@ -27,7 +27,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.rest.ManagedResourceStorage.StorageIO; @@ -282,16 +281,7 @@ public synchronized void storeManagedData(Object managedData) { * Returns this resource's initialization timestamp. */ public String getInitializedOn() { - if (initializedOn == null) - return null; - - StringBuilder dateBuf = new StringBuilder(); - try { - DateUtil.formatDate(initializedOn, null, dateBuf); - } catch (IOException e) { - // safe to ignore - } - return dateBuf.toString(); + return initializedOn == null ? null : initializedOn.toInstant().toString(); } /** @@ -299,17 +289,7 @@ public String getInitializedOn() { * or null if this resource has not been updated since initialization. */ public String getUpdatedSinceInitialization() { - String dateStr = null; - if (lastUpdateSinceInitialization != null) { - StringBuilder dateBuf = new StringBuilder(); - try { - DateUtil.formatDate(lastUpdateSinceInitialization, null, dateBuf); - dateStr = dateBuf.toString(); - } catch (IOException e) { - // safe to ignore here - } - } - return dateStr; + return lastUpdateSinceInitialization == null ? null : lastUpdateSinceInitialization.toInstant().toString(); } /** diff --git a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java index faf049bb5d5c..f4070d38d300 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.Map; -import org.locationtech.spatial4j.shape.Shape; - import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.NumberRangePrefixTreeStrategy; @@ -37,7 +35,8 @@ import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.search.QParser; import org.apache.solr.search.SyntaxError; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.util.DateMathParser; +import org.locationtech.spatial4j.shape.Shape; /** * A field for indexed dates and date ranges. It's mostly compatible with TrieDateField. @@ -75,7 +74,7 @@ protected String getStoredValue(Shape shape, String shapeStr) { if (unitShape.getLevel() == tree.getMaxLevels()) { //fully precise date. We can be fully compatible with TrieDateField. Date date = tree.toCalendar(unitShape).getTime(); - return DateFormatUtil.formatExternal(date); + return date.toInstant().toString(); } } return (shapeStr == null ? shape.toString() : shapeStr);//we don't normalize ranges here; should we? @@ -100,7 +99,7 @@ private Calendar parseCalendar(String str) { if (str.startsWith("NOW") || str.lastIndexOf('Z') >= 0) { //use Solr standard date format parsing rules. //TODO parse a Calendar instead of a Date, rounded according to DateMath syntax. - Date date = DateFormatUtil.parseMath(null, str); + Date date = DateMathParser.parseMath(null, str); Calendar cal = tree.newCal(); cal.setTime(date); return cal; @@ -115,9 +114,9 @@ private Calendar parseCalendar(String str) { } } - /** For easy compatibility with {@link DateFormatUtil#parseMath(Date, String)}. */ + /** For easy compatibility with {@link DateMathParser#parseMath(Date, String)}. */ public Date parseMath(Date now, String rawval) { - return DateFormatUtil.parseMath(now, rawval); + return DateMathParser.parseMath(now, rawval); } @Override @@ -127,7 +126,7 @@ public String shapeToString(Shape shape) { if (unitShape.getLevel() == tree.getMaxLevels()) { //fully precise date. We can be fully compatible with TrieDateField. Date date = tree.toCalendar(unitShape).getTime(); - return DateFormatUtil.formatExternal(date); + return date.toInstant().toString(); } } return shape.toString();//range shape diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java index ccaf8213a955..209c581f1f6a 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java @@ -19,8 +19,7 @@ import java.util.Date; import org.apache.lucene.index.IndexableField; -import org.apache.solr.update.processor.TimestampUpdateProcessorFactory; //jdoc -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.update.processor.TimestampUpdateProcessorFactory; import org.apache.solr.util.DateMathParser; /** @@ -96,7 +95,7 @@ public Date toObject(IndexableField f) { @Override public Object toNativeType(Object val) { if (val instanceof String) { - return DateFormatUtil.parseMath(null, (String)val); + return DateMathParser.parseMath(null, (String)val); } return super.toNativeType(val); } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index 506d8ad9b9ac..bed087e09be0 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -25,9 +26,9 @@ import java.util.Locale; import java.util.Map; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType.LegacyNumericType; +import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.LegacyFloatField; import org.apache.lucene.document.LegacyIntField; import org.apache.lucene.document.LegacyLongField; @@ -41,14 +42,11 @@ import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanQuery.Builder; import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortedSetSelector; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -63,7 +61,7 @@ import org.apache.solr.search.FunctionRangeQuery; import org.apache.solr.search.QParser; import org.apache.solr.search.function.ValueSourceRangeFilter; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.util.DateMathParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -426,13 +424,13 @@ public Query getRangeQuery(QParser parser, SchemaField field, String min, String case DATE: if (matchOnly) { query = DocValuesRangeQuery.newLongRange(field.getName(), - min == null ? null : DateFormatUtil.parseMath(null, min).getTime(), - max == null ? null : DateFormatUtil.parseMath(null, max).getTime(), + min == null ? null : DateMathParser.parseMath(null, min).getTime(), + max == null ? null : DateMathParser.parseMath(null, max).getTime(), minInclusive, maxInclusive); } else { query = LegacyNumericRangeQuery.newLongRange(field.getName(), ps, - min == null ? null : DateFormatUtil.parseMath(null, min).getTime(), - max == null ? null : DateFormatUtil.parseMath(null, max).getTime(), + min == null ? null : DateMathParser.parseMath(null, min).getTime(), + max == null ? null : DateMathParser.parseMath(null, max).getTime(), minInclusive, maxInclusive); } break; @@ -530,7 +528,7 @@ public void readableToIndexed(CharSequence val, BytesRefBuilder result) { LegacyNumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(s)), 0, result); break; case DATE: - LegacyNumericUtils.longToPrefixCoded(DateFormatUtil.parseMath(null, s).getTime(), 0, result); + LegacyNumericUtils.longToPrefixCoded(DateMathParser.parseMath(null, s).getTime(), 0, result); break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); @@ -554,7 +552,7 @@ static String badFieldString(IndexableField f) { @Override public String toExternal(IndexableField f) { return (type == TrieTypes.DATE) - ? DateFormatUtil.formatExternal((Date) toObject(f)) + ? ((Date) toObject(f)).toInstant().toString() : toObject(f).toString(); } @@ -571,7 +569,7 @@ public String indexedToReadable(String _indexedForm) { case DOUBLE: return Double.toString( NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(indexedForm)) ); case DATE: - return DateFormatUtil.formatExternal(new Date(LegacyNumericUtils.prefixCodedToLong(indexedForm))); + return Instant.ofEpochMilli(LegacyNumericUtils.prefixCodedToLong(indexedForm)).toString(); default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); } @@ -594,7 +592,7 @@ public CharsRef indexedToReadable(BytesRef indexedForm, CharsRefBuilder charsRef value = Double.toString( NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(indexedForm)) ); break; case DATE: - value = DateFormatUtil.formatExternal(new Date(LegacyNumericUtils.prefixCodedToLong(indexedForm))); + value = Instant.ofEpochMilli(LegacyNumericUtils.prefixCodedToLong(indexedForm)).toString(); break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); @@ -725,7 +723,7 @@ public IndexableField createField(SchemaField field, Object value, float boost) case DATE: Date date = (value instanceof Date) ? ((Date)value) - : DateFormatUtil.parseMath(null, value.toString()); + : DateMathParser.parseMath(null, value.toString()); f = new LegacyLongField(field.getName(), date.getTime(), ft); break; default: @@ -830,12 +828,12 @@ public Date longToObject(long val) { @Override public String longToString(long val) { - return DateFormatUtil.formatExternal(longToObject(val)); + return longToObject(val).toInstant().toString(); } @Override public long externalToLong(String extVal) { - return DateFormatUtil.parseMath(null, extVal).getTime(); + return DateMathParser.parseMath(null, extVal).getTime(); } } diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java index 0830267652aa..aa94a2e44f40 100644 --- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java @@ -16,7 +16,14 @@ */ package org.apache.solr.search; -import org.locationtech.spatial4j.distance.DistanceUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; @@ -27,8 +34,8 @@ import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.*; -import org.apache.lucene.search.Query; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.spell.JaroWinklerDistance; @@ -39,7 +46,11 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.schema.*; +import org.apache.solr.schema.CurrencyField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.StrField; +import org.apache.solr.schema.TextField; import org.apache.solr.search.facet.AggValueSource; import org.apache.solr.search.facet.AvgAgg; import org.apache.solr.search.facet.CountAgg; @@ -53,12 +64,16 @@ import org.apache.solr.search.function.CollapseScoreFunction; import org.apache.solr.search.function.OrdFieldSource; import org.apache.solr.search.function.ReverseOrdFieldSource; -import org.apache.solr.search.function.distance.*; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.search.function.distance.GeoDistValueSourceParser; +import org.apache.solr.search.function.distance.GeohashFunction; +import org.apache.solr.search.function.distance.GeohashHaversineFunction; +import org.apache.solr.search.function.distance.HaversineFunction; +import org.apache.solr.search.function.distance.SquaredEuclideanFunction; +import org.apache.solr.search.function.distance.StringDistanceFunction; +import org.apache.solr.search.function.distance.VectorDistanceFunction; +import org.apache.solr.util.DateMathParser; import org.apache.solr.util.plugin.NamedListInitializedPlugin; - -import java.io.IOException; -import java.util.*; +import org.locationtech.spatial4j.distance.DistanceUtils; /** * A factory that parses user queries to generate ValueSource instances. @@ -993,8 +1008,10 @@ public void init(NamedList args) { public Date getDate(FunctionQParser fp, String arg) { if (arg == null) return null; - if (arg.startsWith("NOW") || (arg.length() > 0 && Character.isDigit(arg.charAt(0)))) { - return DateFormatUtil.parseMathLenient(null, arg, fp.req); + // check character index 1 to be a digit. Index 0 might be a +/-. + if (arg.startsWith("NOW") || (arg.length() > 1 && Character.isDigit(arg.charAt(1)))) { + Date now = null;//TODO pull from params? + return DateMathParser.parseMath(now, arg); } return null; } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java index b5abf27c4e2f..8d3d0f5b9a88 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRange.java @@ -34,7 +34,6 @@ import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; import org.apache.solr.search.DocSet; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.DateMathParser; public class FacetRange extends FacetRequest { @@ -546,11 +545,11 @@ public Comparable bitsToValue(long bits) { @Override public String formatValue(Comparable val) { - return DateFormatUtil.formatExternal( (Date)val ); + return ((Date)val).toInstant().toString(); } @Override protected Date parseStr(String rawval) { - return DateFormatUtil.parseMath(now, rawval); + return DateMathParser.parseMath(now, rawval); } @Override protected Object parseGap(final String rawval) { diff --git a/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java index 4608344dd0f1..b8a4b5785bcf 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java @@ -18,46 +18,42 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; - -import static org.apache.solr.common.SolrException.ErrorCode.*; - -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CloseHook; import org.apache.solr.core.SolrCore; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.cloud.CloudDescriptor; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.update.AddUpdateCommand; import org.apache.solr.update.CommitUpdateCommand; import org.apache.solr.update.DeleteUpdateCommand; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.plugin.SolrCoreAware; - -import java.text.ParseException; -import java.util.Comparator; -import java.util.ArrayList; -import java.util.List; -import java.util.Collections; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; + /** *

    * Update Processor Factory for managing automatic "expiration" of documents. @@ -404,7 +400,7 @@ public void run() { try { DeleteUpdateCommand del = new DeleteUpdateCommand(req); del.setQuery("{!cache=false}" + expireField + ":[* TO " + - DateFormatUtil.formatExternal(SolrRequestInfo.getRequestInfo().getNOW()) + SolrRequestInfo.getRequestInfo().getNOW().toInstant() + "]"); proc.processDelete(del); diff --git a/solr/core/src/java/org/apache/solr/util/DateFormatUtil.java b/solr/core/src/java/org/apache/solr/util/DateFormatUtil.java deleted file mode 100644 index b4e33be60ecd..000000000000 --- a/solr/core/src/java/org/apache/solr/util/DateFormatUtil.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.util; - -import java.text.DateFormat; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.text.FieldPosition; -import java.text.NumberFormat; -import java.text.ParseException; -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; -import java.util.TimeZone; - -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.DateUtil; -import org.apache.solr.request.SolrQueryRequest; - -public final class DateFormatUtil { - - public static final TimeZone UTC = TimeZone.getTimeZone("UTC"); - /** - * Fixed TimeZone (UTC) needed for parsing/formatting Dates in the - * canonical representation. - */ - public static final TimeZone CANONICAL_TZ = UTC; - /** - * Fixed Locale needed for parsing/formatting Milliseconds in the - * canonical representation. - */ - public static final Locale CANONICAL_LOCALE = Locale.ROOT; - public static final String NOW = "NOW"; - public static final char Z = 'Z'; - - private static final ISO8601CanonicalDateFormat FORMAT_PROTOTYPE = new ISO8601CanonicalDateFormat(); - - /** - * Thread safe DateFormat that can format in the canonical - * ISO8601 date format, not including the trailing "Z" (since it is - * left off in the internal indexed values) - */ - public final static ThreadLocal FORMAT_THREAD_LOCAL = ThreadLocal.withInitial(FORMAT_PROTOTYPE::clone); - - private DateFormatUtil() {} - - /** - * Parses a String which may be a date (in the standard format) - * followed by an optional math expression. - * @param now an optional fixed date to use as "NOW" in the DateMathParser - * @param val the string to parse - */ - public static Date parseMath(Date now, String val) { - String math; - final DateMathParser p = new DateMathParser(); - - if (null != now) p.setNow(now); - - if (val.startsWith(NOW)) { - math = val.substring(NOW.length()); - } else { - final int zz = val.indexOf(Z); - if (0 < zz) { - math = val.substring(zz+1); - try { - // p.setNow(toObject(val.substring(0,zz))); - p.setNow(parseDate(val.substring(0,zz+1))); - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date in Date Math String:'" + val + '\'', e); - } - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date String:'" +val+'\''); - } - } - - if (null == math || math.equals("")) { - return p.getNow(); - } - - try { - return p.parseMath(math); - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date Math String:'" +val+'\'',e); - } - } - - /** - * Return the standard human readable form of the date (with trailing 'Z') - */ - public static String formatExternal(Date d) { - return FORMAT_THREAD_LOCAL.get().format(d) + Z; - } - - /** - * Return the standard human readable form of the date - */ - public static String formatDate(Date d) { - return FORMAT_THREAD_LOCAL.get().format(d); - } - - /** - * Thread safe method that can be used to parse a Date - * without the trailing 'Z' - */ - public static Date parseDate(String s) throws ParseException { - return FORMAT_THREAD_LOCAL.get().parse(s); - } - - /** Parse a date string in the standard format, or any supported by DateUtil.parseDate */ - public static Date parseDateLenient(String s, SolrQueryRequest req) throws ParseException { - // request could define timezone in the future - try { - return DateFormatUtil.FORMAT_THREAD_LOCAL.get().parse(s); - } catch (Exception e) { - return DateUtil.parseDate(s); - } - } - - /** - * Parses a String which may be a date - * followed by an optional math expression. - * @param now an optional fixed date to use as "NOW" in the DateMathParser - * @param val the string to parse - */ - public static Date parseMathLenient(Date now, String val, SolrQueryRequest req) { - String math; - final DateMathParser p = new DateMathParser(); - - if (null != now) p.setNow(now); - - if (val.startsWith(DateFormatUtil.NOW)) { - math = val.substring(DateFormatUtil.NOW.length()); - } else { - final int zz = val.indexOf(DateFormatUtil.Z); - if (0 < zz) { - math = val.substring(zz+1); - try { - // p.setNow(toObject(val.substring(0,zz))); - p.setNow(parseDateLenient(val.substring(0,zz+1), req)); - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date in Date Math String: '" + val + '\'', e); - } - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date String: '" +val+'\''); - } - } - - if (null == math || math.equals("")) { - return p.getNow(); - } - - try { - return p.parseMath(math); - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Invalid Date Math String: '" +val+'\'',e); - } - } - - @SuppressWarnings("serial") - static class ISO8601CanonicalDateFormat extends SimpleDateFormat { - - protected NumberFormat millisParser - = NumberFormat.getIntegerInstance(CANONICAL_LOCALE); - - protected NumberFormat millisFormat = new DecimalFormat - (".###", new DecimalFormatSymbols(CANONICAL_LOCALE)); - - public ISO8601CanonicalDateFormat() { - super("yyyy-MM-dd'T'HH:mm:ss", CANONICAL_LOCALE); - this.setTimeZone(CANONICAL_TZ); - } - - @Override - public Date parse(String i, ParsePosition p) { - /* delegate to SimpleDateFormat for easy stuff */ - Date d = super.parse(i, p); - int milliIndex = p.getIndex(); - /* worry about the milliseconds ourselves */ - if (null != d && - -1 == p.getErrorIndex() && - milliIndex + 1 < i.length() && - '.' == i.charAt(milliIndex)) { - p.setIndex(++milliIndex); // NOTE: ++ to chomp '.' - Number millis = millisParser.parse(i, p); - if (-1 == p.getErrorIndex()) { - int endIndex = p.getIndex(); - d = new Date(d.getTime() - + (long)(millis.doubleValue() * Math.pow(10, (3 - endIndex + milliIndex)))); - } - } - return d; - } - - @Override - public StringBuffer format(Date d, StringBuffer toAppendTo, FieldPosition pos) { - /* delegate to SimpleDateFormat for easy stuff */ - super.format(d, toAppendTo, pos); - /* worry about the milliseconds ourselves */ - long millis = d.getTime() % 1000l; - if (0L == millis) { - return toAppendTo; - } - if (millis < 0L) { - // original date was prior to epoch - millis += 1000L; - } - int posBegin = toAppendTo.length(); - toAppendTo.append(millisFormat.format(millis / 1000d)); - if (DateFormat.MILLISECOND_FIELD == pos.getField()) { - pos.setBeginIndex(posBegin); - pos.setEndIndex(toAppendTo.length()); - } - return toAppendTo; - } - - @Override - public DateFormat clone() { - ISO8601CanonicalDateFormat c = (ISO8601CanonicalDateFormat)super.clone(); - c.millisParser = NumberFormat.getIntegerInstance(CANONICAL_LOCALE); - c.millisFormat = new DecimalFormat(".###", new DecimalFormatSymbols(CANONICAL_LOCALE)); - return c; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/util/DateMathParser.java b/solr/core/src/java/org/apache/solr/util/DateMathParser.java index 9e750dcb9a1f..489a5aad8ec7 100644 --- a/solr/core/src/java/org/apache/solr/util/DateMathParser.java +++ b/solr/core/src/java/org/apache/solr/util/DateMathParser.java @@ -16,18 +16,23 @@ */ package org.apache.solr.util; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.common.params.CommonParams; //jdoc - -import java.util.Date; +import java.text.ParseException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.util.Calendar; -import java.util.TimeZone; +import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.HashMap; -import java.text.ParseException; +import java.util.TimeZone; import java.util.regex.Pattern; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.request.SolrRequestInfo; + /** * A Simple Utility class for parsing "math" like strings relating to Dates. * @@ -92,14 +97,22 @@ * @see SolrRequestInfo#getNOW */ public class DateMathParser { - - public static TimeZone UTC = TimeZone.getTimeZone("UTC"); + + public static final TimeZone UTC = TimeZone.getTimeZone("UTC"); /** Default TimeZone for DateMath rounding (UTC) */ public static final TimeZone DEFAULT_MATH_TZ = UTC; + /** Default Locale for DateMath rounding (Locale.ROOT) */ public static final Locale DEFAULT_MATH_LOCALE = Locale.ROOT; + /** + * Differs by {@link DateTimeFormatter#ISO_INSTANT} in that it's lenient. + * @see #parseNoMath(String) + */ + public static final DateTimeFormatter PARSER = new DateTimeFormatterBuilder() + .parseCaseInsensitive().parseLenient().appendInstant().toFormatter(Locale.ROOT); + /** * A mapping from (uppercased) String labels idenyifying time units, * to the corresponding Calendar constant used to set/add/roll that unit @@ -115,6 +128,7 @@ public class DateMathParser { */ public static final Map CALENDAR_UNITS = makeUnitsMap(); + /** @see #CALENDAR_UNITS */ private static Map makeUnitsMap() { @@ -213,7 +227,60 @@ public static void round(Calendar c, String unit) { } - + /** + * Parses a String which may be a date (in the standard ISO-8601 format) + * followed by an optional math expression. + * @param now an optional fixed date to use as "NOW" + * @param val the string to parse + */ + public static Date parseMath(Date now, String val) { + String math; + final DateMathParser p = new DateMathParser(); + + if (null != now) p.setNow(now); + + if (val.startsWith("NOW")) { + math = val.substring("NOW".length()); + } else { + final int zz = val.indexOf('Z'); + if (zz == -1) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Invalid Date String:'" + val + '\''); + } + math = val.substring(zz+1); + try { + p.setNow(parseNoMath(val.substring(0, zz + 1))); + } catch (DateTimeParseException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Invalid Date in Date Math String:'" + val + '\'', e); + } + } + + if (null == math || math.equals("")) { + return p.getNow(); + } + + try { + return p.parseMath(math); + } catch (ParseException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Invalid Date Math String:'" +val+'\'',e); + } + } + + /** + * Parsing Solr dates without DateMath. + * This is the standard/pervasive ISO-8601 UTC format but is configured with some leniency. + * + * Callers should almost always call {@link #parseMath(Date, String)} instead. + * + * @throws DateTimeParseException if it can't parse + */ + private static Date parseNoMath(String val) { + //TODO write the equivalent of a Date::from; avoids Instant -> Date + return new Date(PARSER.parse(val, Instant::from).toEpochMilli()); + } + private TimeZone zone; private Locale loc; private Date now; @@ -227,7 +294,6 @@ public static void round(Calendar c, String unit) { */ public DateMathParser() { this(null, DEFAULT_MATH_LOCALE); - } /** diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 25ea2c53beb7..9d45a0dabc9d 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; @@ -24,8 +26,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -874,18 +874,19 @@ public void testDateMath() { } - public void testDateRoundtrip() { - assertU(adoc("id", "99", "bday", "99-01-01T12:34:56.789Z")); - assertU(commit()); - assertQ("year should be canonicallized to 4 digits", - req("q", "id:99"), - "//date[@name='bday'][.='0099-01-01T12:34:56.789Z']"); - assertU(adoc("id", "99", "bday", "1999-01-01T12:34:56.900Z")); - assertU(commit()); - assertQ("millis should be canonicallized to no trailing zeros", - req("q", "id:99"), - "//date[@name='bday'][.='1999-01-01T12:34:56.9Z']"); - } + // commented after SOLR-8904; both are false +// public void testDateRoundtrip() { +// assertU(adoc("id", "99", "bday", "99-01-01T12:34:56.789Z")); +// assertU(commit()); +// assertQ("year should be canonicallized to 4 digits", +// req("q", "id:99"), +// "//date[@name='bday'][.='0099-01-01T12:34:56.789Z']"); +// assertU(adoc("id", "99", "bday", "1999-01-01T12:34:56.900Z")); +// assertU(commit()); +// assertQ("millis should be canonicallized to no trailing zeros", +// req("q", "id:99"), +// "//date[@name='bday'][.='1999-01-01T12:34:56.9Z']"); +// } @Test public void testPatternReplaceFilter() { diff --git a/solr/core/src/test/org/apache/solr/TestTrie.java b/solr/core/src/test/org/apache/solr/TestTrie.java index 0c1497feb209..07935b209bf8 100644 --- a/solr/core/src/test/org/apache/solr/TestTrie.java +++ b/solr/core/src/test/org/apache/solr/TestTrie.java @@ -16,19 +16,18 @@ */ package org.apache.solr; +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.TimeZone; + import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.TrieField; import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DateFormatUtil; import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; -import java.text.SimpleDateFormat; -import java.util.Locale; -import java.util.TimeZone; - /** * Tests for TrieField functionality * @@ -173,7 +172,7 @@ public void testTrieDateRangeSearch() throws Exception { format.setTimeZone(TimeZone.getTimeZone("UTC")); assertU(delQ("*:*")); - DateMathParser dmp = new DateMathParser(DateFormatUtil.UTC, Locale.ROOT); + DateMathParser dmp = new DateMathParser(DateMathParser.UTC, Locale.ROOT); String largestDate = ""; for (int i = 0; i < 10; i++) { // index 10 days starting with today @@ -222,7 +221,7 @@ public void testTrieFacet_PrecisionStep() throws Exception { // For tdate tests SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT); format.setTimeZone(TimeZone.getTimeZone("UTC")); - DateMathParser dmp = new DateMathParser(DateFormatUtil.UTC, Locale.ROOT); + DateMathParser dmp = new DateMathParser(DateMathParser.UTC, Locale.ROOT); for (int i = 0; i < 10; i++) { long l = Integer.MAX_VALUE + i*1L; diff --git a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java index e7de4d1d663d..199423b5b53d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java +++ b/solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java @@ -17,6 +17,8 @@ package org.apache.solr.cloud; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -54,7 +56,7 @@ void addDocuments(CloudSolrClient cloudSolrClient, final Integer docKey = new Integer(numDocs); SolrInputDocument doc = new SolrInputDocument(); doc.setField(keyField, ""+docKey); - final int MM = TestMiniSolrCloudCluster.random().nextInt(60); + final int MM = TestMiniSolrCloudCluster.random().nextInt(60); // minutes if (minTimestampMM == null || MM <= minTimestampMM.intValue()) { if (minTimestampMM != null && MM < minTimestampMM.intValue()) { minTimestampDocKeys.clear(); @@ -69,7 +71,7 @@ void addDocuments(CloudSolrClient cloudSolrClient, maxTimestampMM = new Integer(MM); maxTimestampDocKeys.add(docKey); } - doc.setField(timestampField, "2016-01-01T00:"+MM+":00Z"); + doc.setField(timestampField, ZonedDateTime.of(2016, 1, 1, 0, MM, 0, 0, ZoneOffset.UTC).toInstant().toString()); doc.setField(oddField, ""+(numDocs % 2)); doc.setField(quadField, ""+(numDocs % 4)+1); cloudSolrClient.add(doc); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java index 833cadd923b9..dc7e0b04c927 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java @@ -16,6 +16,16 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.commons.lang.StringUtils; import org.apache.lucene.util.TestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; @@ -29,22 +39,11 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.StatsParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.DateFormatUtil; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import static org.apache.solr.common.params.FacetParams.FACET; import static org.apache.solr.common.params.FacetParams.FACET_LIMIT; import static org.apache.solr.common.params.FacetParams.FACET_MISSING; @@ -474,7 +473,7 @@ private static String buildFilter(PivotField constraint) { // otherwise, build up a term filter... String prefix = "{!term f=" + constraint.getField() + "}"; if (value instanceof Date) { - return prefix + DateFormatUtil.formatExternal((Date)value); + return prefix + ((Date) value).toInstant(); } else { return prefix + value; } diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java index 176b57f5e0b4..0fa4e8c2e2f5 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotLargeTest.java @@ -834,6 +834,7 @@ private void doTestPivotRanges() throws Exception { rfc = pf.getFacetRanges().get(0).getCounts(); for (RangeFacet.Count c : rfc) { + assertEquals(0, c.getCount()); // no docs in our ranges for this pivot drill down } @@ -927,7 +928,7 @@ private void setupDistributedPivotFacetDocuments() throws Exception{ addPivotDoc(oneShard, "id", getDocNum(), "place_s", "cardiff", "company_t", "microsoft polecat","pay_i",5824,"hiredate_dt", "2012-11-01T12:30:00Z"); addPivotDoc(oneShard, "id", getDocNum(), "place_s", "cardiff", "company_t", "microsoft ","pay_i",6539,"hiredate_dt", "2012-11-01T12:30:00Z"); addPivotDoc(oneShard, "id", getDocNum(), "place_s", "medical staffing network holdings, inc.", "company_t", "microsoft ","pay_i",6539,"hiredate_dt", "2012-11-01T12:30:00Z", "special_s", "xxx"); - addPivotDoc(oneShard, "id", getDocNum(), "place_s", "cardiff", "company_t", "polecat","pay_i",4352,"hiredate_dt", "2012-1-01T12:30:00Z", "special_s", "xxx"); + addPivotDoc(oneShard, "id", getDocNum(), "place_s", "cardiff", "company_t", "polecat","pay_i",4352,"hiredate_dt", "2012-01-01T12:30:00Z", "special_s", "xxx"); addPivotDoc(oneShard, "id", getDocNum(), "place_s", "krakaw", "company_t", "polecat","pay_i",4352,"hiredate_dt", "2012-11-01T12:30:00Z", "special_s", SPECIAL); addPivotDoc(twoShard, "id", getDocNum(), "place_s", "cardiff", "company_t", "microsoft","pay_i",12,"hiredate_dt", "2012-11-01T12:30:00Z", "special_s", SPECIAL); diff --git a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java index 3b99eb88692d..1bb860e20a0d 100644 --- a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java +++ b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java @@ -36,7 +36,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.SchemaField; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.TimeZoneUtils; import org.junit.BeforeClass; import org.junit.Ignore; @@ -790,8 +789,8 @@ public static void indexDateFacets() { //note: add_doc duplicates bday to bday_drf and a_tdt to a_drf (date range field) add_doc(i, "201", f, "1976-07-04T12:08:56.235Z", ff, "1900-01-01T"+ooo); add_doc(i, "202", f, "1976-07-05T00:00:00.000Z", ff, "1976-07-01T"+ooo); - add_doc(i, "203", f, "1976-07-15T00:07:67.890Z", ff, "1976-07-04T"+ooo); - add_doc(i, "204", f, "1976-07-21T00:07:67.890Z", ff, "1976-07-05T"+ooo); + add_doc(i, "203", f, "1976-07-15T00:07:57.890Z", ff, "1976-07-04T"+ooo); + add_doc(i, "204", f, "1976-07-21T00:07:57.890Z", ff, "1976-07-05T"+ooo); add_doc(i, "205", f, "1976-07-13T12:12:25.255Z", ff, "1976-07-05T"+xxx); add_doc(i, "206", f, "1976-07-03T17:01:23.456Z", ff, "1976-07-07T"+ooo); add_doc(i, "207", f, "1976-07-12T12:12:25.255Z", ff, "1976-07-13T"+ooo); @@ -3006,8 +3005,8 @@ private ModifiableSolrParams getRandomParamsDate() { } int gapNum = random().nextInt(100) + 1; - params.add(FacetParams.FACET_RANGE_START, DateFormatUtil.formatExternal(dates[0])); - params.add(FacetParams.FACET_RANGE_END, DateFormatUtil.formatExternal(dates[1])); + params.add(FacetParams.FACET_RANGE_START, dates[0].toInstant().toString()); + params.add(FacetParams.FACET_RANGE_END, dates[1].toInstant().toString()); params.add(FacetParams.FACET_RANGE_GAP, String.format(Locale.ROOT, "+%d%s", gapNum, gapUnit)); addCommonRandomRangeParams(params); params.add(FacetParams.FACET_RANGE, field); diff --git a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java index c9b59d64fb6e..d10ea71fbdb5 100644 --- a/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java @@ -16,16 +16,18 @@ */ package org.apache.solr.response; +import java.io.StringWriter; +import java.time.Instant; +import java.util.Arrays; +import java.util.Date; + import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.SolrReturnFields; -import org.junit.*; - -import java.io.StringWriter; -import java.util.Arrays; +import org.junit.BeforeClass; +import org.junit.Test; public class TestCSVResponseWriter extends SolrTestCaseJ4 { @BeforeClass @@ -123,7 +125,7 @@ public void testCSVOutput() throws Exception { d.addField("foo_b",false); d.addField("foo_f",1.414f); d.addField("foo_d",-1.0E300); - d.addField("foo_dt", DateUtil.parseDate("2000-01-02T03:04:05Z")); + d.addField("foo_dt", new Date(Instant.parse("2000-01-02T03:04:05Z").toEpochMilli())); d.addField("score", "2.718"); d = new SolrDocument(); diff --git a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java index 8b6ad1d8a3c2..9e88b6e7cf65 100644 --- a/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/DateFieldTest.java @@ -18,24 +18,18 @@ import java.io.File; import java.nio.file.Paths; -import java.text.ParseException; import java.util.Collections; import java.util.Date; -import java.util.Locale; import org.apache.lucene.index.IndexableField; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.DateFormatUtil; -import org.apache.solr.util.DateMathParser; -import org.junit.Ignore; public class DateFieldTest extends SolrTestCaseJ4 { private final String testInstanceDir = TEST_HOME() + File.separator + "collection1"; private final String testConfHome = testInstanceDir + File.separator + "conf"+ File.separator; private TrieDateField f = null; - private DateMathParser p = new DateMathParser(DateFormatUtil.UTC, Locale.ROOT); @Override public void setUp() throws Exception { @@ -50,142 +44,20 @@ public void setUp() throws Exception { f.init(schema, Collections.emptyMap()); } - public void assertFormatParsed(String expected, String input) throws ParseException { - assertEquals("Input: " + input, expected, DateFormatUtil.formatDate(DateFormatUtil.parseMath(new Date(), input))); - } - - public void assertFormatDate(String expected, long input) { - assertEquals("Input: " + input, expected, DateFormatUtil.formatDate(new Date(input))); - } - - public void testToInternal() throws Exception { - assertFormatParsed("1995-12-31T23:59:59.999", "1995-12-31T23:59:59.999666Z"); - assertFormatParsed("1995-12-31T23:59:59.999", "1995-12-31T23:59:59.999Z"); - assertFormatParsed("1995-12-31T23:59:59.99", "1995-12-31T23:59:59.99Z"); - assertFormatParsed("1995-12-31T23:59:59.9", "1995-12-31T23:59:59.9Z"); - assertFormatParsed("1995-12-31T23:59:59", "1995-12-31T23:59:59Z"); - - // here the input isn't in the canonical form, but we should be forgiving - assertFormatParsed("1995-12-31T23:59:59.99", "1995-12-31T23:59:59.990Z"); - assertFormatParsed("1995-12-31T23:59:59.9", "1995-12-31T23:59:59.900Z"); - assertFormatParsed("1995-12-31T23:59:59.9", "1995-12-31T23:59:59.90Z"); - assertFormatParsed("1995-12-31T23:59:59", "1995-12-31T23:59:59.000Z"); - assertFormatParsed("1995-12-31T23:59:59", "1995-12-31T23:59:59.00Z"); - assertFormatParsed("1995-12-31T23:59:59", "1995-12-31T23:59:59.0Z"); - - // kind of kludgy, but we have other tests for the actual date math - assertFormatParsed(DateFormatUtil.formatDate(p.parseMath("/DAY")), "NOW/DAY"); - - // as of Solr 1.3 - assertFormatParsed("1995-12-31T00:00:00", "1995-12-31T23:59:59Z/DAY"); - assertFormatParsed("1995-12-31T00:00:00", "1995-12-31T23:59:59.123Z/DAY"); - assertFormatParsed("1995-12-31T00:00:00", "1995-12-31T23:59:59.123999Z/DAY"); - } - - public void testToInternalObj() throws Exception { - assertFormatDate("1995-12-31T23:59:59.999", 820454399999l); - assertFormatDate("1995-12-31T23:59:59.99", 820454399990l); - assertFormatDate("1995-12-31T23:59:59.9", 820454399900l); - assertFormatDate("1995-12-31T23:59:59", 820454399000l); - } - - public void assertParseMath(long expected, String input) { - Date d = new Date(0); - assertEquals("Input: "+input, expected, DateFormatUtil.parseMath(d, input).getTime()); - } - - // as of Solr1.3 - public void testParseMath() { - assertParseMath(820454699999l, "1995-12-31T23:59:59.999765Z+5MINUTES"); - assertParseMath(820454699999l, "1995-12-31T23:59:59.999Z+5MINUTES"); - assertParseMath(820454699990l, "1995-12-31T23:59:59.99Z+5MINUTES"); - assertParseMath(194918400000l, "1976-03-06T03:06:00Z/DAY"); - - // here the input isn't in the canonical form, but we should be forgiving - assertParseMath(820454699990l, "1995-12-31T23:59:59.990Z+5MINUTES"); - assertParseMath(194918400000l, "1976-03-06T03:06:00.0Z/DAY"); - assertParseMath(194918400000l, "1976-03-06T03:06:00.00Z/DAY"); - assertParseMath(194918400000l, "1976-03-06T03:06:00.000Z/DAY"); - } - - public void testFormatter() { - // just after epoch - assertFormat("1970-01-01T00:00:00.005", 5L); - assertFormat("1970-01-01T00:00:00", 0L); - assertFormat("1970-01-01T00:00:00.37", 370L); - assertFormat("1970-01-01T00:00:00.9", 900L); - - // well after epoch - assertFormat("1999-12-31T23:59:59.005", 946684799005L); - assertFormat("1999-12-31T23:59:59", 946684799000L); - assertFormat("1999-12-31T23:59:59.37", 946684799370L); - assertFormat("1999-12-31T23:59:59.9", 946684799900L); - - // waaaay after epoch - assertFormat("12345-12-31T23:59:59.005", 327434918399005L); - assertFormat("12345-12-31T23:59:59", 327434918399000L); - assertFormat("12345-12-31T23:59:59.37", 327434918399370L); - assertFormat("12345-12-31T23:59:59.9", 327434918399900L); - - // well before epoch - assertFormat("0299-12-31T23:59:59", -52700112001000L); - assertFormat("0299-12-31T23:59:59.123", -52700112000877L); - assertFormat("0299-12-31T23:59:59.09", -52700112000910L); - - } - - /** - * Using dates in the canonical format, verify that parsing+formating - * is an identify function - */ - public void testRoundTrip() throws Exception { - - // typical dates, various precision - assertRoundTrip("1995-12-31T23:59:59.987Z"); - assertRoundTrip("1995-12-31T23:59:59.98Z"); - assertRoundTrip("1995-12-31T23:59:59.9Z"); - assertRoundTrip("1995-12-31T23:59:59Z"); - assertRoundTrip("1976-03-06T03:06:00Z"); - - // dates with atypical years - assertRoundTrip("0001-01-01T01:01:01Z"); - assertRoundTrip("12021-12-01T03:03:03Z"); - } - - @Ignore("SOLR-2773: Non-Positive years don't work") - public void testRoundTripNonPositiveYear() throws Exception { - - // :TODO: ambiguity about year zero - // assertRoundTrip("0000-04-04T04:04:04Z"); - - // dates with negative years - assertRoundTrip("-0005-05-05T05:05:05Z"); - assertRoundTrip("-2021-12-01T04:04:04Z"); - assertRoundTrip("-12021-12-01T02:02:02Z"); - - // :TODO: assertFormat and assertToObject some negative years - - } - - protected void assertFormat(final String expected, final long millis) { - assertEquals(expected, DateFormatUtil.formatDate(new Date(millis))); - } - - protected void assertRoundTrip(String canonicalDate) throws Exception { - Date d = DateFormatUtil.parseDate(canonicalDate); - String result = DateFormatUtil.formatExternal(d); - assertEquals("d:" + d.getTime(), canonicalDate, result); - - } - + // NOTE: Many other tests were moved to DateMathParserTest public void testCreateField() { int props = FieldProperties.INDEXED ^ FieldProperties.STORED; SchemaField sf = new SchemaField( "test", f, props, null ); + // String IndexableField out = f.createField(sf, "1995-12-31T23:59:59Z", 1.0f ); - assertEquals(820454399000l, f.toObject( out ).getTime() ); - - out = f.createField(sf, new Date(820454399000l), 1.0f ); - assertEquals(820454399000l, f.toObject( out ).getTime() ); + assertEquals(820454399000L, f.toObject( out ).getTime() ); + // Date obj + out = f.createField(sf, new Date(820454399000L), 1.0f ); + assertEquals(820454399000L, f.toObject( out ).getTime() ); + // Date math + out = f.createField(sf, "1995-12-31T23:59:59.99Z+5MINUTES", 1.0f); + assertEquals(820454699990L, f.toObject( out ).getTime() ); } + } diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java index 73000b03f3eb..48a3f2266bc7 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java +++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java @@ -22,14 +22,12 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import java.io.File; - import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDateTime; import java.time.Month; import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -71,9 +69,9 @@ public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase { private static final Pattern STORED_FIELD_NAME_PATTERN = Pattern.compile("_dv$"); static { - START_RANDOM_EPOCH_MILLIS = LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0) + START_RANDOM_EPOCH_MILLIS = LocalDateTime.of(-11000, Month.JANUARY, 1, 0, 0)// BC .toInstant(ZoneOffset.UTC).toEpochMilli(); - END_RANDOM_EPOCH_MILLIS = LocalDateTime.of(2030, Month.DECEMBER, 31, 23, 59, 59, 999_000_000) + END_RANDOM_EPOCH_MILLIS = LocalDateTime.of(11000, Month.DECEMBER, 31, 23, 59, 59, 999_000_000) // AD, 5 digit year .toInstant(ZoneOffset.UTC).toEpochMilli(); try { DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); @@ -216,8 +214,7 @@ private String[] nextValues(int arity, String valueType) throws Exception { } case "date": { long epochMillis = TestUtil.nextLong(random(), START_RANDOM_EPOCH_MILLIS, END_RANDOM_EPOCH_MILLIS); - LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis), ZoneOffset.UTC); - values[i] = dateTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + 'Z'; + values[i] = Instant.ofEpochMilli(epochMillis).toString(); break; } default: throw new Exception("unknown type '" + valueType + "'"); diff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java index 09fa721eb59e..e1726f82a4b6 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java @@ -24,7 +24,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.schema.TrieDateField; -import org.apache.solr.util.DateFormatUtil; +import org.apache.solr.util.DateMathParser; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; @@ -597,45 +597,45 @@ public void testRemoveDateUsingDateType() throws Exception { doc = new SolrInputDocument(); doc.setField("id", "10001"); TrieDateField trieDF = new TrieDateField(); - Date tempDate = DateFormatUtil.parseMath(null, "2014-02-01T12:00:00Z"); - doc.setField("dateRemove", new Date[]{DateFormatUtil.parseMath(null, "2014-02-01T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-07-02T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-03T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-03T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-04T12:00:00Z") + Date tempDate = DateMathParser.parseMath(null, "2014-02-01T12:00:00Z"); + doc.setField("dateRemove", new Date[]{DateMathParser.parseMath(null, "2014-02-01T12:00:00Z"), + DateMathParser.parseMath(null, "2014-07-02T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-03T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-03T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-04T12:00:00Z") }); assertU(adoc(doc)); doc = new SolrInputDocument(); doc.setField("id", "10002"); - doc.setField("dateRemove", new Date[]{DateFormatUtil.parseMath(null, "2014-02-01T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-07-02T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-02T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-03T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-04T12:00:00Z") + doc.setField("dateRemove", new Date[]{DateMathParser.parseMath(null, "2014-02-01T12:00:00Z"), + DateMathParser.parseMath(null, "2014-07-02T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-02T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-03T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-04T12:00:00Z") }); assertU(adoc(doc)); doc = new SolrInputDocument(); doc.setField("id", "10020"); - doc.setField("dateRemove", new Date[]{DateFormatUtil.parseMath(null, "2014-02-01T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-03T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-04T12:00:00Z") + doc.setField("dateRemove", new Date[]{DateMathParser.parseMath(null, "2014-02-01T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-03T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-04T12:00:00Z") }); assertU(adoc(doc)); doc = new SolrInputDocument(); doc.setField("id", "10021"); - doc.setField("dateRemove", new Date[]{DateFormatUtil.parseMath(null, "2014-02-01T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-02T12:00:00Z"), - DateFormatUtil.parseMath(null, "2014-02-04T12:00:00Z") + doc.setField("dateRemove", new Date[]{DateMathParser.parseMath(null, "2014-02-01T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-02T12:00:00Z"), + DateMathParser.parseMath(null, "2014-02-04T12:00:00Z") }); assertU(adoc(doc)); assertU(commit()); assertQ(req("q", "dateRemove:*", "indent", "true"), "//result[@numFound = '4']"); - String dateString = DateFormatUtil.parseMath(null, "2014-02-02T12:00:00Z").toString(); + String dateString = DateMathParser.parseMath(null, "2014-02-02T12:00:00Z").toString(); // assertQ(req("q", "dateRemove:"+URLEncoder.encode(dateString, "UTF-8"), "indent", "true"), "//result[@numFound = '3']"); // assertQ(req("q", "dateRemove:\"2014-09-02T12:00:00Z\"", "indent", "true"), "//result[@numFound = '3']"); // assertQ(req("q", "dateRemove:"+dateString, "indent", "true"), "//result[@numFound = '3']"); //Sun Feb 02 10:00:00 FNT 2014 @@ -645,8 +645,8 @@ public void testRemoveDateUsingDateType() throws Exception { doc = new SolrInputDocument(); doc.setField("id", "10001"); List removeList = new ArrayList(); - removeList.add(DateFormatUtil.parseMath(null, "2014-09-02T12:00:00Z")); - removeList.add(DateFormatUtil.parseMath(null, "2014-09-03T12:00:00Z")); + removeList.add(DateMathParser.parseMath(null, "2014-09-02T12:00:00Z")); + removeList.add(DateMathParser.parseMath(null, "2014-09-03T12:00:00Z")); doc.setField("dateRemove", ImmutableMap.of("remove", removeList)); //behavior when hitting Solr through ZK assertU(adoc(doc)); @@ -658,8 +658,8 @@ public void testRemoveDateUsingDateType() throws Exception { doc = new SolrInputDocument(); doc.setField("id", "10021"); removeList = new ArrayList(); - removeList.add(DateFormatUtil.parseMath(null, "2014-09-02T12:00:00Z")); - removeList.add(DateFormatUtil.parseMath(null, "2014-09-03T12:00:00Z")); + removeList.add(DateMathParser.parseMath(null, "2014-09-02T12:00:00Z")); + removeList.add(DateMathParser.parseMath(null, "2014-09-03T12:00:00Z")); doc.setField("dateRemove", ImmutableMap.of("remove", removeList)); //behavior when hitting Solr through ZK assertU(adoc(doc)); assertU(commit()); @@ -669,7 +669,7 @@ public void testRemoveDateUsingDateType() throws Exception { doc = new SolrInputDocument(); doc.setField("id", "10001"); - doc.setField("dateRemove", ImmutableMap.of("remove", DateFormatUtil.parseMath(null, "2014-09-01T12:00:00Z"))); //behavior when hitting Solr directly + doc.setField("dateRemove", ImmutableMap.of("remove", DateMathParser.parseMath(null, "2014-09-01T12:00:00Z"))); //behavior when hitting Solr directly assertU(adoc(doc)); assertU(commit()); diff --git a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java index c3623e8919e4..726c71c9617d 100644 --- a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java +++ b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java @@ -16,20 +16,20 @@ */ package org.apache.solr.util; -import static org.apache.solr.util.DateFormatUtil.UTC; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.util.DateMathParser; - -import java.text.SimpleDateFormat; import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.Calendar; import java.util.Date; -import java.util.TimeZone; +import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.HashMap; -import java.text.ParseException; +import java.util.TimeZone; + +import org.apache.lucene.util.LuceneTestCase; + +import static org.apache.solr.util.DateMathParser.UTC; /** * Tests that the functions in DateMathParser @@ -333,6 +333,111 @@ public void testParseMathExceptions() throws Exception { } } - + + /* + PARSING / FORMATTING (without date math) Formerly in DateFieldTest. + */ + + + public void testFormatter() { + assertFormat("1995-12-31T23:59:59.999Z", 820454399999l); + assertFormat("1995-12-31T23:59:59.990Z", 820454399990l); + assertFormat("1995-12-31T23:59:59.900Z", 820454399900l); + assertFormat("1995-12-31T23:59:59Z", 820454399000l); + + // just after epoch + assertFormat("1970-01-01T00:00:00.005Z", 5L); + assertFormat("1970-01-01T00:00:00Z", 0L); + assertFormat("1970-01-01T00:00:00.370Z", 370L); + assertFormat("1970-01-01T00:00:00.900Z", 900L); + + // well after epoch + assertFormat("1999-12-31T23:59:59.005Z", 946684799005L); + assertFormat("1999-12-31T23:59:59Z", 946684799000L); + assertFormat("1999-12-31T23:59:59.370Z", 946684799370L); + assertFormat("1999-12-31T23:59:59.900Z", 946684799900L); + + // waaaay after epoch ('+' is required for more than 4 digits in a year) + assertFormat("+12345-12-31T23:59:59.005Z", 327434918399005L); + assertFormat("+12345-12-31T23:59:59Z", 327434918399000L); + assertFormat("+12345-12-31T23:59:59.370Z", 327434918399370L); + assertFormat("+12345-12-31T23:59:59.900Z", 327434918399900L); + + // well before epoch + assertFormat("0299-12-31T23:59:59Z", -52700112001000L); + assertFormat("0299-12-31T23:59:59.123Z", -52700112000877L); + assertFormat("0299-12-31T23:59:59.090Z", -52700112000910L); + + // BC (negative years) + assertFormat("-12021-12-01T02:02:02Z", Instant.parse("-12021-12-01T02:02:02Z").toEpochMilli()); + } + + private void assertFormat(final String expected, final long millis) { + assertEquals(expected, Instant.ofEpochMilli(millis).toString()); + } + + /** + * Using dates in the canonical format, verify that parsing+formatting + * is an identify function + */ + public void testRoundTrip() throws Exception { + // NOTE: the 2nd arg is what the round trip result looks like (may be null if same as input) + + assertParseFormatEquals("1995-12-31T23:59:59.999666Z", "1995-12-31T23:59:59.999Z"); // beyond millis is truncated + assertParseFormatEquals("1995-12-31T23:59:59.999Z", "1995-12-31T23:59:59.999Z"); + assertParseFormatEquals("1995-12-31T23:59:59.99Z", "1995-12-31T23:59:59.990Z"); + assertParseFormatEquals("1995-12-31T23:59:59.9Z", "1995-12-31T23:59:59.900Z"); + assertParseFormatEquals("1995-12-31T23:59:59Z", "1995-12-31T23:59:59Z"); + + // here the input isn't in the canonical form, but we should be forgiving + assertParseFormatEquals("1995-12-31T23:59:59.990Z", "1995-12-31T23:59:59.990Z"); + assertParseFormatEquals("1995-12-31T23:59:59.900Z", "1995-12-31T23:59:59.900Z"); + assertParseFormatEquals("1995-12-31T23:59:59.90Z", "1995-12-31T23:59:59.900Z"); + assertParseFormatEquals("1995-12-31T23:59:59.000Z", "1995-12-31T23:59:59Z"); + assertParseFormatEquals("1995-12-31T23:59:59.00Z", "1995-12-31T23:59:59Z"); + assertParseFormatEquals("1995-12-31T23:59:59.0Z", "1995-12-31T23:59:59Z"); + + // kind of kludgy, but we have other tests for the actual date math + //assertParseFormatEquals("NOW/DAY", p.parseMath("/DAY").toInstant().toString()); + + // as of Solr 1.3 + assertParseFormatEquals("1995-12-31T23:59:59Z/DAY", "1995-12-31T00:00:00Z"); + assertParseFormatEquals("1995-12-31T23:59:59.123Z/DAY", "1995-12-31T00:00:00Z"); + assertParseFormatEquals("1995-12-31T23:59:59.123999Z/DAY", "1995-12-31T00:00:00Z"); + + // typical dates, various precision (0,1,2,3 digits of millis) + assertParseFormatEquals("1995-12-31T23:59:59.987Z", null); + assertParseFormatEquals("1995-12-31T23:59:59.98Z", "1995-12-31T23:59:59.980Z");//add 0 ms + assertParseFormatEquals("1995-12-31T23:59:59.9Z", "1995-12-31T23:59:59.900Z");//add 00 ms + assertParseFormatEquals("1995-12-31T23:59:59Z", null); + assertParseFormatEquals("1976-03-06T03:06:00Z", null); + assertParseFormatEquals("1995-12-31T23:59:59.987654Z", "1995-12-31T23:59:59.987Z");//truncate nanoseconds off + + // dates with atypical years + assertParseFormatEquals("0001-01-01T01:01:01Z", null); + assertParseFormatEquals("+12021-12-01T03:03:03Z", null); + + assertParseFormatEquals("0000-04-04T04:04:04Z", null); // note: 0 AD is also known as 1 BC + + // dates with negative years (BC) + assertParseFormatEquals("-0005-05-05T05:05:05Z", null); + assertParseFormatEquals("-2021-12-01T04:04:04Z", null); + assertParseFormatEquals("-12021-12-01T02:02:02Z", null); + } + + public void testParseLenient() throws Exception { + // dates that only parse thanks to lenient mode of DateTimeFormatter + assertParseFormatEquals("10995-12-31T23:59:59.990Z", "+10995-12-31T23:59:59.990Z"); // missing '+' 5 digit year + assertParseFormatEquals("995-1-2T3:4:5Z", "0995-01-02T03:04:05Z"); // wasn't 0 padded + } + + private void assertParseFormatEquals(String inputStr, String expectedStr) { + if (expectedStr == null) { + expectedStr = inputStr; + } + Date inputDate = DateMathParser.parseMath(null, inputStr); + String resultStr = inputDate.toInstant().toString(); + assertEquals("d:" + inputDate.getTime(), expectedStr, resultStr); + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java index d41673dbb580..be7021075da8 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrQuery.java @@ -16,14 +16,6 @@ */ package org.apache.solr.client.solrj; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.HighlightParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.StatsParams; -import org.apache.solr.common.params.TermsParams; -import org.apache.solr.common.util.DateUtil; - import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -31,6 +23,13 @@ import java.util.Locale; import java.util.regex.Pattern; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.params.HighlightParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.StatsParams; +import org.apache.solr.common.params.TermsParams; + /** * This is an augmented SolrParams with get/set/add fields for common fields used @@ -266,9 +265,9 @@ public SolrQuery addNumericRangeFacet(String field, Number start, Number end, Nu */ public SolrQuery addDateRangeFacet(String field, Date start, Date end, String gap) { add(FacetParams.FACET_RANGE, field); - add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_START), DateUtil.getThreadLocalDateFormat().format(start)); - add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_END), DateUtil.getThreadLocalDateFormat().format(end)); - add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_GAP), gap); + add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_START), start.toInstant().toString()); + add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_END), end.toInstant().toString()); + add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_GAP), gap); this.set(FacetParams.FACET, true); return this; } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java index e794dab51644..272335ab643f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java @@ -16,31 +16,30 @@ */ package org.apache.solr.client.solrj.impl; -import org.apache.solr.client.solrj.ResponseParser; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.DateUtil; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.XMLErrorLogger; -import org.apache.solr.common.EmptyEntityResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; - import java.io.InputStream; import java.io.Reader; import java.lang.invoke.MethodHandles; +import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.common.EmptyEntityResolver; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.XMLErrorLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * * @@ -173,7 +172,7 @@ protected enum KnownType { @Override public Date read( String txt ) { try { - return DateUtil.parseDate(txt); + return new Date(Instant.parse(txt).toEpochMilli()); } catch( Exception ex ) { ex.printStackTrace(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 31d54a89fa0e..beed40eec9ac 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -16,16 +16,6 @@ */ package org.apache.solr.client.solrj.util; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.util.Base64; -import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.common.util.DateUtil; -import org.apache.solr.common.util.XML; - import java.io.IOException; import java.io.StringWriter; import java.io.Writer; @@ -36,6 +26,14 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.Base64; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.XML; + /** * @@ -111,7 +109,7 @@ public static void writeXML( SolrInputDocument doc, Writer writer ) throws IOExc private static void writeVal(Writer writer, float boost, String name, Object v, String update) throws IOException { if (v instanceof Date) { - v = DateUtil.getThreadLocalDateFormat().format( (Date)v ); + v = ((Date)v).toInstant().toString(); } else if (v instanceof byte[]) { byte[] bytes = (byte[]) v; v = Base64.byteArrayToBase64(bytes, 0, bytes.length); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java index 43fdff4d095a..816a2cca18a4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrQueryTest.java @@ -16,14 +16,6 @@ */ package org.apache.solr.client.solrj; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrQuery.SortClause; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.FacetParams; - -import junit.framework.Assert; -import org.apache.solr.common.util.DateUtil; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; @@ -33,6 +25,12 @@ import java.util.Locale; import java.util.TimeZone; +import junit.framework.Assert; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrQuery.SortClause; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.FacetParams; + /** * * @@ -301,8 +299,8 @@ public void testFacetDateRange() { q.addDateRangeFacet("field", start, end, "+1MONTH"); assertEquals("true", q.get(FacetParams.FACET)); assertEquals("field", q.get(FacetParams.FACET_RANGE)); - assertEquals(DateUtil.getThreadLocalDateFormat().format(start), q.get("f.field." + FacetParams.FACET_RANGE_START)); - assertEquals(DateUtil.getThreadLocalDateFormat().format(end), q.get("f.field." + FacetParams.FACET_RANGE_END)); + assertEquals(start.toInstant().toString(), q.get("f.field." + FacetParams.FACET_RANGE_START)); + assertEquals(end.toInstant().toString(), q.get("f.field." + FacetParams.FACET_RANGE_END)); assertEquals("+1MONTH", q.get("f.field." + FacetParams.FACET_RANGE_GAP)); } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java index 1eea4541914d..7a69815c7bf0 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java @@ -20,6 +20,8 @@ import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Date; import java.util.List; import junit.framework.Assert; @@ -27,7 +29,6 @@ import org.apache.lucene.util.TestRuleLimitSysouts.Limit; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrResourceLoader; import org.junit.Test; @@ -82,8 +83,8 @@ public void testRangeFacets() throws Exception { assertEquals("4.0", price.getCounts().get(4).getValue()); assertEquals(0, price.getCounts().get(4).getCount()); - assertEquals(DateUtil.parseDate("2005-02-13T15:26:37Z"), manufacturedateDt.getStart()); - assertEquals(DateUtil.parseDate("2008-02-13T15:26:37Z"), manufacturedateDt.getEnd()); + assertEquals(new Date(Instant.parse("2005-02-13T15:26:37Z").toEpochMilli()), manufacturedateDt.getStart()); + assertEquals(new Date(Instant.parse("2008-02-13T15:26:37Z").toEpochMilli()), manufacturedateDt.getEnd()); assertEquals("+1YEAR", manufacturedateDt.getGap()); assertEquals("2005-02-13T15:26:37Z", manufacturedateDt.getCounts().get(0).getValue()); assertEquals(4, manufacturedateDt.getCounts().get(0).getCount()); diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index 812663eb4495..a751459e5964 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -16,8 +16,31 @@ */ package org.apache.solr; -import junit.framework.Assert; +import javax.servlet.Filter; +import java.io.File; +import java.io.IOException; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.atomic.AtomicInteger; +import junit.framework.Assert; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.Constants; import org.apache.lucene.util.TestUtil; @@ -26,7 +49,6 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; @@ -37,7 +59,6 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.DateFormatUtil; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -48,31 +69,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.Filter; - -import java.io.File; -import java.io.IOException; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.atomic.AtomicInteger; - /** * Helper base class for distributed search test cases * @@ -1086,7 +1082,7 @@ public static class RandDate extends RandVal { public Object val() { long v = r.nextLong(); Date d = new Date(v); - return DateFormatUtil.formatExternal(d); + return d.toInstant().toString(); } } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 6e511d7e4d65..3ce252f9b4d0 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -37,7 +37,18 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; import java.util.logging.Level; import com.carrotsearch.randomizedtesting.RandomizedContext; @@ -86,7 +97,6 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.servlet.DirectSolrConnection; import org.apache.solr.util.AbstractSolrTestCase; -import org.apache.solr.util.DateFormatUtil; import org.apache.solr.util.RefCounted; import org.apache.solr.util.RevertDefaultThreadHandlerRule; import org.apache.solr.util.SSLTestConfig; @@ -2036,7 +2046,7 @@ public static Object skewed(Object likely, Object unlikely) { * @see #randomSkewedDate */ public static String randomDate() { - return DateFormatUtil.formatExternal(new Date(random().nextLong())); + return Instant.ofEpochMilli(random().nextLong()).toString(); } /** From 44e0ac38567e19465ebf74a160064b8a642ec6b6 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 30 Mar 2016 15:00:29 -0400 Subject: [PATCH 0245/1113] SOLR-8903: Move SolrJ DateUtil to contrib/extraction as ExtractionDateUtil. And removed obsolete methods. (cherry picked from commit 5e5fd66) --- solr/CHANGES.txt | 6 + .../extraction/ExtractingRequestHandler.java | 23 ++-- .../extraction/ExtractionDateUtil.java} | 119 +++--------------- .../extraction/SolrContentHandler.java | 12 +- .../extraction/TestExtractionDateUtil.java | 61 +++++++++ .../solr/morphlines/cell/SolrCellBuilder.java | 18 ++- .../cell/SolrCellMorphlineTest.java | 4 +- .../apache/solr/common/util/TestDateUtil.java | 35 ------ 8 files changed, 111 insertions(+), 167 deletions(-) rename solr/{solrj/src/java/org/apache/solr/common/util/DateUtil.java => contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractionDateUtil.java} (65%) create mode 100644 solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestExtractionDateUtil.java delete mode 100644 solr/solrj/src/test/org/apache/solr/common/util/TestDateUtil.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0397e7aa5910..2c5fc28033d6 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -168,6 +168,9 @@ Upgrading from Solr 5.x When there is a non-zero number of milliseconds, it is padded with zeros to 3 digits. Negative year (BC) dates are now possible. Parsing: It is now an error to supply a portion of the date out of its, range, like 67 seconds. +* SolrJ no longer includes DateUtil. If for some reason you need to format or parse dates, simply use Instant.format() + and Instant.parse(). + Detailed Change List ---------------------- @@ -525,6 +528,9 @@ Other Changes now parse (and format) dates with a leading '+' or '-' (BC dates or dates > 4 digit year. [value] and ms() and contrib/analytics now parse with date math. (David Smiley) +* SOLR-8904: DateUtil in SolrJ moved to the extraction contrib as ExtractionDateUtil. Obsolete methods were removed. + (David Smiley) + ================== 5.5.1 ================== Bug Fixes diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java index af70c623a33b..82fe633aad3c 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java @@ -17,31 +17,30 @@ package org.apache.solr.handler.extraction; +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; + import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.ContentStreamHandlerBase; +import org.apache.solr.handler.loader.ContentStreamLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.solr.handler.ContentStreamHandlerBase; -import org.apache.solr.handler.loader.ContentStreamLoader; import org.apache.tika.config.TikaConfig; import org.apache.tika.mime.MimeTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; - /** * Handler for rich documents like PDF or Word or any other file format that Tika handles that need the text to be extracted @@ -59,7 +58,7 @@ public class ExtractingRequestHandler extends ContentStreamHandlerBase implement protected ParseContextConfig parseContextConfig; - protected Collection dateFormats = DateUtil.DEFAULT_DATE_FORMATS; + protected Collection dateFormats = ExtractionDateUtil.DEFAULT_DATE_FORMATS; protected SolrContentHandlerFactory factory; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/DateUtil.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractionDateUtil.java similarity index 65% rename from solr/solrj/src/java/org/apache/solr/common/util/DateUtil.java rename to solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractionDateUtil.java index 9bf7a2ba11e7..b7ccf8203da4 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/DateUtil.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractionDateUtil.java @@ -14,11 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.common.util; -import java.io.IOException; -import java.text.DateFormat; +package org.apache.solr.handler.extraction; + import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -32,7 +34,7 @@ /** * This class has some code from HttpClient DateUtil. */ -public class DateUtil { +public class ExtractionDateUtil { //start HttpClient /** * Date format pattern used to parse HTTP date headers in RFC 1123 format. @@ -67,6 +69,12 @@ public class DateUtil { //--------------------------------------------------------------------------------------- + /** + * Differs by {@link DateTimeFormatter#ISO_INSTANT} in that it's lenient. + */ + public static final DateTimeFormatter ISO_8601_PARSER = new DateTimeFormatterBuilder() + .parseCaseInsensitive().parseLenient().appendInstant().toFormatter(Locale.ROOT); + /** * A suite of default date formats that can be parsed, and thus transformed to the Solr specific format */ @@ -95,9 +103,12 @@ public static Date parseDate(String d) throws ParseException { } public static Date parseDate(String d, Collection fmts) throws ParseException { - // 2007-04-26T08:05:04Z - if (d.endsWith("Z") && d.length() > 20) { - return getThreadLocalDateFormat().parse(d); + if (d.length() > 0 && d.charAt(d.length() - 1) == 'Z') { + try { + return new Date(ISO_8601_PARSER.parse(d, Instant::from).toEpochMilli()); + } catch (Exception e) { + //ignore; perhaps we can parse with one of the formats below... + } } return parseDate(d, fmts, null); } @@ -140,6 +151,7 @@ public static Date parseDate( dateValue = dateValue.substring(1, dateValue.length() - 1); } + //TODO upgrade to Java 8 DateTimeFormatter. But how to deal with the GMT as a default? SimpleDateFormat dateParser = null; Iterator formatIter = dateFormats.iterator(); @@ -163,97 +175,4 @@ public static Date parseDate( throw new ParseException("Unable to parse the date " + dateValue, 0); } - - /** - * Returns a formatter that can be use by the current thread if needed to - * convert Date objects to the Internal representation. - * - * @return The {@link java.text.DateFormat} for the current thread - */ - public static DateFormat getThreadLocalDateFormat() { - return fmtThreadLocal.get(); - } - - public static TimeZone UTC = TimeZone.getTimeZone("UTC"); - private static ThreadLocalDateFormat fmtThreadLocal = new ThreadLocalDateFormat(); - - private static class ThreadLocalDateFormat extends ThreadLocal { - DateFormat proto; - - public ThreadLocalDateFormat() { - super(); - //2007-04-26T08:05:04Z - SimpleDateFormat tmp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT); - tmp.setTimeZone(UTC); - proto = tmp; - } - - @Override - protected DateFormat initialValue() { - return (DateFormat) proto.clone(); - } - } - - /** Formats the date and returns the calendar instance that was used (which may be reused) */ - public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException { - // using a stringBuilder for numbers can be nice since - // a temporary string isn't used (it's added directly to the - // builder's buffer. - - StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder(); - if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT); - cal.setTime(date); - - int i = cal.get(Calendar.YEAR); - sb.append(i); - sb.append('-'); - i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1 - if (i<10) sb.append('0'); - sb.append(i); - sb.append('-'); - i=cal.get(Calendar.DAY_OF_MONTH); - if (i<10) sb.append('0'); - sb.append(i); - sb.append('T'); - i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format - if (i<10) sb.append('0'); - sb.append(i); - sb.append(':'); - i=cal.get(Calendar.MINUTE); - if (i<10) sb.append('0'); - sb.append(i); - sb.append(':'); - i=cal.get(Calendar.SECOND); - if (i<10) sb.append('0'); - sb.append(i); - i=cal.get(Calendar.MILLISECOND); - if (i != 0) { - sb.append('.'); - if (i<100) sb.append('0'); - if (i<10) sb.append('0'); - sb.append(i); - - // handle canonical format specifying fractional - // seconds shall not end in '0'. Given the slowness of - // integer div/mod, simply checking the last character - // is probably the fastest way to check. - int lastIdx = sb.length()-1; - if (sb.charAt(lastIdx)=='0') { - lastIdx--; - if (sb.charAt(lastIdx)=='0') { - lastIdx--; - } - sb.setLength(lastIdx+1); - } - - } - sb.append('Z'); - - if (out != sb) - out.append(sb); - - return cal; - } - - } \ No newline at end of file diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java index 442e64c1e4ea..77794512f305 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.extraction; import java.lang.invoke.MethodHandles; -import java.text.DateFormat; import java.util.ArrayDeque; import java.util.Collection; import java.util.Collections; @@ -31,7 +30,6 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; @@ -83,7 +81,7 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara private Set literalFieldNames = null; public SolrContentHandler(Metadata metadata, SolrParams params, IndexSchema schema) { - this(metadata, params, schema, DateUtil.DEFAULT_DATE_FORMATS); + this(metadata, params, schema, ExtractionDateUtil.DEFAULT_DATE_FORMATS); } @@ -317,7 +315,7 @@ public void ignorableWhitespace(char[] chars, int offset, int length) throws SAX /** * Can be used to transform input values based on their {@link org.apache.solr.schema.SchemaField} *

    - * This implementation only formats dates using the {@link org.apache.solr.common.util.DateUtil}. + * This implementation only formats dates using the {@link ExtractionDateUtil}. * * @param val The value to transform * @param schFld The {@link org.apache.solr.schema.SchemaField} @@ -328,10 +326,8 @@ protected String transformValue(String val, SchemaField schFld) { if (schFld != null && schFld.getType() instanceof TrieDateField) { //try to transform the date try { - Date date = DateUtil.parseDate(val, dateFormats); - DateFormat df = DateUtil.getThreadLocalDateFormat(); - result = df.format(date); - + Date date = ExtractionDateUtil.parseDate(val, dateFormats); // may throw + result = date.toInstant().toString();//ISO format } catch (Exception e) { // Let the specific fieldType handle errors // throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid value: " + val + " for field: " + schFld, e); diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestExtractionDateUtil.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestExtractionDateUtil.java new file mode 100644 index 000000000000..9632cb9db682 --- /dev/null +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestExtractionDateUtil.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.extraction; + +import java.text.ParseException; +import java.util.Date; +import java.util.Locale; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestExtractionDateUtil extends LuceneTestCase { + + public void testISO8601() throws Exception { + // dates with atypical years + assertParseFormatEquals("0001-01-01T01:01:01Z", null); + assertParseFormatEquals("+12021-12-01T03:03:03Z", null); + + assertParseFormatEquals("0000-04-04T04:04:04Z", null); // note: 0 AD is also known as 1 BC + + // dates with negative years (BC) + assertParseFormatEquals("-0005-05-05T05:05:05Z", null); + assertParseFormatEquals("-2021-12-01T04:04:04Z", null); + assertParseFormatEquals("-12021-12-01T02:02:02Z", null); + + // dates that only parse thanks to lenient mode of DateTimeFormatter + assertParseFormatEquals("10995-12-31T23:59:59.990Z", "+10995-12-31T23:59:59.990Z"); // missing '+' 5 digit year + assertParseFormatEquals("995-1-2T3:4:5Z", "0995-01-02T03:04:05Z"); // wasn't 0 padded + } + + private static void assertParseFormatEquals(String inputStr, String expectedStr) throws ParseException { + if (expectedStr == null) { + expectedStr = inputStr; + } + Date inputDate = ExtractionDateUtil.parseDate(inputStr); + String resultStr = inputDate.toInstant().toString(); + assertEquals("d:" + inputDate.getTime(), expectedStr, resultStr); + } + + public void testParseDate() throws ParseException { + assertParsedDate(1226583351000L, "Thu Nov 13 04:35:51 AKST 2008"); + } + + private static void assertParsedDate(long ts, String dateStr) throws ParseException { + long parsed = ExtractionDateUtil.parseDate(dateStr).getTime(); + assertTrue(String.format(Locale.ENGLISH, "Incorrect parsed timestamp: %d != %d (%s)", ts, parsed, dateStr), Math.abs(ts - parsed) <= 1000L); + } +} diff --git a/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java b/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java index a5c73bc1bba6..00045b28a141 100644 --- a/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java +++ b/solr/contrib/morphlines-cell/src/java/org/apache/solr/morphlines/cell/SolrCellBuilder.java @@ -29,13 +29,19 @@ import java.util.Map.Entry; import java.util.TreeMap; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.io.Closeables; +import com.typesafe.config.Config; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.handler.extraction.ExtractingParams; +import org.apache.solr.handler.extraction.ExtractionDateUtil; import org.apache.solr.handler.extraction.SolrContentHandler; import org.apache.solr.handler.extraction.SolrContentHandlerFactory; import org.apache.solr.morphlines.solr.SolrLocator; @@ -50,7 +56,6 @@ import org.apache.tika.sax.xpath.Matcher; import org.apache.tika.sax.xpath.MatchingContentHandler; import org.apache.tika.sax.xpath.XPathParser; - import org.kitesdk.morphline.api.Command; import org.kitesdk.morphline.api.CommandBuilder; import org.kitesdk.morphline.api.MorphlineCompilationException; @@ -63,13 +68,6 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; -import com.google.common.io.Closeables; -import com.typesafe.config.Config; - /** * Command that pipes the first attachment of a record into one of the given Tika parsers, then maps * the Tika output back to a record using SolrCell. @@ -151,7 +149,7 @@ public SolrCell(CommandBuilder builder, Config config, Command parent, Command c cellParams.put(ExtractingParams.XPATH_EXPRESSION, xpathExpr); } - this.dateFormats = getConfigs().getStringList(config, "dateFormats", new ArrayList<>(DateUtil.DEFAULT_DATE_FORMATS)); + this.dateFormats = getConfigs().getStringList(config, "dateFormats", new ArrayList<>(ExtractionDateUtil.DEFAULT_DATE_FORMATS)); String handlerStr = getConfigs().getString(config, "solrContentHandlerFactory", TrimSolrContentHandlerFactory.class.getName()); Class factoryClass; diff --git a/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java b/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java index 9a43b423f92e..2cdbd3c5da98 100644 --- a/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java +++ b/solr/contrib/morphlines-cell/src/test/org/apache/solr/morphlines/cell/SolrCellMorphlineTest.java @@ -25,7 +25,7 @@ import org.apache.lucene.util.Constants; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.MapSolrParams; -import org.apache.solr.common.util.DateUtil; +import org.apache.solr.handler.extraction.ExtractionDateUtil; import org.apache.solr.handler.extraction.SolrContentHandler; import org.apache.solr.morphlines.solr.AbstractSolrMorphlineTestBase; import org.apache.solr.schema.IndexSchema; @@ -270,7 +270,7 @@ public void testTransformValue() { // which will cause the ContentHandler to be invoked. metadata.set(fieldName, getFoobarWithNonChars()); StripNonCharSolrContentHandlerFactory contentHandlerFactory = - new StripNonCharSolrContentHandlerFactory(DateUtil.DEFAULT_DATE_FORMATS); + new StripNonCharSolrContentHandlerFactory(ExtractionDateUtil.DEFAULT_DATE_FORMATS); IndexSchema schema = h.getCore().getLatestSchema(); SolrContentHandler contentHandler = contentHandlerFactory.createSolrContentHandler(metadata, new MapSolrParams(new HashMap()), schema); diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestDateUtil.java b/solr/solrj/src/test/org/apache/solr/common/util/TestDateUtil.java deleted file mode 100644 index 1eab4f5b11f0..000000000000 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestDateUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.common.util; - -import java.text.ParseException; -import java.util.Locale; - -import org.apache.lucene.util.LuceneTestCase; - -public class TestDateUtil extends LuceneTestCase { - - public void testParseDate() throws ParseException { - assertParsedDate(1226583351000L, "Thu Nov 13 04:35:51 AKST 2008"); - } - - private static void assertParsedDate(long ts, String dateStr) throws ParseException { - long parsed = DateUtil.parseDate(dateStr).getTime(); - assertTrue(String.format(Locale.ENGLISH, "Incorrect parsed timestamp: %d != %d (%s)", ts, parsed, dateStr), Math.abs(ts - parsed) <= 1000L); - } - -} From 08dae30f738b5766b29600cf58dbaa74419ea0fa Mon Sep 17 00:00:00 2001 From: nknize Date: Wed, 30 Mar 2016 18:14:20 -0500 Subject: [PATCH 0246/1113] * LUCENE-7094: BBoxStrategy and PointVectorStrategy now support PointValues (in addition to legacy numeric trie). Their APIs were changed a little and also made more consistent. PointValues/Trie is optional, DocValues is optional, stored value is optional. --- lucene/CHANGES.txt | 6 + .../lucene/spatial/bbox/BBoxStrategy.java | 318 ++++++++++++------ .../spatial/vector/PointVectorStrategy.java | 177 ++++++++-- .../lucene/spatial/DistanceStrategyTest.java | 40 +-- .../lucene/spatial/PortedSolr3Test.java | 13 +- .../spatial/QueryEqualsHashCodeTest.java | 10 +- .../lucene/spatial/SpatialTestCase.java | 42 +-- .../lucene/spatial/bbox/TestBBoxStrategy.java | 68 ++-- .../composite/CompositeStrategyTest.java | 17 +- .../serialized/SerializedStrategyTest.java | 7 +- .../spatial/spatial4j/Geo3dRptTest.java | 15 +- .../vector/TestPointVectorStrategy.java | 55 ++- .../org/apache/solr/schema/BBoxField.java | 6 +- .../schema/SpatialPointVectorFieldType.java | 23 +- 14 files changed, 528 insertions(+), 269 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index b80b577e1f4d..0e127acfd01d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -111,6 +111,12 @@ New Features API Changes +* LUCENE-7094: BBoxStrategy and PointVectorStrategy now support + PointValues (in addition to legacy numeric trie). Their APIs + were changed a little and also made more consistent. PointValues/Trie + is optional, DocValues is optional, stored value is optional. + (Nick Knize, David Smiley) + * LUCENE-6067: Accountable.getChildResources has a default implementation returning the empty list. (Robert Muir) diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java index 253b290dab8f..63a113839b5e 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java @@ -16,15 +16,15 @@ */ package org.apache.lucene.spatial.bbox; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; -import org.apache.lucene.document.LegacyDoubleField; +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LegacyDoubleField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; @@ -41,6 +41,10 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.NumericUtils; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; /** @@ -63,7 +67,7 @@ *

    * This uses 4 double fields for minX, maxX, minY, maxY * and a boolean to mark a dateline cross. Depending on the particular {@link - * SpatialOperation}s, there are a variety of {@link org.apache.lucene.search.LegacyNumericRangeQuery}s to be + * SpatialOperation}s, there are a variety of range queries on {@link DoublePoint}s to be * done. * The {@link #makeOverlapRatioValueSource(org.locationtech.spatial4j.shape.Rectangle, double)} * works by calculating the query bbox overlap percentage against the indexed @@ -74,6 +78,35 @@ */ public class BBoxStrategy extends SpatialStrategy { + // note: we use a FieldType to articulate the options we want on the field. We don't use it as-is with a Field, we + // create more than one Field. + + /** + * pointValues, docValues, and nothing else. + */ + public static FieldType DEFAULT_FIELDTYPE; + + @Deprecated + public static FieldType LEGACY_FIELDTYPE; + static { + // Default: pointValues + docValues + FieldType type = new FieldType(); + type.setDimensions(1, Double.BYTES);//pointValues (assume Double) + type.setDocValuesType(DocValuesType.NUMERIC);//docValues + type.setStored(false); + type.freeze(); + DEFAULT_FIELDTYPE = type; + // Legacy default: legacyNumerics + docValues + type = new FieldType(); + type.setIndexOptions(IndexOptions.DOCS); + type.setNumericType(FieldType.LegacyNumericType.DOUBLE); + type.setNumericPrecisionStep(8);// same as solr default + type.setDocValuesType(DocValuesType.NUMERIC);//docValues + type.setStored(false); + type.freeze(); + LEGACY_FIELDTYPE = type; + } + public static final String SUFFIX_MINX = "__minX"; public static final String SUFFIX_MAXX = "__maxX"; public static final String SUFFIX_MINY = "__minY"; @@ -84,17 +117,45 @@ public class BBoxStrategy extends SpatialStrategy { * The Bounding Box gets stored as four fields for x/y min/max and a flag * that says if the box crosses the dateline (xdl). */ - protected final String field_bbox; - protected final String field_minX; - protected final String field_minY; - protected final String field_maxX; - protected final String field_maxY; - protected final String field_xdl; // crosses dateline + final String field_bbox; + final String field_minX; + final String field_minY; + final String field_maxX; + final String field_maxY; + final String field_xdl; // crosses dateline + + private final FieldType optionsFieldType;//from constructor; aggregate field type used to express all options + private final int fieldsLen; + private final boolean hasStored; + private final boolean hasDocVals; + private final boolean hasPointVals; + // equiv to "hasLegacyNumerics": + private final FieldType legacyNumericFieldType; // not stored; holds precision step. + private final FieldType xdlFieldType; - protected FieldType fieldType;//for the 4 numbers - protected FieldType xdlFieldType; + /** + * Creates a new {@link BBoxStrategy} instance that uses {@link DoublePoint} and {@link DoublePoint#newRangeQuery} + */ + public static BBoxStrategy newInstance(SpatialContext ctx, String fieldNamePrefix) { + return new BBoxStrategy(ctx, fieldNamePrefix, DEFAULT_FIELDTYPE); + } + + /** + * Creates a new {@link BBoxStrategy} instance that uses {@link LegacyDoubleField} for backwards compatibility + * @deprecated LegacyNumerics will be removed + */ + @Deprecated + public static BBoxStrategy newLegacyInstance(SpatialContext ctx, String fieldNamePrefix) { + return new BBoxStrategy(ctx, fieldNamePrefix, LEGACY_FIELDTYPE); + } - public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix) { + /** + * Creates this strategy. + * {@code fieldType} is used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL + * field too. Search requires pointValues (or legacy numerics), and relevancy requires docValues. If these features + * aren't needed then disable them. + */ + public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) { super(ctx, fieldNamePrefix); field_bbox = fieldNamePrefix; field_minX = fieldNamePrefix + SUFFIX_MINX; @@ -103,34 +164,49 @@ public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix) { field_maxY = fieldNamePrefix + SUFFIX_MAXY; field_xdl = fieldNamePrefix + SUFFIX_XDL; - FieldType fieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - fieldType.setNumericPrecisionStep(8);//Solr's default - fieldType.setDocValuesType(DocValuesType.NUMERIC); - setFieldType(fieldType); - } + fieldType.freeze(); + this.optionsFieldType = fieldType; - private int getPrecisionStep() { - return fieldType.numericPrecisionStep(); - } + int numQuads = 0; + if ((this.hasStored = fieldType.stored())) { + numQuads++; + } + if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) { + numQuads++; + } + if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { + numQuads++; + } + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (hasPointVals) { + throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); + } + if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + } + numQuads++; + legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType.freeze(); + } else { + legacyNumericFieldType = null; + } - public FieldType getFieldType() { - return fieldType; + if (hasPointVals || legacyNumericFieldType != null) { // if we have an index... + xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED); + xdlFieldType.setIndexOptions(IndexOptions.DOCS); + xdlFieldType.freeze(); + } else { + xdlFieldType = null; + } + + this.fieldsLen = numQuads * 4 + (xdlFieldType != null ? 1 : 0); } - /** Used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL field too. Search - * requires indexed=true, and relevancy requires docValues. If these features aren't needed then disable them. - * {@link FieldType#freeze()} is called on the argument. */ - public void setFieldType(FieldType fieldType) { - fieldType.freeze(); - this.fieldType = fieldType; - //only double's supported right now - if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) - throw new IllegalArgumentException("BBoxStrategy only supports doubles at this time."); - //for xdlFieldType, copy some similar options. Don't do docValues since it isn't needed here. - xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED); - xdlFieldType.setStored(fieldType.stored()); - xdlFieldType.setIndexOptions(fieldType.indexOptions()); - xdlFieldType.freeze(); + /** Returns a field type representing the set of field options. This is identical to what was passed into the + * constructor. It's frozen. */ + public FieldType getFieldType() { + return optionsFieldType; } //--------------------------------- @@ -142,40 +218,41 @@ public Field[] createIndexableFields(Shape shape) { return createIndexableFields(shape.getBoundingBox()); } - public Field[] createIndexableFields(Rectangle bbox) { - Field[] fields = new Field[5]; - fields[0] = new ComboField(field_minX, bbox.getMinX(), fieldType); - fields[1] = new ComboField(field_maxX, bbox.getMaxX(), fieldType); - fields[2] = new ComboField(field_minY, bbox.getMinY(), fieldType); - fields[3] = new ComboField(field_maxY, bbox.getMaxY(), fieldType); - fields[4] = new ComboField(field_xdl, bbox.getCrossesDateLine()?"T":"F", xdlFieldType); - return fields; - } - - /** Field subclass circumventing Field limitations. This one instance can have any combination of indexed, stored, - * and docValues. - */ - private static class ComboField extends Field { - private ComboField(String name, Object value, FieldType type) { - super(name, type);//this expert constructor allows us to have a field that has docValues & indexed/stored - super.fieldsData = value; + private Field[] createIndexableFields(Rectangle bbox) { + Field[] fields = new Field[fieldsLen]; + int idx = -1; + if (hasStored) { + fields[++idx] = new StoredField(field_minX, bbox.getMinX()); + fields[++idx] = new StoredField(field_minY, bbox.getMinY()); + fields[++idx] = new StoredField(field_maxX, bbox.getMaxX()); + fields[++idx] = new StoredField(field_maxY, bbox.getMaxY()); } - - //Is this a hack? We assume that numericValue() is only called for DocValues purposes. - @Override - public Number numericValue() { - //Numeric DocValues only supports Long, - final Number number = super.numericValue(); - if (number == null) - return null; - if (fieldType().numericType() == FieldType.LegacyNumericType.DOUBLE) - return Double.doubleToLongBits(number.doubleValue()); - if (fieldType().numericType() == FieldType.LegacyNumericType.FLOAT) - return Float.floatToIntBits(number.floatValue()); - return number.longValue(); + if (hasDocVals) { + fields[++idx] = new DoubleDocValuesField(field_minX, bbox.getMinX()); + fields[++idx] = new DoubleDocValuesField(field_minY, bbox.getMinY()); + fields[++idx] = new DoubleDocValuesField(field_maxX, bbox.getMaxX()); + fields[++idx] = new DoubleDocValuesField(field_maxY, bbox.getMaxY()); } + if (hasPointVals) { + fields[++idx] = new DoublePoint(field_minX, bbox.getMinX()); + fields[++idx] = new DoublePoint(field_minY, bbox.getMinY()); + fields[++idx] = new DoublePoint(field_maxX, bbox.getMaxX()); + fields[++idx] = new DoublePoint(field_maxY, bbox.getMaxY()); + } + if (legacyNumericFieldType != null) { + fields[++idx] = new LegacyDoubleField(field_minX, bbox.getMinX(), legacyNumericFieldType); + fields[++idx] = new LegacyDoubleField(field_minY, bbox.getMinY(), legacyNumericFieldType); + fields[++idx] = new LegacyDoubleField(field_maxX, bbox.getMaxX(), legacyNumericFieldType); + fields[++idx] = new LegacyDoubleField(field_maxY, bbox.getMaxY(), legacyNumericFieldType); + } + if (xdlFieldType != null) { + fields[++idx] = new Field(field_xdl, bbox.getCrossesDateLine()?"T":"F", xdlFieldType); + } + assert idx == fields.length - 1; + return fields; } + //--------------------------------- // Value Source / Relevancy //--------------------------------- @@ -253,8 +330,8 @@ Query makeContains(Rectangle bbox) { // Y conditions // docMinY <= queryExtent.getMinY() AND docMaxY >= queryExtent.getMaxY() - Query qMinY = LegacyNumericRangeQuery.newDoubleRange(field_minY, getPrecisionStep(), null, bbox.getMinY(), false, true); - Query qMaxY = LegacyNumericRangeQuery.newDoubleRange(field_maxY, getPrecisionStep(), bbox.getMaxY(), null, true, false); + Query qMinY = this.makeNumericRangeQuery(field_minY, null, bbox.getMinY(), false, true); + Query qMaxY = this.makeNumericRangeQuery(field_maxY, bbox.getMaxY(), null, true, false); Query yConditions = this.makeQuery(BooleanClause.Occur.MUST, qMinY, qMaxY); // X conditions @@ -266,8 +343,8 @@ Query makeContains(Rectangle bbox) { // X Conditions for documents that do not cross the date line, // documents that contain the min X and max X of the query envelope, // docMinX <= queryExtent.getMinX() AND docMaxX >= queryExtent.getMaxX() - Query qMinX = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), null, bbox.getMinX(), false, true); - Query qMaxX = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), bbox.getMaxX(), null, true, false); + Query qMinX = this.makeNumericRangeQuery(field_minX, null, bbox.getMinX(), false, true); + Query qMaxX = this.makeNumericRangeQuery(field_maxX, bbox.getMaxX(), null, true, false); Query qMinMax = this.makeQuery(BooleanClause.Occur.MUST, qMinX, qMaxX); Query qNonXDL = this.makeXDL(false, qMinMax); @@ -278,8 +355,8 @@ Query makeContains(Rectangle bbox) { // the left portion of the document contains the min X of the query // OR the right portion of the document contains the max X of the query, // docMinXLeft <= queryExtent.getMinX() OR docMaxXRight >= queryExtent.getMaxX() - Query qXDLLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), null, bbox.getMinX(), false, true); - Query qXDLRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), bbox.getMaxX(), null, true, false); + Query qXDLLeft = this.makeNumericRangeQuery(field_minX, null, bbox.getMinX(), false, true); + Query qXDLRight = this.makeNumericRangeQuery(field_maxX, bbox.getMaxX(), null, true, false); Query qXDLLeftRight = this.makeQuery(BooleanClause.Occur.SHOULD, qXDLLeft, qXDLRight); Query qXDL = this.makeXDL(true, qXDLLeftRight); @@ -302,8 +379,8 @@ Query makeContains(Rectangle bbox) { // the left portion of the document contains the min X of the query // AND the right portion of the document contains the max X of the query, // docMinXLeft <= queryExtent.getMinX() AND docMaxXRight >= queryExtent.getMaxX() - Query qXDLLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), null, bbox.getMinX(), false, true); - Query qXDLRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), bbox.getMaxX(), null, true, false); + Query qXDLLeft = this.makeNumericRangeQuery(field_minX, null, bbox.getMinX(), false, true); + Query qXDLRight = this.makeNumericRangeQuery(field_maxX, bbox.getMaxX(), null, true, false); Query qXDLLeftRight = this.makeXDL(true, this.makeQuery(BooleanClause.Occur.MUST, qXDLLeft, qXDLRight)); Query qWorld = makeQuery(BooleanClause.Occur.MUST, @@ -328,8 +405,8 @@ Query makeDisjoint(Rectangle bbox) { // Y conditions // docMinY > queryExtent.getMaxY() OR docMaxY < queryExtent.getMinY() - Query qMinY = LegacyNumericRangeQuery.newDoubleRange(field_minY, getPrecisionStep(), bbox.getMaxY(), null, false, false); - Query qMaxY = LegacyNumericRangeQuery.newDoubleRange(field_maxY, getPrecisionStep(), null, bbox.getMinY(), false, false); + Query qMinY = this.makeNumericRangeQuery(field_minY, bbox.getMaxY(), null, false, false); + Query qMaxY = this.makeNumericRangeQuery(field_maxY, null, bbox.getMinY(), false, false); Query yConditions = this.makeQuery(BooleanClause.Occur.SHOULD, qMinY, qMaxY); // X conditions @@ -340,14 +417,15 @@ Query makeDisjoint(Rectangle bbox) { // X Conditions for documents that do not cross the date line, // docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX() - Query qMinX = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMaxX(), null, false, false); + Query qMinX = this.makeNumericRangeQuery(field_minX, bbox.getMaxX(), null, false, false); if (bbox.getMinX() == -180.0 && ctx.isGeo()) {//touches dateline; -180 == 180 BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(qMinX, BooleanClause.Occur.MUST); bq.add(makeNumberTermQuery(field_maxX, 180.0), BooleanClause.Occur.MUST_NOT); qMinX = bq.build(); } - Query qMaxX = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMinX(), false, false); + Query qMaxX = this.makeNumericRangeQuery(field_maxX, null, bbox.getMinX(), false, false); + if (bbox.getMaxX() == 180.0 && ctx.isGeo()) {//touches dateline; -180 == 180 BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(qMaxX, BooleanClause.Occur.MUST); @@ -368,8 +446,8 @@ Query makeDisjoint(Rectangle bbox) { // where: docMaxXLeft = 180.0, docMinXRight = -180.0 // (docMaxXLeft < queryExtent.getMinX()) equates to (180.0 < queryExtent.getMinX()) and is ignored // (docMinXRight > queryExtent.getMaxX()) equates to (-180.0 > queryExtent.getMaxX()) and is ignored - Query qMinXLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMaxX(), null, false, false); - Query qMaxXRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMinX(), false, false); + Query qMinXLeft = this.makeNumericRangeQuery(field_minX, bbox.getMaxX(), null, false, false); + Query qMaxXRight = this.makeNumericRangeQuery(field_maxX, null, bbox.getMinX(), false, false); Query qLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qMinXLeft, qMaxXRight); Query qXDL = this.makeXDL(true, qLeftRight); @@ -383,10 +461,10 @@ Query makeDisjoint(Rectangle bbox) { // the document must be disjoint to both the left and right query portions // (docMinX > queryExtent.getMaxX()Left OR docMaxX < queryExtent.getMinX()) AND (docMinX > queryExtent.getMaxX() OR docMaxX < queryExtent.getMinX()Left) // where: queryExtent.getMaxX()Left = 180.0, queryExtent.getMinX()Left = -180.0 - Query qMinXLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), 180.0, null, false, false); - Query qMaxXLeft = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMinX(), false, false); - Query qMinXRight = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMaxX(), null, false, false); - Query qMaxXRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, -180.0, false, false); + Query qMinXLeft = this.makeNumericRangeQuery(field_minX, 180.0, null, false, false); + Query qMaxXLeft = this.makeNumericRangeQuery(field_maxX, null, bbox.getMinX(), false, false); + Query qMinXRight = this.makeNumericRangeQuery(field_minX, bbox.getMaxX(), null, false, false); + Query qMaxXRight = this.makeNumericRangeQuery(field_maxX, null, -180.0, false, false); Query qLeft = this.makeQuery(BooleanClause.Occur.SHOULD, qMinXLeft, qMaxXLeft); Query qRight = this.makeQuery(BooleanClause.Occur.SHOULD, qMinXRight, qMaxXRight); Query qLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qLeft, qRight); @@ -478,8 +556,8 @@ Query makeWithin(Rectangle bbox) { // Y conditions // docMinY >= queryExtent.getMinY() AND docMaxY <= queryExtent.getMaxY() - Query qMinY = LegacyNumericRangeQuery.newDoubleRange(field_minY, getPrecisionStep(), bbox.getMinY(), null, true, false); - Query qMaxY = LegacyNumericRangeQuery.newDoubleRange(field_maxY, getPrecisionStep(), null, bbox.getMaxY(), false, true); + Query qMinY = this.makeNumericRangeQuery(field_minY, bbox.getMinY(), null, true, false); + Query qMaxY = this.makeNumericRangeQuery(field_maxY, null, bbox.getMaxY(), false, true); Query yConditions = this.makeQuery(BooleanClause.Occur.MUST, qMinY, qMaxY); // X conditions @@ -493,8 +571,8 @@ Query makeWithin(Rectangle bbox) { // queries that do not cross the date line // docMinX >= queryExtent.getMinX() AND docMaxX <= queryExtent.getMaxX() - Query qMinX = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMinX(), null, true, false); - Query qMaxX = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMaxX(), false, true); + Query qMinX = this.makeNumericRangeQuery(field_minX, bbox.getMinX(), null, true, false); + Query qMaxX = this.makeNumericRangeQuery(field_maxX, null, bbox.getMaxX(), false, true); Query qMinMax = this.makeQuery(BooleanClause.Occur.MUST, qMinX, qMaxX); double edge = 0;//none, otherwise opposite dateline of query @@ -517,14 +595,14 @@ else if (bbox.getMaxX() == 180.0) // the document should be within the left portion of the query // docMinX >= queryExtent.getMinX() AND docMaxX <= 180.0 - Query qMinXLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMinX(), null, true, false); - Query qMaxXLeft = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, 180.0, false, true); + Query qMinXLeft = this.makeNumericRangeQuery(field_minX, bbox.getMinX(), null, true, false); + Query qMaxXLeft = this.makeNumericRangeQuery(field_maxX, null, 180.0, false, true); Query qLeft = this.makeQuery(BooleanClause.Occur.MUST, qMinXLeft, qMaxXLeft); // the document should be within the right portion of the query // docMinX >= -180.0 AND docMaxX <= queryExtent.getMaxX() - Query qMinXRight = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), -180.0, null, true, false); - Query qMaxXRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMaxX(), false, true); + Query qMinXRight = this.makeNumericRangeQuery(field_minX, -180.0, null, true, false); + Query qMaxXRight = this.makeNumericRangeQuery(field_maxX, null, bbox.getMaxX(), false, true); Query qRight = this.makeQuery(BooleanClause.Occur.MUST, qMinXRight, qMaxXRight); // either left or right conditions should occur, @@ -537,8 +615,8 @@ else if (bbox.getMaxX() == 180.0) // AND the right portion of the document must be within the right portion of the query // docMinXLeft >= queryExtent.getMinX() AND docMaxXLeft <= 180.0 // AND docMinXRight >= -180.0 AND docMaxXRight <= queryExtent.getMaxX() - Query qXDLLeft = LegacyNumericRangeQuery.newDoubleRange(field_minX, getPrecisionStep(), bbox.getMinX(), null, true, false); - Query qXDLRight = LegacyNumericRangeQuery.newDoubleRange(field_maxX, getPrecisionStep(), null, bbox.getMaxX(), false, true); + Query qXDLLeft = this.makeNumericRangeQuery(field_minX, bbox.getMinX(), null, true, false); + Query qXDLRight = this.makeNumericRangeQuery(field_maxX, null, bbox.getMaxX(), false, true); Query qXDLLeftRight = this.makeQuery(BooleanClause.Occur.MUST, qXDLLeft, qXDLRight); Query qXDL = this.makeXDL(true, qXDLLeftRight); @@ -581,9 +659,49 @@ private Query makeXDL(boolean crossedDateLine, Query query) { } private Query makeNumberTermQuery(String field, double number) { - BytesRefBuilder bytes = new BytesRefBuilder(); - LegacyNumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(number), 0, bytes); - return new TermQuery(new Term(field, bytes.get())); + if (hasPointVals) { + return DoublePoint.newExactQuery(field, number); + } else if (legacyNumericFieldType != null) { + BytesRefBuilder bytes = new BytesRefBuilder(); + LegacyNumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(number), 0, bytes); + return new TermQuery(new Term(field, bytes.get())); + } + throw new UnsupportedOperationException("An index is required for this operation."); } + /** + * Returns a numeric range query based on FieldType + * {@link LegacyNumericRangeQuery} is used for indexes created using {@code FieldType.LegacyNumericType} + * {@link DoublePoint#newRangeQuery} is used for indexes created using {@link DoublePoint} fields + * + * @param fieldname field name. must not be null. + * @param min minimum value of the range. + * @param max maximum value of the range. + * @param minInclusive include the minimum value if true. + * @param maxInclusive include the maximum value if true + */ + private Query makeNumericRangeQuery(String fieldname, Double min, Double max, boolean minInclusive, boolean maxInclusive) { + if (hasPointVals) { + if (min == null) { + min = Double.NEGATIVE_INFINITY; + } + + if (max == null) { + max = Double.POSITIVE_INFINITY; + } + + if (minInclusive == false) { + min = Math.nextUp(min); + } + + if (maxInclusive == false) { + max = Math.nextDown(max); + } + + return DoublePoint.newRangeQuery(fieldname, min, max); + } else if (legacyNumericFieldType != null) {// todo remove legacy numeric support in 7.0 + return LegacyNumericRangeQuery.newDoubleRange(fieldname, legacyNumericFieldType.numericPrecisionStep(), min, max, minInclusive, maxInclusive); + } + throw new UnsupportedOperationException("An index is required for this operation."); + } } diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java index f5f5f34121f7..197547c1d562 100644 --- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java +++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java @@ -16,14 +16,14 @@ */ package org.apache.lucene.spatial.vector; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Circle; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; -import org.apache.lucene.document.LegacyDoubleField; +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LegacyDoubleField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.queries.function.FunctionRangeQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause; @@ -35,10 +35,15 @@ import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; /** - * Simple {@link SpatialStrategy} which represents Points in two numeric {@link - * org.apache.lucene.document.LegacyDoubleField}s. The Strategy's best feature is decent distance sort. + * Simple {@link SpatialStrategy} which represents Points in two numeric fields. + * The Strategy's best feature is decent distance sort. * *

    * Characteristics: @@ -49,7 +54,7 @@ *

  • {@link * org.apache.lucene.spatial.query.SpatialOperation#Intersects} and {@link * SpatialOperation#IsWithin} is supported.
    • - *
  • Uses the FieldCache for + *
  • Requires DocValues for * {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point)} and for * searching with a Circle.
    • * @@ -57,8 +62,8 @@ *

    * Implementation: *

    - * This is a simple Strategy. Search works with {@link org.apache.lucene.search.LegacyNumericRangeQuery}s on - * an x and y pair of fields. A Circle query does the same bbox query but adds a + * This is a simple Strategy. Search works with a pair of range queries on two {@link DoublePoint}s representing + * x & y fields. A Circle query does the same bbox query but adds a * ValueSource filter on * {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point)}. *

    @@ -71,26 +76,105 @@ */ public class PointVectorStrategy extends SpatialStrategy { + // note: we use a FieldType to articulate the options we want on the field. We don't use it as-is with a Field, we + // create more than one Field. + + /** + * pointValues, docValues, and nothing else. + */ + public static FieldType DEFAULT_FIELDTYPE; + + @Deprecated + public static FieldType LEGACY_FIELDTYPE; + static { + // Default: pointValues + docValues + FieldType type = new FieldType(); + type.setDimensions(1, Double.BYTES);//pointValues (assume Double) + type.setDocValuesType(DocValuesType.NUMERIC);//docValues + type.setStored(false); + type.freeze(); + DEFAULT_FIELDTYPE = type; + // Legacy default: legacyNumerics + type = new FieldType(); + type.setIndexOptions(IndexOptions.DOCS); + type.setNumericType(FieldType.LegacyNumericType.DOUBLE); + type.setNumericPrecisionStep(8);// same as solr default + type.setDocValuesType(DocValuesType.NONE);//no docValues! + type.setStored(false); + type.freeze(); + LEGACY_FIELDTYPE = type; + } + public static final String SUFFIX_X = "__x"; public static final String SUFFIX_Y = "__y"; private final String fieldNameX; private final String fieldNameY; - public int precisionStep = 8; // same as solr default + private final int fieldsLen; + private final boolean hasStored; + private final boolean hasDocVals; + private final boolean hasPointVals; + // equiv to "hasLegacyNumerics": + private final FieldType legacyNumericFieldType; // not stored; holds precision step. + + /** + * Create a new {@link PointVectorStrategy} instance that uses {@link DoublePoint} and {@link DoublePoint#newRangeQuery} + */ + public static PointVectorStrategy newInstance(SpatialContext ctx, String fieldNamePrefix) { + return new PointVectorStrategy(ctx, fieldNamePrefix, DEFAULT_FIELDTYPE); + } + + /** + * Create a new {@link PointVectorStrategy} instance that uses {@link LegacyDoubleField} for backwards compatibility. + * However, back-compat is limited; we don't support circle queries or {@link #makeDistanceValueSource(Point, double)} + * since that requires docValues (the legacy config didn't have that). + * + * @deprecated LegacyNumerics will be removed + */ + @Deprecated + public static PointVectorStrategy newLegacyInstance(SpatialContext ctx, String fieldNamePrefix) { + return new PointVectorStrategy(ctx, fieldNamePrefix, LEGACY_FIELDTYPE); + } - public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix) { + /** + * Create a new instance configured with the provided FieldType options. See {@link #DEFAULT_FIELDTYPE}. + * a field type is used to articulate the desired options (namely pointValues, docValues, stored). Legacy numerics + * is configurable this way too. + */ + public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) { super(ctx, fieldNamePrefix); this.fieldNameX = fieldNamePrefix+SUFFIX_X; this.fieldNameY = fieldNamePrefix+SUFFIX_Y; - } - public void setPrecisionStep( int p ) { - precisionStep = p; - if (precisionStep<=0 || precisionStep>=64) - precisionStep=Integer.MAX_VALUE; + int numPairs = 0; + if ((this.hasStored = fieldType.stored())) { + numPairs++; + } + if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) { + numPairs++; + } + if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) { + numPairs++; + } + if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.numericType() != null) { + if (hasPointVals) { + throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive"); + } + if (fieldType.numericType() != FieldType.LegacyNumericType.DOUBLE) { + throw new IllegalArgumentException(getClass() + " does not support " + fieldType.numericType()); + } + numPairs++; + legacyNumericFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); + legacyNumericFieldType.setNumericPrecisionStep(fieldType.numericPrecisionStep()); + legacyNumericFieldType.freeze(); + } else { + legacyNumericFieldType = null; + } + this.fieldsLen = numPairs * 2; } + String getFieldNameX() { return fieldNameX; } @@ -108,12 +192,26 @@ public Field[] createIndexableFields(Shape shape) { /** @see #createIndexableFields(org.locationtech.spatial4j.shape.Shape) */ public Field[] createIndexableFields(Point point) { - FieldType doubleFieldType = new FieldType(LegacyDoubleField.TYPE_NOT_STORED); - doubleFieldType.setNumericPrecisionStep(precisionStep); - Field[] f = new Field[2]; - f[0] = new LegacyDoubleField(fieldNameX, point.getX(), doubleFieldType); - f[1] = new LegacyDoubleField(fieldNameY, point.getY(), doubleFieldType); - return f; + Field[] fields = new Field[fieldsLen]; + int idx = -1; + if (hasStored) { + fields[++idx] = new StoredField(fieldNameX, point.getX()); + fields[++idx] = new StoredField(fieldNameY, point.getY()); + } + if (hasDocVals) { + fields[++idx] = new DoubleDocValuesField(fieldNameX, point.getX()); + fields[++idx] = new DoubleDocValuesField(fieldNameY, point.getY()); + } + if (hasPointVals) { + fields[++idx] = new DoublePoint(fieldNameX, point.getX()); + fields[++idx] = new DoublePoint(fieldNameY, point.getY()); + } + if (legacyNumericFieldType != null) { + fields[++idx] = new LegacyDoubleField(fieldNameX, point.getX(), legacyNumericFieldType); + fields[++idx] = new LegacyDoubleField(fieldNameY, point.getY(), legacyNumericFieldType); + } + assert idx == fields.length - 1; + return fields; } @Override @@ -165,14 +263,27 @@ private Query makeWithin(Rectangle bbox) { return bq.build(); } - private LegacyNumericRangeQuery rangeQuery(String fieldName, Double min, Double max) { - return LegacyNumericRangeQuery.newDoubleRange( - fieldName, - precisionStep, - min, - max, - true, - true);//inclusive - } + /** + * Returns a numeric range query based on FieldType + * {@link LegacyNumericRangeQuery} is used for indexes created using {@code FieldType.LegacyNumericType} + * {@link DoublePoint#newRangeQuery} is used for indexes created using {@link DoublePoint} fields + */ + private Query rangeQuery(String fieldName, Double min, Double max) { + if (hasPointVals) { + if (min == null) { + min = Double.NEGATIVE_INFINITY; + } + + if (max == null) { + max = Double.POSITIVE_INFINITY; + } + + return DoublePoint.newRangeQuery(fieldName, min, max); + } else if (legacyNumericFieldType != null) {// todo remove legacy numeric support in 7.0 + return LegacyNumericRangeQuery.newDoubleRange(fieldName, legacyNumericFieldType.numericPrecisionStep(), min, max, true, true);//inclusive + } + //TODO try doc-value range query? + throw new UnsupportedOperationException("An index is required for this operation."); + } } diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java index 160267905436..d54e1c970a08 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java @@ -22,11 +22,6 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.spatial.bbox.BBoxStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; @@ -37,6 +32,9 @@ import org.apache.lucene.spatial.serialized.SerializedDVStrategy; import org.apache.lucene.spatial.vector.PointVectorStrategy; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; public class DistanceStrategyTest extends StrategyTestCase { @ParametersFactory(argumentFormatting = "strategy=%s") @@ -59,10 +57,18 @@ public static Iterable parameters() { strategy = new RecursivePrefixTreeStrategy(grid, "recursive_packedquad"); ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); - strategy = new PointVectorStrategy(ctx, "pointvector"); + strategy = PointVectorStrategy.newInstance(ctx, "pointvector"); + ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); + +// Can't test this without un-inverting since PVS legacy config didn't have docValues. +// However, note that Solr's tests use UninvertingReader and thus test this. +// strategy = PointVectorStrategy.newLegacyInstance(ctx, "pointvector_legacy"); +// ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); + + strategy = BBoxStrategy.newInstance(ctx, "bbox"); ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); - strategy = new BBoxStrategy(ctx, "bbox"); + strategy = BBoxStrategy.newLegacyInstance(ctx, "bbox_legacy"); ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); strategy = new SerializedDVStrategy(ctx, "serialized"); @@ -76,22 +82,6 @@ public DistanceStrategyTest(String suiteName, SpatialStrategy strategy) { this.strategy = strategy; } - @Override - public void setUp() throws Exception { - super.setUp(); - if (strategy instanceof BBoxStrategy && random().nextBoolean()) {//disable indexing sometimes - BBoxStrategy bboxStrategy = (BBoxStrategy)strategy; - final FieldType fieldType = new FieldType(bboxStrategy.getFieldType()); - fieldType.setIndexOptions(IndexOptions.NONE); - bboxStrategy.setFieldType(fieldType); - } - } - - @Override - protected boolean needsDocValues() { - return true; - } - @Test public void testDistanceOrder() throws IOException { adoc("100", ctx.makePoint(2, 1)); @@ -108,9 +98,9 @@ public void testDistanceOrder() throws IOException { @Test public void testRecipScore() throws IOException { - Point p100 = ctx.makePoint(2, 1); + Point p100 = ctx.makePoint(2.02, 0.98); adoc("100", p100); - Point p101 = ctx.makePoint(-1, 4); + Point p101 = ctx.makePoint(-1.001, 4.001); adoc("101", p101); adoc("103", (Shape)null);//test score for nothing adoc("999", ctx.makePoint(2, 1));//test deleted diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java index a081497c387a..f55daf64bb0e 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/PortedSolr3Test.java @@ -22,10 +22,6 @@ import java.util.Set; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.distance.DistanceUtils; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; @@ -36,6 +32,10 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.vector.PointVectorStrategy; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Shape; /** * Based off of Solr 3's SpatialFilterTest. @@ -62,7 +62,10 @@ public static Iterable parameters() { strategy = new TermQueryPrefixTreeStrategy(grid, "termquery_geohash"); ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); - strategy = new PointVectorStrategy(ctx, "pointvector"); + strategy = PointVectorStrategy.newInstance(ctx, "pointvector"); + ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); + + strategy = PointVectorStrategy.newInstance(ctx, "pointvector_legacy"); ctorArgs.add(new Object[]{strategy.getFieldName(), strategy}); return ctorArgs; diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java index 5dbb8f837e85..c14fe546f977 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/QueryEqualsHashCodeTest.java @@ -19,8 +19,6 @@ import java.util.ArrayList; import java.util.Collection; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.bbox.BBoxStrategy; import org.apache.lucene.spatial.composite.CompositeSpatialStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; @@ -34,6 +32,8 @@ import org.apache.lucene.spatial.vector.PointVectorStrategy; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Shape; public class QueryEqualsHashCodeTest extends LuceneTestCase { @@ -57,8 +57,10 @@ public void testEqualsHashCode() { RecursivePrefixTreeStrategy recursive_geohash = new RecursivePrefixTreeStrategy(gridGeohash, "recursive_geohash"); strategies.add(recursive_geohash); strategies.add(new TermQueryPrefixTreeStrategy(gridQuad, "termquery_quad")); - strategies.add(new PointVectorStrategy(ctx, "pointvector")); - strategies.add(new BBoxStrategy(ctx, "bbox")); + strategies.add(PointVectorStrategy.newInstance(ctx, "pointvector")); + strategies.add(PointVectorStrategy.newLegacyInstance(ctx, "pointvector_legacy")); + strategies.add(BBoxStrategy.newInstance(ctx, "bbox")); + strategies.add(BBoxStrategy.newLegacyInstance(ctx, "bbox_legacy")); final SerializedDVStrategy serialized = new SerializedDVStrategy(ctx, "serialized"); strategies.add(serialized); strategies.add(new CompositeSpatialStrategy("composite", recursive_geohash, serialized)); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java index 8ccb9afdbe56..9f4ba0271b91 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialTestCase.java @@ -18,34 +18,26 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Random; import java.util.logging.Logger; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.distance.DistanceUtils; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.apache.lucene.uninverting.UninvertingReader; -import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; -import org.apache.lucene.util.TestUtil; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomGaussian; @@ -66,36 +58,16 @@ public abstract class SpatialTestCase extends LuceneTestCase { protected SpatialContext ctx;//subclass must initialize - protected Map uninvertMap = new HashMap<>(); - @Override public void setUp() throws Exception { super.setUp(); - // TODO: change this module to index docvalues instead of uninverting - uninvertMap.clear(); - uninvertMap.put("pointvector__x", Type.LEGACY_DOUBLE); - uninvertMap.put("pointvector__y", Type.LEGACY_DOUBLE); - directory = newDirectory(); - final Random random = random(); - analyzer = new MockAnalyzer(random); - indexWriter = new RandomIndexWriter(random,directory, newIWConfig(random, analyzer)); - indexReader = UninvertingReader.wrap(indexWriter.getReader(), uninvertMap); + analyzer = new MockAnalyzer(random()); + indexWriter = new RandomIndexWriter(random(), directory, LuceneTestCase.newIndexWriterConfig(random(), analyzer)); + indexReader = indexWriter.getReader(); indexSearcher = newSearcher(indexReader); } - protected IndexWriterConfig newIWConfig(Random random, Analyzer analyzer) { - final IndexWriterConfig indexWriterConfig = LuceneTestCase.newIndexWriterConfig(random, analyzer); - //TODO can we randomly choose a doc-values supported format? - if (needsDocValues()) - indexWriterConfig.setCodec( TestUtil.getDefaultCodec()); - return indexWriterConfig; - } - - protected boolean needsDocValues() { - return false; - } - @Override public void tearDown() throws Exception { IOUtils.close(indexWriter, indexReader, analyzer, directory); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java index 20a7202ba331..01e925926d20 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java @@ -19,12 +19,6 @@ import java.io.IOException; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.context.SpatialContextFactory; -import org.locationtech.spatial4j.distance.DistanceUtils; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; -import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; @@ -36,14 +30,15 @@ import org.apache.lucene.spatial.util.ShapeAreaValueSource; import org.junit.Ignore; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; public class TestBBoxStrategy extends RandomSpatialOpStrategyTestCase { - @Override - protected boolean needsDocValues() { - return true; - } - @Override protected Shape randomIndexedShape() { Rectangle world = ctx.getWorldBounds(); @@ -97,13 +92,17 @@ public void testOperations() throws IOException { factory.worldBounds = new RectangleImpl(-300, 300, -100, 100, null); this.ctx = factory.newSpatialContext(); } - this.strategy = new BBoxStrategy(ctx, "bbox"); + // randomly test legacy (numeric) and point based bbox strategy + if (random().nextBoolean()) { + this.strategy = BBoxStrategy.newInstance(ctx, "bbox"); + } else { + this.strategy = BBoxStrategy.newLegacyInstance(ctx, "bbox"); + } //test we can disable docValues for predicate tests if (random().nextBoolean()) { - BBoxStrategy bboxStrategy = (BBoxStrategy) strategy; - FieldType fieldType = new FieldType(bboxStrategy.getFieldType()); + FieldType fieldType = new FieldType(((BBoxStrategy)strategy).getFieldType()); fieldType.setDocValuesType(DocValuesType.NONE); - bboxStrategy.setFieldType(fieldType); + strategy = new BBoxStrategy(ctx, strategy.getFieldName(), fieldType); } for (SpatialOperation operation : SpatialOperation.values()) { if (operation == SpatialOperation.Overlaps) @@ -189,7 +188,11 @@ public void testAlongDatelineOppositeSign() throws IOException { private void setupGeo() { this.ctx = SpatialContext.GEO; - this.strategy = new BBoxStrategy(ctx, "bbox"); + if (random().nextBoolean()) { + this.strategy = BBoxStrategy.newInstance(ctx, "bbox"); + } else { + this.strategy = BBoxStrategy.newLegacyInstance(ctx, "bbox"); + } } // OLD STATIC TESTS (worthless?) @@ -225,8 +228,29 @@ protected Shape convertShapeFromGetDocuments(Shape shape) { return shape.getBoundingBox(); } + private BBoxStrategy setupNeedsDocValuesOnly() throws IOException { + this.ctx = SpatialContext.GEO; + FieldType fieldType; + // random legacy or not legacy + String FIELD_PREFIX = "bbox"; + if (random().nextBoolean()) { + fieldType = new FieldType(BBoxStrategy.DEFAULT_FIELDTYPE); + if (random().nextBoolean()) { + fieldType.setDimensions(0, 0); + } + } else { + fieldType = new FieldType(BBoxStrategy.LEGACY_FIELDTYPE); + if (random().nextBoolean()) { + fieldType.setIndexOptions(IndexOptions.NONE); + } + } + + strategy = new BBoxStrategy(ctx, FIELD_PREFIX, fieldType); + return (BBoxStrategy)strategy; + } + public void testOverlapRatio() throws IOException { - setupGeo(); + setupNeedsDocValuesOnly(); //Simply assert null shape results in 0 adoc("999", (Shape) null); @@ -279,14 +303,7 @@ private Rectangle shiftedRect(double minX, double maxX, double minY, double maxY } public void testAreaValueSource() throws IOException { - setupGeo(); - //test we can disable indexed for this test - BBoxStrategy bboxStrategy = (BBoxStrategy) strategy; - if (random().nextBoolean()) { - FieldType fieldType = new FieldType(bboxStrategy.getFieldType()); - fieldType.setIndexOptions(IndexOptions.NONE); - bboxStrategy.setFieldType(fieldType); - } + BBoxStrategy bboxStrategy = setupNeedsDocValuesOnly(); adoc("100", ctx.makeRectangle(0, 20, 40, 80)); adoc("999", (Shape) null); @@ -298,4 +315,5 @@ public void testAreaValueSource() throws IOException { checkValueSource(new ShapeAreaValueSource(bboxStrategy.makeShapeValueSource(), ctx, true, 2.0), new float[]{783.86f, 0f}, 0.01f); // testing with a different multiplier } + } diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java index 7d49e8b2e4cb..e74c2b5d0749 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/composite/CompositeStrategyTest.java @@ -19,12 +19,6 @@ import java.io.IOException; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.context.SpatialContextFactory; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; -import org.locationtech.spatial4j.shape.impl.RectangleImpl; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; @@ -33,6 +27,12 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.serialized.SerializedDVStrategy; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.context.SpatialContextFactory; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; +import org.locationtech.spatial4j.shape.impl.RectangleImpl; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; @@ -101,11 +101,6 @@ public void testOperations() throws IOException { } } - @Override - protected boolean needsDocValues() { - return true;//due to SerializedDVStrategy - } - @Override protected Shape randomIndexedShape() { return randomShape(); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java index 6a73d23d2f28..8466ae13d4f0 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/serialized/SerializedStrategyTest.java @@ -18,11 +18,11 @@ import java.io.IOException; -import org.locationtech.spatial4j.context.SpatialContext; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.StrategyTestCase; import org.junit.Before; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; public class SerializedStrategyTest extends StrategyTestCase { @@ -34,11 +34,6 @@ public void setUp() throws Exception { this.strategy = new SerializedDVStrategy(ctx, "serialized"); } - @Override - protected boolean needsDocValues() { - return (strategy instanceof SerializedDVStrategy); - } - @Test public void testBasicOperaions() throws IOException { getAddAndVerifyIndexedDocuments(DATA_SIMPLE_BBOX); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java index e62b857faeb7..2bef2b7cae18 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java @@ -21,10 +21,6 @@ import java.util.List; import com.carrotsearch.randomizedtesting.annotations.Repeat; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.Shape; import org.apache.lucene.spatial.composite.CompositeSpatialStrategy; import org.apache.lucene.spatial.prefix.RandomSpatialOpStrategyTestCase; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; @@ -33,13 +29,17 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.serialized.SerializedDVStrategy; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; -import org.apache.lucene.spatial3d.geom.GeoStandardCircle; import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.GeoStandardCircle; import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Test; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.Shape; import static org.locationtech.spatial4j.distance.DistanceUtils.DEGREES_TO_RADIANS; @@ -63,11 +63,6 @@ protected RecursivePrefixTreeStrategy newRPT() { return rpt; } - @Override - protected boolean needsDocValues() { - return true;//due to SerializedDVStrategy - } - private void setupStrategy() { //setup setupGeohashGrid(); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java index 69f8c4da7abf..ac5ab9534557 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/vector/TestPointVectorStrategy.java @@ -16,9 +16,12 @@ */ package org.apache.lucene.spatial.vector; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.shape.Circle; -import org.locationtech.spatial4j.shape.Point; +import java.io.IOException; +import java.text.ParseException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.SpatialMatchConcern; import org.apache.lucene.spatial.StrategyTestCase; @@ -26,8 +29,9 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.junit.Before; import org.junit.Test; - -import java.io.IOException; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.shape.Circle; +import org.locationtech.spatial4j.shape.Point; public class TestPointVectorStrategy extends StrategyTestCase { @@ -36,11 +40,11 @@ public class TestPointVectorStrategy extends StrategyTestCase { public void setUp() throws Exception { super.setUp(); this.ctx = SpatialContext.GEO; - this.strategy = new PointVectorStrategy(ctx, getClass().getSimpleName()); } @Test public void testCircleShapeSupport() { + this.strategy = PointVectorStrategy.newInstance(ctx, getClass().getSimpleName()); Circle circle = ctx.makeCircle(ctx.makePoint(0, 0), 10); SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); Query query = this.strategy.makeQuery(args); @@ -50,6 +54,7 @@ public void testCircleShapeSupport() { @Test(expected = UnsupportedOperationException.class) public void testInvalidQueryShape() { + this.strategy = PointVectorStrategy.newInstance(ctx, getClass().getSimpleName()); Point point = ctx.makePoint(0, 0); SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, point); this.strategy.makeQuery(args); @@ -57,7 +62,45 @@ public void testInvalidQueryShape() { @Test public void testCitiesIntersectsBBox() throws IOException { + // note: does not require docValues + if (random().nextBoolean()) { + this.strategy = PointVectorStrategy.newInstance(ctx, getClass().getSimpleName()); + } else { + // switch to legacy instance sometimes, which has no docValues + this.strategy = PointVectorStrategy.newLegacyInstance(ctx, getClass().getSimpleName()); + } getAddAndVerifyIndexedDocuments(DATA_WORLD_CITIES_POINTS); executeQueries(SpatialMatchConcern.FILTER, QTEST_Cities_Intersects_BBox); } + + @Test + public void testFieldOptions() throws IOException, ParseException { + // It's not stored; test it isn't. + this.strategy = PointVectorStrategy.newInstance(ctx, getClass().getSimpleName()); + adoc("99", "POINT(-5.0 8.2)"); + commit(); + SearchResults results = executeQuery(new MatchAllDocsQuery(), 1); + Document document = results.results.get(0).document; + assertNull("not stored", document.getField(strategy.getFieldName() + PointVectorStrategy.SUFFIX_X)); + assertNull("not stored", document.getField(strategy.getFieldName() + PointVectorStrategy.SUFFIX_Y)); + deleteAll(); + + // Now we mark it stored. We also disable pointvalues... + FieldType fieldType = new FieldType(PointVectorStrategy.DEFAULT_FIELDTYPE); + fieldType.setStored(true); + fieldType.setDimensions(0, 0);//disable point values + this.strategy = new PointVectorStrategy(ctx, getClass().getSimpleName(), fieldType); + adoc("99", "POINT(-5.0 8.2)"); + commit(); + results = executeQuery(new MatchAllDocsQuery(), 1); + document = results.results.get(0).document; + assertEquals("stored", -5.0, document.getField(strategy.getFieldName() + PointVectorStrategy.SUFFIX_X).numericValue()); + assertEquals("stored", 8.2, document.getField(strategy.getFieldName() + PointVectorStrategy.SUFFIX_Y).numericValue()); + + // Test a query fails without point values + expectThrows(UnsupportedOperationException.class, () -> { + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(-10.0, 10.0, -5.0, 5.0)); + this.strategy.makeQuery(args); + }); + } } diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java index f655e68ce068..e41d3c6e4e0a 100644 --- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java +++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; -import org.locationtech.spatial4j.shape.Rectangle; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource; @@ -31,6 +30,7 @@ import org.apache.lucene.spatial.util.ShapeAreaValueSource; import org.apache.solr.common.SolrException; import org.apache.solr.search.QParser; +import org.locationtech.spatial4j.shape.Rectangle; public class BBoxField extends AbstractSpatialFieldType implements SchemaAware { private static final String PARAM_QUERY_TARGET_PROPORTION = "queryTargetProportion"; @@ -133,7 +133,6 @@ protected BBoxStrategy newSpatialStrategy(String fieldName) { registerSubFields(schema, fieldName, numberType, booleanType); } - BBoxStrategy strategy = new BBoxStrategy(ctx, fieldName); //Solr's FieldType ought to expose Lucene FieldType. Instead as a hack we create a Field with a dummy value. final SchemaField solrNumField = new SchemaField("_", numberType);//dummy temp org.apache.lucene.document.FieldType luceneType = @@ -145,8 +144,7 @@ protected BBoxStrategy newSpatialStrategy(String fieldName) { luceneType = new org.apache.lucene.document.FieldType(luceneType); luceneType.setDocValuesType(DocValuesType.NUMERIC); } - strategy.setFieldType(luceneType); - return strategy; + return new BBoxStrategy(ctx, fieldName, luceneType); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java index 99bf3febec22..18d80a35146c 100644 --- a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java @@ -16,12 +16,12 @@ */ package org.apache.solr.schema; -import org.apache.lucene.spatial.vector.PointVectorStrategy; - import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.lucene.spatial.vector.PointVectorStrategy; + /** * @see PointVectorStrategy * @lucene.experimental @@ -77,11 +77,24 @@ public void inform(IndexSchema schema) { } } + @Override + public org.apache.lucene.document.FieldType.LegacyNumericType getNumericType() { + return org.apache.lucene.document.FieldType.LegacyNumericType.DOUBLE; + } + @Override protected PointVectorStrategy newSpatialStrategy(String fieldName) { - PointVectorStrategy strategy = new PointVectorStrategy(ctx, fieldName); - strategy.setPrecisionStep(precisionStep); - return strategy; + // TODO update to how BBoxField does things + if (this.getNumericType() != null) { + // create strategy based on legacy numerics + // todo remove in 7.0 + org.apache.lucene.document.FieldType fieldType = + new org.apache.lucene.document.FieldType(PointVectorStrategy.LEGACY_FIELDTYPE); + fieldType.setNumericPrecisionStep(precisionStep); + return new PointVectorStrategy(ctx, fieldName, fieldType); + } else { + return PointVectorStrategy.newInstance(ctx, fieldName); + } } } From 2f9a966b709a1a658f88bc317fea3cf9a3955f31 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 31 Mar 2016 14:45:11 +0200 Subject: [PATCH 0247/1113] LUCENE-7156 - fixed precision and accuracy calculations (cherry picked from commit d08f327) --- .../utils/ConfusionMatrixGenerator.java | 41 +++++----- .../utils/ConfusionMatrixGeneratorTest.java | 75 +++++++++++-------- 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java index 17f5b21afc69..c9ecc4b3e822 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java @@ -106,11 +106,11 @@ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifi if (aLong != null) { stringLongMap.put(classified, aLong + 1); } else { - stringLongMap.put(classified, 1l); + stringLongMap.put(classified, 1L); } } else { stringLongMap = new HashMap<>(); - stringLongMap.put(classified, 1l); + stringLongMap.put(classified, 1L); counts.put(correctAnswer, stringLongMap); } @@ -225,23 +225,29 @@ public double getF1Measure(String klass) { */ public double getAccuracy() { if (this.accuracy == -1) { - double cc = 0d; - double wc = 0d; - for (Map.Entry> entry : linearizedMatrix.entrySet()) { - String correctAnswer = entry.getKey(); - for (Map.Entry classifiedAnswers : entry.getValue().entrySet()) { - Long value = classifiedAnswers.getValue(); - if (value != null) { - if (correctAnswer.equals(classifiedAnswers.getKey())) { - cc += value; - } else { - wc += value; - } + double tp = 0d; + double tn = 0d; + double fp = 0d; + double fn = 0d; + for (Map.Entry> classification : linearizedMatrix.entrySet()) { + String klass = classification.getKey(); + for (Map.Entry entry : classification.getValue().entrySet()) { + if (klass.equals(entry.getKey())) { + tp += entry.getValue(); + } else { + fn += entry.getValue(); + } + } + for (Map values : linearizedMatrix.values()) { + if (values.containsKey(klass)) { + fp += values.get(klass); + } else { + tn++; } } } - this.accuracy = cc / (cc + wc); + this.accuracy = (tp + tn) / (fp + fn + tp + tn); } return this.accuracy; } @@ -253,7 +259,7 @@ public double getAccuracy() { */ public double getPrecision() { double tp = 0; - double fp = -linearizedMatrix.size(); + double fp = 0; for (Map.Entry> classification : linearizedMatrix.entrySet()) { String klass = classification.getKey(); for (Map.Entry entry : classification.getValue().entrySet()) { @@ -268,8 +274,7 @@ public double getPrecision() { } } - return tp + fp > 0 ? tp / (tp + fp) : 0; - + return tp > 0 ? tp / (tp + fp) : 0; } /** diff --git a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java index e582b79d6159..d1966ec7c815 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java @@ -65,12 +65,15 @@ public List> getClasses(String text, int max) thr assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); double avgClassificationTime = confusionMatrix.getAvgClassificationTime(); assertTrue(avgClassificationTime >= 0d ); - assertTrue(confusionMatrix.getAccuracy() >= 0d); - assertTrue(confusionMatrix.getAccuracy() <= 1d); - assertTrue(confusionMatrix.getPrecision() >= 0d); - assertTrue(confusionMatrix.getPrecision() <= 1d); - assertTrue(confusionMatrix.getRecall() >= 0d); - assertTrue(confusionMatrix.getRecall() <= 1d); + double accuracy = confusionMatrix.getAccuracy(); + assertTrue(accuracy >= 0d); + assertTrue(accuracy <= 1d); + double precision = confusionMatrix.getPrecision(); + assertTrue(precision >= 0d); + assertTrue(precision <= 1d); + double recall = confusionMatrix.getRecall(); + assertTrue(recall >= 0d); + assertTrue(recall <= 1d); } finally { if (reader != null) { reader.close(); @@ -90,12 +93,15 @@ public void testGetConfusionMatrixWithSNB() throws Exception { assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); assertTrue(confusionMatrix.getAvgClassificationTime() >= 0d); - assertTrue(confusionMatrix.getAccuracy() >= 0d); - assertTrue(confusionMatrix.getAccuracy() <= 1d); - assertTrue(confusionMatrix.getPrecision() >= 0d); - assertTrue(confusionMatrix.getPrecision() <= 1d); - assertTrue(confusionMatrix.getRecall() >= 0d); - assertTrue(confusionMatrix.getRecall() <= 1d); + double accuracy = confusionMatrix.getAccuracy(); + assertTrue(accuracy >= 0d); + assertTrue(accuracy <= 1d); + double precision = confusionMatrix.getPrecision(); + assertTrue(precision >= 0d); + assertTrue(precision <= 1d); + double recall = confusionMatrix.getRecall(); + assertTrue(recall >= 0d); + assertTrue(recall <= 1d); } finally { if (reader != null) { reader.close(); @@ -115,12 +121,15 @@ public void testGetConfusionMatrixWithCNB() throws Exception { assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); assertTrue(confusionMatrix.getAvgClassificationTime() >= 0d); - assertTrue(confusionMatrix.getAccuracy() >= 0d); - assertTrue(confusionMatrix.getAccuracy() <= 1d); - assertTrue(confusionMatrix.getPrecision() >= 0d); - assertTrue(confusionMatrix.getPrecision() <= 1d); - assertTrue(confusionMatrix.getRecall() >= 0d); - assertTrue(confusionMatrix.getRecall() <= 1d); + double accuracy = confusionMatrix.getAccuracy(); + assertTrue(accuracy >= 0d); + assertTrue(accuracy <= 1d); + double precision = confusionMatrix.getPrecision(); + assertTrue(precision >= 0d); + assertTrue(precision <= 1d); + double recall = confusionMatrix.getRecall(); + assertTrue(recall >= 0d); + assertTrue(recall <= 1d); } finally { if (reader != null) { reader.close(); @@ -140,12 +149,15 @@ public void testGetConfusionMatrixWithKNN() throws Exception { assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); assertTrue(confusionMatrix.getAvgClassificationTime() >= 0d); - assertTrue(confusionMatrix.getAccuracy() >= 0d); - assertTrue(confusionMatrix.getAccuracy() <= 1d); - assertTrue(confusionMatrix.getPrecision() >= 0d); - assertTrue(confusionMatrix.getPrecision() <= 1d); - assertTrue(confusionMatrix.getRecall() >= 0d); - assertTrue(confusionMatrix.getRecall() <= 1d); + double accuracy = confusionMatrix.getAccuracy(); + assertTrue(accuracy >= 0d); + assertTrue(accuracy <= 1d); + double precision = confusionMatrix.getPrecision(); + assertTrue(precision >= 0d); + assertTrue(precision <= 1d); + double recall = confusionMatrix.getRecall(); + assertTrue(recall >= 0d); + assertTrue(recall <= 1d); } finally { if (reader != null) { reader.close(); @@ -165,12 +177,15 @@ public void testGetConfusionMatrixWithBP() throws Exception { assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); assertTrue(confusionMatrix.getAvgClassificationTime() >= 0d); - assertTrue(confusionMatrix.getAccuracy() >= 0d); - assertTrue(confusionMatrix.getAccuracy() <= 1d); - assertTrue(confusionMatrix.getPrecision() >= 0d); - assertTrue(confusionMatrix.getPrecision() <= 1d); - assertTrue(confusionMatrix.getRecall() >= 0d); - assertTrue(confusionMatrix.getRecall() <= 1d); + double accuracy = confusionMatrix.getAccuracy(); + assertTrue(accuracy >= 0d); + assertTrue(accuracy <= 1d); + double precision = confusionMatrix.getPrecision(); + assertTrue(precision >= 0d); + assertTrue(precision <= 1d); + double recall = confusionMatrix.getRecall(); + assertTrue(recall >= 0d); + assertTrue(recall <= 1d); assertTrue(confusionMatrix.getPrecision("true") >= 0d); assertTrue(confusionMatrix.getPrecision("true") <= 1d); assertTrue(confusionMatrix.getPrecision("false") >= 0d); From 0dcf822f40f685748525698e52f0030e88cb6de9 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 31 Mar 2016 11:53:57 -0400 Subject: [PATCH 0248/1113] LUCENE-7150: add Geo3DPoint.newDistance/Box/PolygonQuery --- lucene/CHANGES.txt | 3 + .../apache/lucene/spatial3d/Geo3DPoint.java | 167 +++++++++++++++++- .../lucene/spatial3d/TestGeo3DPoint.java | 20 +-- 3 files changed, 174 insertions(+), 16 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0e127acfd01d..7b7c9d38daae 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -18,6 +18,9 @@ API Changes * LUCENE-7141: Switch OfflineSorter's ByteSequencesReader to BytesRefIterator (Mike McCandless) +* LUCENE-7150: Spatial3d gets useful APIs to create common shape + queries, matching LatLonPoint. (Karl Wright via Mike McCandless) + Optimizations * LUCENE-7071: Reduce bytes copying in OfflineSorter, giving ~10% diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index 955a2bc0f36a..6b34518a769b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -16,12 +16,19 @@ */ package org.apache.lucene.spatial3d; +import java.util.List; +import java.util.ArrayList; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.PointValues; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -39,6 +46,12 @@ * @lucene.experimental */ public final class Geo3DPoint extends Field { + /** Mean radius of the earth, in meters */ + protected final static double MEAN_EARTH_RADIUS_METERS = 6371008.7714; + + /** How many radians are in one earth surface meter */ + protected final static double RADIANS_PER_METER = 1.0 / MEAN_EARTH_RADIUS_METERS; + /** Indexing {@link FieldType}. */ public static final FieldType TYPE = new FieldType(); static { @@ -47,21 +60,135 @@ public final class Geo3DPoint extends Field { } /** - * Creates a new Geo3DPoint field with the specified lat, lon (in radians). + * Creates a new Geo3DPoint field with the specified latitude, longitude (in degrees). * - * @throws IllegalArgumentException if the field name is null or lat or lon are out of bounds + * @throws IllegalArgumentException if the field name is null or latitude or longitude are out of bounds */ - public Geo3DPoint(String name, double lat, double lon) { + public Geo3DPoint(String name, double latitude, double longitude) { super(name, TYPE); - // Translate lat/lon to x,y,z: - final GeoPoint point = new GeoPoint(PlanetModel.WGS84, lat, lon); + checkLatitude(latitude); + checkLongitude(longitude); + // Translate latitude/longitude to x,y,z: + final GeoPoint point = new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude)); fillFieldsData(point.x, point.y, point.z); } + /** Converts degress to radians */ + protected static double fromDegrees(final double degrees) { + return Math.toRadians(degrees); + } + + /** Converts earth-surface meters to radians */ + protected static double fromMeters(final double meters) { + return meters * RADIANS_PER_METER; + } + + /** + * Create a query for matching points within the specified distance of the supplied location. + * @param field field name. must not be null. Note that because + * {@link PlanetModel#WGS84} is used, this query is approximate and may have up + * to 0.5% error. + * + * @param latitude latitude at the center: must be within standard +/-90 coordinate bounds. + * @param longitude longitude at the center: must be within standard +/-180 coordinate bounds. + * @param radiusMeters maximum distance from the center in meters: must be non-negative and finite. + * @return query matching points within this distance + * @throws IllegalArgumentException if {@code field} is null, location has invalid coordinates, or radius is invalid. + */ + public static Query newDistanceQuery(final String field, final double latitude, final double longitude, final double radiusMeters) { + checkLatitude(latitude); + checkLongitude(longitude); + final GeoShape shape = GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude), fromMeters(radiusMeters)); + return newShapeQuery(field, shape); + } + + /** + * Create a query for matching a box. + *

    + * The box may cross over the dateline. + * @param field field name. must not be null. + * @param minLatitude latitude lower bound: must be within standard +/-90 coordinate bounds. + * @param maxLatitude latitude upper bound: must be within standard +/-90 coordinate bounds. + * @param minLongitude longitude lower bound: must be within standard +/-180 coordinate bounds. + * @param maxLongitude longitude upper bound: must be within standard +/-180 coordinate bounds. + * @return query matching points within this box + * @throws IllegalArgumentException if {@code field} is null, or the box has invalid coordinates. + */ + public static Query newBoxQuery(final String field, final double minLatitude, final double maxLatitude, final double minLongitude, final double maxLongitude) { + checkLatitude(minLatitude); + checkLongitude(minLongitude); + checkLatitude(maxLatitude); + checkLongitude(maxLongitude); + final GeoShape shape = GeoBBoxFactory.makeGeoBBox(PlanetModel.WGS84, + fromDegrees(maxLatitude), fromDegrees(minLatitude), fromDegrees(minLongitude), fromDegrees(maxLongitude)); + return newShapeQuery(field, shape); + } + + /** + * Create a query for matching a polygon. + *

    + * The supplied {@code polyLatitudes}/{@code polyLongitudes} must be clockwise or counter-clockwise. + * @param field field name. must not be null. + * @param polyLatitudes latitude values for points of the polygon: must be within standard +/-90 coordinate bounds. + * @param polyLongitudes longitude values for points of the polygon: must be within standard +/-180 coordinate bounds. + * @return query matching points within this polygon + */ + public static Query newPolygonQuery(final String field, final double[] polyLatitudes, final double[] polyLongitudes) { + if (polyLatitudes.length != polyLongitudes.length) { + throw new IllegalArgumentException("same number of latitudes and longitudes required"); + } + if (polyLatitudes.length < 4) { + throw new IllegalArgumentException("need three or more points"); + } + if (polyLatitudes[0] != polyLatitudes[polyLatitudes.length-1] || polyLongitudes[0] != polyLongitudes[polyLongitudes.length-1]) { + throw new IllegalArgumentException("last point must equal first point"); + } + + final List polyPoints = new ArrayList<>(polyLatitudes.length-1); + for (int i = 0; i < polyLatitudes.length-1; i++) { + final double latitude = polyLatitudes[i]; + final double longitude = polyLongitudes[i]; + checkLatitude(latitude); + checkLongitude(longitude); + polyPoints.add(new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude))); + } + // We don't know what the sense of the polygon is without providing the index of one vertex we know to be convex. + // Since that doesn't fit with the "super-simple API" requirements, we just use the index of the first one, and people have to just + // know to do it that way. + final int convexPointIndex = 0; + final GeoShape shape = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, polyPoints, convexPointIndex); + return newShapeQuery(field, shape); + } + + /** + * Create a query for matching a path. + *

    + * @param field field name. must not be null. + * @param pathLatitudes latitude values for points of the path: must be within standard +/-90 coordinate bounds. + * @param pathLongitudes longitude values for points of the path: must be within standard +/-180 coordinate bounds. + * @param pathWidthMeters width of the path in meters. + * @return query matching points within this polygon + */ + public static Query newPathQuery(final String field, final double[] pathLatitudes, final double[] pathLongitudes, final double pathWidthMeters) { + if (pathLatitudes.length != pathLongitudes.length) { + throw new IllegalArgumentException("same number of latitudes and longitudes required"); + } + final GeoPoint[] points = new GeoPoint[pathLatitudes.length]; + for (int i = 0; i < pathLatitudes.length; i++) { + final double latitude = pathLatitudes[i]; + final double longitude = pathLongitudes[i]; + checkLatitude(latitude); + checkLongitude(longitude); + points[i] = new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude)); + } + final GeoShape shape = new GeoPath(PlanetModel.WGS84, fromMeters(pathWidthMeters), points); + return newShapeQuery(field, shape); + } + /** * Creates a new Geo3DPoint field with the specified x,y,z. * - * @throws IllegalArgumentException if the field name is null or lat or lon are out of bounds + * @throws IllegalArgumentException if the field name is null or latitude or longitude are out of bounds */ public Geo3DPoint(String name, double x, double y, double z) { super(name, TYPE); @@ -112,4 +239,32 @@ public String toString() { result.append('>'); return result.toString(); } + + // TODO LUCENE-7152: share this with GeoUtils.java from spatial module + + /** Minimum longitude value. */ + private static final double MIN_LON_INCL = -180.0D; + + /** Maximum longitude value. */ + private static final double MAX_LON_INCL = 180.0D; + + /** Minimum latitude value. */ + private static final double MIN_LAT_INCL = -90.0D; + + /** Maximum latitude value. */ + private static final double MAX_LAT_INCL = 90.0D; + + /** validates latitude value is within standard +/-90 coordinate bounds */ + private static void checkLatitude(double latitude) { + if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) { + throw new IllegalArgumentException("invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL); + } + } + + /** validates longitude value is within standard +/-180 coordinate bounds */ + private static void checkLongitude(double longitude) { + if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) { + throw new IllegalArgumentException("invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL); + } + } } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 5c0044f43714..ccaf0937f730 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -115,7 +115,7 @@ public void testBasic() throws Exception { iwc.setCodec(getCodec()); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); - doc.add(new Geo3DPoint("field", toRadians(50.7345267), toRadians(-97.5303555))); + doc.add(new Geo3DPoint("field", 50.7345267, -97.5303555)); w.addDocument(doc); IndexReader r = DirectoryReader.open(w); // We can't wrap with "exotic" readers because the query must see the BKD3DDVFormat: @@ -128,7 +128,7 @@ public void testBasic() throws Exception { } private static double toRadians(double degrees) { - return Math.PI*(degrees/360.0); + return Math.PI*(degrees/180.0); } private static PlanetModel getPlanetModel() { @@ -508,13 +508,13 @@ private void doTestRandom(int count) throws Exception { if (x == 0) { // Identical lat to old point lats[docID] = lats[oldDocID]; - lons[docID] = toRadians(randomLon()); + lons[docID] = randomLon(); if (VERBOSE) { System.err.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + lons[docID] + " (same lat as doc=" + oldDocID + ")"); } } else if (x == 1) { // Identical lon to old point - lats[docID] = toRadians(randomLat()); + lats[docID] = randomLat(); lons[docID] = lons[oldDocID]; if (VERBOSE) { System.err.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + lons[docID] + " (same lon as doc=" + oldDocID + ")"); @@ -529,8 +529,8 @@ private void doTestRandom(int count) throws Exception { } } } else { - lats[docID] = toRadians(randomLat()); - lons[docID] = toRadians(randomLon()); + lats[docID] = randomLat(); + lons[docID] = randomLon(); haveRealDoc = true; if (VERBOSE) { System.err.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + lons[docID]); @@ -759,7 +759,7 @@ public void collect(int doc) { if (Double.isNaN(lats[id]) == false) { // Accurate point: - GeoPoint point1 = new GeoPoint(PlanetModel.WGS84, lats[id], lons[id]); + GeoPoint point1 = new GeoPoint(PlanetModel.WGS84, toRadians(lats[id]), toRadians(lons[id])); // Quantized point (32 bits per dim): GeoPoint point2 = quantize(PlanetModel.WGS84.getMaximumMagnitude(), point1); @@ -795,12 +795,12 @@ public void collect(int doc) { } public void testToString() { - Geo3DPoint point = new Geo3DPoint("point", toRadians(44.244272), toRadians(7.769736)); - assertEquals("Geo3DPoint ", point.toString()); + Geo3DPoint point = new Geo3DPoint("point", 44.244272, 7.769736); + assertEquals("Geo3DPoint ", point.toString()); } public void testShapeQueryToString() { - assertEquals("PointInGeo3DShapeQuery: field=point: Shape: GeoStandardCircle: {planetmodel=PlanetModel.WGS84, center=[lat=0.3861041107739683, lon=0.06780373760536706], radius=0.1(5.729577951308232)}", + assertEquals("PointInGeo3DShapeQuery: field=point: Shape: GeoStandardCircle: {planetmodel=PlanetModel.WGS84, center=[lat=0.7722082215479366, lon=0.13560747521073413], radius=0.1(5.729577951308232)}", Geo3DPoint.newShapeQuery("point", GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, toRadians(44.244272), toRadians(7.769736), 0.1)).toString()); } From 74b5a7a9722523980630211d416883d76b804f90 Mon Sep 17 00:00:00 2001 From: jbernste Date: Thu, 31 Mar 2016 11:39:55 -0400 Subject: [PATCH 0249/1113] SOLR-8924: RollupStream breaks with null values in the group by buckets --- .../solrj/io/stream/metrics/Bucket.java | 11 ++++- .../client/solrj/io/stream/StreamingTest.java | 48 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/metrics/Bucket.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/metrics/Bucket.java index 12ee2815e5d6..8e2293c18015 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/metrics/Bucket.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/metrics/Bucket.java @@ -22,6 +22,7 @@ public class Bucket implements Serializable { private static final long serialVersionUID = 1; + private static final String NULL_VALUE = "NULL"; private String bucketKey; @@ -33,8 +34,14 @@ public Bucket(String bucketKey) { this.bucketKey = bucketKey; } - public Object getBucketValue(Tuple tuple) { - return tuple.get(bucketKey); + public Object getBucketValue(Tuple tuple) + { + Object o = tuple.get(bucketKey); + if(o == null) { + return NULL_VALUE; + } else { + return o; + } } public String toString() { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java index 3091a19083fb..61253e1d9ee6 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java @@ -1364,6 +1364,52 @@ private void testRollupStream() throws Exception { assertTrue(count.doubleValue() == 2); + //Test will null value in the grouping field + indexr(id, "12", "a_s", null, "a_i", "14", "a_f", "10"); + commit(); + + paramsA = mapParams("q","*:*","fl","a_s,a_i,a_f","sort", "a_s asc", "qt", "/export"); + stream = new CloudSolrStream(zkHost, "collection1", paramsA); + + Bucket[] buckets1 = {new Bucket("a_s")}; + + Metric[] metrics1 = {new SumMetric("a_i"), + new SumMetric("a_f"), + new MinMetric("a_i"), + new MinMetric("a_f"), + new MaxMetric("a_i"), + new MaxMetric("a_f"), + new MeanMetric("a_i"), + new MeanMetric("a_f"), + new CountMetric()}; + + rollupStream = new RollupStream(stream, buckets1, metrics1); + tuples = getTuples(rollupStream); + //Check that we've got the extra NULL bucket + assert(tuples.size() == 4); + tuple = tuples.get(0); + assert(tuple.getString("a_s").equals("NULL")); + + sumi = tuple.getDouble("sum(a_i)"); + sumf = tuple.getDouble("sum(a_f)"); + mini = tuple.getDouble("min(a_i)"); + minf = tuple.getDouble("min(a_f)"); + maxi = tuple.getDouble("max(a_i)"); + maxf = tuple.getDouble("max(a_f)"); + avgi = tuple.getDouble("avg(a_i)"); + avgf = tuple.getDouble("avg(a_f)"); + count = tuple.getDouble("count(*)"); + + assertTrue(sumi.doubleValue() == 14.0D); + assertTrue(sumf.doubleValue() == 10.0D); + assertTrue(mini.doubleValue() == 14.0D); + assertTrue(minf.doubleValue() == 10.0D); + assertTrue(maxi.doubleValue() == 14.0D); + assertTrue(maxf.doubleValue() == 10.0D); + assertTrue(avgi.doubleValue() == 14.0D); + assertTrue(avgf.doubleValue() == 10.0D); + assertTrue(count.doubleValue() == 1); + del("*:*"); commit(); @@ -1577,7 +1623,7 @@ private void testZeroParallelReducerStream() throws Exception { String zkHost = zkServer.getZkAddress(); streamFactory.withCollectionZkHost("collection1", zkHost); - Map paramsA = mapParams("q","blah","fl","id,a_s,a_i,a_f","sort", "a_s asc,a_f asc", "partitionKeys", "a_s"); + Map paramsA = mapParams("q", "blah", "fl", "id,a_s,a_i,a_f","sort", "a_s asc,a_f asc", "partitionKeys", "a_s"); CloudSolrStream stream = new CloudSolrStream(zkHost, "collection1", paramsA); ReducerStream rstream = new ReducerStream(stream, new FieldEqualitor("a_s"), From 82ed2383f0c84787023bf2329217980207a91cf8 Mon Sep 17 00:00:00 2001 From: nknize Date: Thu, 31 Mar 2016 13:40:04 -0500 Subject: [PATCH 0250/1113] replace svn up with git pull in buildAndPushRelease.py --- dev-tools/scripts/buildAndPushRelease.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py index 396a0765253b..62d5f6515c2a 100644 --- a/dev-tools/scripts/buildAndPushRelease.py +++ b/dev-tools/scripts/buildAndPushRelease.py @@ -73,8 +73,8 @@ def prepare(root, version, gpgKeyID, gpgPassword): os.remove(LOG) os.chdir(root) - print(' svn up...') - run('svn up') + print(' git pull...') + run('git pull') rev = getGitRev() print(' git rev: %s' % rev) @@ -236,7 +236,7 @@ def parse_config(): if config.rc_num <= 0: parser.error('Release Candidate number must be a positive integer') if not os.path.isdir(config.root): - # TODO: add additional svn check to ensure dir is a real lucene-solr checkout + # TODO: add additional git check to ensure dir is a real lucene-solr checkout parser.error('Root path is not a valid lucene-solr checkout') config.version = read_version(config.root) From ffdfceba5371b1c3f96b44c727025f2f27bbf12b Mon Sep 17 00:00:00 2001 From: jbernste Date: Thu, 31 Mar 2016 16:23:59 -0400 Subject: [PATCH 0251/1113] SOLR-8888: Add shortestPath Streaming Expression --- .../apache/solr/handler/StreamHandler.java | 2 + .../solrj/io/graph/ShortestPathStream.java | 490 ++++++++++++++++++ .../client/solrj/io/graph/package-info.java | 22 + .../solrj/io/graph/GraphExpressionTest.java | 404 +++++++++++++++ .../solr/client/solrj/io/graph/GraphTest.java | 387 ++++++++++++++ 5 files changed, 1305 insertions(+) create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/package-info.java create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index c7bac9f68532..226058ee0c00 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -28,6 +28,7 @@ import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.graph.ShortestPathStream; import org.apache.solr.client.solrj.io.ops.ConcatOperation; import org.apache.solr.client.solrj.io.ops.DistinctOperation; import org.apache.solr.client.solrj.io.ops.GroupOperation; @@ -115,6 +116,7 @@ public void inform(SolrCore core) { .withFunctionName("complement", ComplementStream.class) .withFunctionName("daemon", DaemonStream.class) .withFunctionName("topic", TopicStream.class) + .withFunctionName("shortestPath", ShortestPathStream.class) // metrics diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java new file mode 100644 index 000000000000..bb9b09df3e8c --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/ShortestPathStream.java @@ -0,0 +1,490 @@ +package org.apache.solr.client.solrj.io.graph; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.ArrayList; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.solr.client.solrj.io.eq.MultipleFieldEqualitor; +import org.apache.solr.client.solrj.io.stream.*; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.eq.FieldEqualitor; +import org.apache.solr.client.solrj.io.stream.expr.Expressible; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrjNamedThreadFactory; + +public class ShortestPathStream extends TupleStream implements Expressible { + + private static final long serialVersionUID = 1; + + private String fromNode; + private String toNode; + private String fromField; + private String toField; + private int joinBatchSize; + private int maxDepth; + private String zkHost; + private String collection; + private LinkedList shortestPaths = new LinkedList(); + private boolean found; + private StreamContext streamContext; + private int threads; + private Map queryParams; + + public ShortestPathStream(String zkHost, + String collection, + String fromNode, + String toNode, + String fromField, + String toField, + Map queryParams, + int joinBatchSize, + int threads, + int maxDepth) { + + init(zkHost, + collection, + fromNode, + toNode, + fromField, + toField, + queryParams, + joinBatchSize, + threads, + maxDepth); + } + + public ShortestPathStream(StreamExpression expression, StreamFactory factory) throws IOException { + + String collectionName = factory.getValueOperand(expression, 0); + List namedParams = factory.getNamedOperands(expression); + StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression, "zkHost"); + + // Collection Name + if(null == collectionName) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - collectionName expected as first operand",expression)); + } + + String fromNode = null; + StreamExpressionNamedParameter fromExpression = factory.getNamedOperand(expression, "from"); + + if(fromExpression == null) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - from param is required",expression)); + } else { + fromNode = ((StreamExpressionValue)fromExpression.getParameter()).getValue(); + } + + String toNode = null; + StreamExpressionNamedParameter toExpression = factory.getNamedOperand(expression, "to"); + + if(toExpression == null) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - to param is required", expression)); + } else { + toNode = ((StreamExpressionValue)toExpression.getParameter()).getValue(); + } + + String fromField = null; + String toField = null; + + StreamExpressionNamedParameter edgeExpression = factory.getNamedOperand(expression, "edge"); + + if(edgeExpression == null) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - edge param is required", expression)); + } else { + String edge = ((StreamExpressionValue)edgeExpression.getParameter()).getValue(); + String[] fields = edge.split("="); + if(fields.length != 2) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - edge param separated by and = and must contain two fields", expression)); + } + fromField = fields[0].trim(); + toField = fields[1].trim(); + } + + int threads = 6; + + StreamExpressionNamedParameter threadsExpression = factory.getNamedOperand(expression, "threads"); + + if(threadsExpression != null) { + threads = Integer.parseInt(((StreamExpressionValue)threadsExpression.getParameter()).getValue()); + } + + int partitionSize = 250; + + StreamExpressionNamedParameter partitionExpression = factory.getNamedOperand(expression, "partitionSize"); + + if(partitionExpression != null) { + partitionSize = Integer.parseInt(((StreamExpressionValue)partitionExpression.getParameter()).getValue()); + } + + int maxDepth = 0; + + StreamExpressionNamedParameter depthExpression = factory.getNamedOperand(expression, "maxDepth"); + + if(depthExpression == null) { + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - maxDepth param is required", expression)); + } else { + maxDepth = Integer.parseInt(((StreamExpressionValue) depthExpression.getParameter()).getValue()); + } + + Map params = new HashMap(); + for(StreamExpressionNamedParameter namedParam : namedParams){ + if(!namedParam.getName().equals("zkHost") && + !namedParam.getName().equals("to") && + !namedParam.getName().equals("from") && + !namedParam.getName().equals("edge") && + !namedParam.getName().equals("maxDepth") && + !namedParam.getName().equals("threads") && + !namedParam.getName().equals("partitionSize")) + { + params.put(namedParam.getName(), namedParam.getParameter().toString().trim()); + } + } + + // zkHost, optional - if not provided then will look into factory list to get + String zkHost = null; + if(null == zkHostExpression){ + zkHost = factory.getCollectionZkHost(collectionName); + if(zkHost == null) { + zkHost = factory.getDefaultZkHost(); + } + } else if(zkHostExpression.getParameter() instanceof StreamExpressionValue) { + zkHost = ((StreamExpressionValue)zkHostExpression.getParameter()).getValue(); + } + + if(null == zkHost){ + throw new IOException(String.format(Locale.ROOT,"invalid expression %s - zkHost not found for collection '%s'",expression,collectionName)); + } + + // We've got all the required items + init(zkHost, collectionName, fromNode, toNode, fromField, toField, params, partitionSize, threads, maxDepth); + } + + private void init(String zkHost, + String collection, + String fromNode, + String toNode, + String fromField, + String toField, + Map queryParams, + int joinBatchSize, + int threads, + int maxDepth) { + this.zkHost = zkHost; + this.collection = collection; + this.fromNode = fromNode; + this.toNode = toNode; + this.fromField = fromField; + this.toField = toField; + this.queryParams = queryParams; + this.joinBatchSize = joinBatchSize; + this.threads = threads; + this.maxDepth = maxDepth; + } + + @Override + public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException { + + StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass())); + + // collection + expression.addParameter(collection); + + Set entries = queryParams.entrySet(); + // parameters + for(Map.Entry param : entries){ + String value = param.getValue().toString(); + + // SOLR-8409: This is a special case where the params contain a " character + // Do note that in any other BASE streams with parameters where a " might come into play + // that this same replacement needs to take place. + value = value.replace("\"", "\\\""); + + expression.addParameter(new StreamExpressionNamedParameter(param.getKey().toString(), value)); + } + + expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost)); + expression.addParameter(new StreamExpressionNamedParameter("maxDepth", Integer.toString(maxDepth))); + expression.addParameter(new StreamExpressionNamedParameter("threads", Integer.toString(threads))); + expression.addParameter(new StreamExpressionNamedParameter("partitionSize", Integer.toString(joinBatchSize))); + expression.addParameter(new StreamExpressionNamedParameter("from", fromNode)); + expression.addParameter(new StreamExpressionNamedParameter("to", toNode)); + expression.addParameter(new StreamExpressionNamedParameter("edge", fromField+"="+toField)); + return expression; + } + + public void setStreamContext(StreamContext context) { + this.streamContext = context; + } + + public List children() { + List l = new ArrayList(); + return l; + } + + public void open() throws IOException { + + List>> allVisited = new ArrayList(); + Map visited = new HashMap(); + visited.put(this.fromNode, null); + + allVisited.add(visited); + int depth = 0; + Map> nextVisited = null; + List targets = new ArrayList(); + ExecutorService threadPool = null; + + try { + + threadPool = ExecutorUtil.newMDCAwareFixedThreadPool(threads, new SolrjNamedThreadFactory("ShortestPathStream")); + + //Breadth first search + TRAVERSE: + while (targets.size() == 0 && depth < maxDepth) { + Set nodes = visited.keySet(); + Iterator it = nodes.iterator(); + nextVisited = new HashMap(); + int batchCount = 0; + List queryNodes = new ArrayList(); + List futures = new ArrayList(); + JOIN: + //Queue up all the batches + while (it.hasNext()) { + String node = it.next(); + queryNodes.add(node); + ++batchCount; + if (batchCount == joinBatchSize || !it.hasNext()) { + try { + JoinRunner joinRunner = new JoinRunner(queryNodes); + Future> future = threadPool.submit(joinRunner); + futures.add(future); + } catch (Exception e) { + throw new RuntimeException(e); + } + batchCount = 0; + queryNodes = new ArrayList(); + } + } + + try { + //Process the batches as they become available + OUTER: + for (Future> future : futures) { + List edges = future.get(); + INNER: + for (Edge edge : edges) { + if (toNode.equals(edge.to)) { + targets.add(edge); + if(nextVisited.containsKey(edge.to)) { + List parents = nextVisited.get(edge.to); + parents.add(edge.from); + } else { + List parents = new ArrayList(); + parents.add(edge.from); + nextVisited.put(edge.to, parents); + } + } else { + if (!cycle(edge.to, allVisited)) { + if(nextVisited.containsKey(edge.to)) { + List parents = nextVisited.get(edge.to); + parents.add(edge.from); + } else { + List parents = new ArrayList(); + parents.add(edge.from); + nextVisited.put(edge.to, parents); + } + } + } + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + allVisited.add(nextVisited); + visited = nextVisited; + ++depth; + } + } finally { + threadPool.shutdown(); + } + + Set finalPaths = new HashSet(); + if(targets.size() > 0) { + for(Edge edge : targets) { + List paths = new ArrayList(); + LinkedList path = new LinkedList(); + path.addFirst(edge.to); + paths.add(path); + //Walk back up the tree a collect the parent nodes. + INNER: + for (int i = allVisited.size() - 1; i >= 0; --i) { + Map> v = allVisited.get(i); + Iterator it = paths.iterator(); + List newPaths = new ArrayList(); + while(it.hasNext()) { + LinkedList p = it.next(); + List parents = v.get(p.peekFirst()); + if (parents != null) { + for(String parent : parents) { + LinkedList newPath = new LinkedList(); + newPath.addAll(p); + newPath.addFirst(parent); + newPaths.add(newPath); + } + paths = newPaths; + } + } + } + + for(LinkedList p : paths) { + String s = p.toString(); + if (!finalPaths.contains(s)){ + Tuple shortestPath = new Tuple(new HashMap()); + shortestPath.put("path", p); + shortestPaths.add(shortestPath); + finalPaths.add(s); + } + } + } + } + } + + private class JoinRunner implements Callable> { + + private List nodes; + private List edges = new ArrayList(); + + public JoinRunner(List nodes) { + this.nodes = nodes; + } + + public List call() { + + Map joinParams = new HashMap(); + String fl = fromField + "," + toField; + + joinParams.putAll(queryParams); + joinParams.put("fl", fl); + joinParams.put("qt", "/export"); + joinParams.put("sort", toField + " asc,"+fromField +" asc"); + + StringBuffer nodeQuery = new StringBuffer(); + + for(String node : nodes) { + nodeQuery.append(node).append(" "); + } + + String q = fromField + ":(" + nodeQuery.toString().trim() + ")"; + + joinParams.put("q", q); + TupleStream stream = null; + try { + stream = new UniqueStream(new CloudSolrStream(zkHost, collection, joinParams), new MultipleFieldEqualitor(new FieldEqualitor(toField), new FieldEqualitor(fromField))); + stream.setStreamContext(streamContext); + stream.open(); + BATCH: + while (true) { + Tuple tuple = stream.read(); + if (tuple.EOF) { + break BATCH; + } + String _toNode = tuple.getString(toField); + String _fromNode = tuple.getString(fromField); + Edge edge = new Edge(_fromNode, _toNode); + edges.add(edge); + } + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + try { + stream.close(); + } catch(Exception ce) { + throw new RuntimeException(ce); + } + } + return edges; + } + } + + private class Edge { + + private String from; + private String to; + + public Edge(String from, String to) { + this.from = from; + this.to = to; + } + } + + private boolean cycle(String node, List>> allVisited) { + //Check all visited trees for each level to see if we've encountered this node before. + for(Map> visited : allVisited) { + if(visited.containsKey(node)) { + return true; + } + } + + return false; + } + + public void close() throws IOException { + this.found = false; + } + + public Tuple read() throws IOException { + if(shortestPaths.size() > 0) { + found = true; + Tuple t = shortestPaths.removeFirst(); + return t; + } else { + Map m = new HashMap(); + m.put("EOF", true); + if(!found) { + m.put("sorry", "No path found"); + } + return new Tuple(m); + } + } + + public int getCost() { + return 0; + } + + @Override + public StreamComparator getStreamSort() { + return null; + } +} \ No newline at end of file diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/package-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/package-info.java new file mode 100644 index 000000000000..b34e0dd0e07d --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/graph/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Streaming Graph Traversals + **/ +package org.apache.solr.client.solrj.io.graph; + diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java new file mode 100644 index 000000000000..db58a905f7ca --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java @@ -0,0 +1,404 @@ +package org.apache.solr.client.solrj.io.graph; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.*; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.cloud.AbstractZkTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * All base tests will be done with CloudSolrStream. Under the covers CloudSolrStream uses SolrStream so + * SolrStream will get fully exercised through these tests. + * + **/ + +@Slow +@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"}) +public class GraphExpressionTest extends AbstractFullDistribZkTestBase { + + private static final String SOLR_HOME = getFile("solrj" + File.separator + "solr").getAbsolutePath(); + + static { + schemaString = "schema-streaming.xml"; + } + + @BeforeClass + public static void beforeSuperClass() { + AbstractZkTestCase.SOLRHOME = new File(SOLR_HOME()); + } + + @AfterClass + public static void afterSuperClass() { + + } + + protected String getCloudSolrConfig() { + return "solrconfig-streaming.xml"; + } + + + @Override + public String getSolrHome() { + return SOLR_HOME; + } + + public static String SOLR_HOME() { + return SOLR_HOME; + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + // we expect this time of exception as shards go up and down... + //ignoreException(".*"); + + System.setProperty("numShards", Integer.toString(sliceCount)); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + resetExceptionIgnores(); + } + + public GraphExpressionTest() { + super(); + sliceCount = 2; + } + + @Test + public void testAll() throws Exception{ + assertNotNull(cloudClient); + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForRecoveriesToFinish(false); + + del("*:*"); + commit(); + + testShortestPathStream(); + } + + private void testShortestPathStream() throws Exception { + + indexr(id, "0", "from_s", "jim", "to_s", "mike", "predicate_s", "knows"); + indexr(id, "1", "from_s", "jim", "to_s", "dave", "predicate_s", "knows"); + indexr(id, "2", "from_s", "jim", "to_s", "stan", "predicate_s", "knows"); + indexr(id, "3", "from_s", "dave", "to_s", "stan", "predicate_s", "knows"); + indexr(id, "4", "from_s", "dave", "to_s", "bill", "predicate_s", "knows"); + indexr(id, "5", "from_s", "dave", "to_s", "mike", "predicate_s", "knows"); + indexr(id, "20", "from_s", "dave", "to_s", "alex", "predicate_s", "knows"); + indexr(id, "21", "from_s", "alex", "to_s", "steve", "predicate_s", "knows"); + indexr(id, "6", "from_s", "stan", "to_s", "alice", "predicate_s", "knows"); + indexr(id, "7", "from_s", "stan", "to_s", "mary", "predicate_s", "knows"); + indexr(id, "8", "from_s", "stan", "to_s", "dave", "predicate_s", "knows"); + indexr(id, "10", "from_s", "mary", "to_s", "mike", "predicate_s", "knows"); + indexr(id, "11", "from_s", "mary", "to_s", "max", "predicate_s", "knows"); + indexr(id, "12", "from_s", "mary", "to_s", "jim", "predicate_s", "knows"); + indexr(id, "13", "from_s", "mary", "to_s", "steve", "predicate_s", "knows"); + + commit(); + + List tuples = null; + Set paths = null; + ShortestPathStream stream = null; + StreamContext context = new StreamContext(); + SolrClientCache cache = new SolrClientCache(); + context.setSolrClientCache(cache); + + StreamFactory factory = new StreamFactory() + .withCollectionZkHost("collection1", zkServer.getZkAddress()) + .withFunctionName("shortestPath", ShortestPathStream.class); + + Map params = new HashMap(); + params.put("fq", "predicate_s:knows"); + + stream = (ShortestPathStream)factory.constructStream("shortestPath(collection1, " + + "from=\"jim\", " + + "to=\"steve\"," + + "edge=\"from_s=to_s\"," + + "fq=\"predicate_s:knows\","+ + "threads=\"3\","+ + "partitionSize=\"3\","+ + "maxDepth=\"6\")"); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 2); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, dave, alex, steve]")); + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + //Test with batch size of 1 + + params.put("fq", "predicate_s:knows"); + + stream = (ShortestPathStream)factory.constructStream("shortestPath(collection1, " + + "from=\"jim\", " + + "to=\"steve\"," + + "edge=\"from_s=to_s\"," + + "fq=\"predicate_s:knows\","+ + "threads=\"3\","+ + "partitionSize=\"1\","+ + "maxDepth=\"6\")"); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 2); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, dave, alex, steve]")); + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + //Test with bad predicate + + + stream = (ShortestPathStream)factory.constructStream("shortestPath(collection1, " + + "from=\"jim\", " + + "to=\"steve\"," + + "edge=\"from_s=to_s\"," + + "fq=\"predicate_s:crap\","+ + "threads=\"3\","+ + "partitionSize=\"3\","+ + "maxDepth=\"6\")"); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 0); + + //Test with depth 2 + + stream = (ShortestPathStream)factory.constructStream("shortestPath(collection1, " + + "from=\"jim\", " + + "to=\"steve\"," + + "edge=\"from_s=to_s\"," + + "fq=\"predicate_s:knows\","+ + "threads=\"3\","+ + "partitionSize=\"3\","+ + "maxDepth=\"2\")"); + + + stream.setStreamContext(context); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 0); + + //Take out alex + params.put("fq", "predicate_s:knows NOT to_s:alex"); + + stream = (ShortestPathStream)factory.constructStream("shortestPath(collection1, " + + "from=\"jim\", " + + "to=\"steve\"," + + "edge=\"from_s=to_s\"," + + "fq=\" predicate_s:knows NOT to_s:alex\","+ + "threads=\"3\","+ + "partitionSize=\"3\","+ + "maxDepth=\"6\")"); + + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + assertTrue(tuples.size() == 1); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + cache.close(); + del("*:*"); + commit(); + } + + protected List getTuples(TupleStream tupleStream) throws IOException { + tupleStream.open(); + List tuples = new ArrayList(); + for(Tuple t = tupleStream.read(); !t.EOF; t = tupleStream.read()) { + tuples.add(t); + } + tupleStream.close(); + return tuples; + } + protected boolean assertOrder(List tuples, int... ids) throws Exception { + return assertOrderOf(tuples, "id", ids); + } + protected boolean assertOrderOf(List tuples, String fieldName, int... ids) throws Exception { + int i = 0; + for(int val : ids) { + Tuple t = tuples.get(i); + Long tip = (Long)t.get(fieldName); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + protected boolean assertMapOrder(List tuples, int... ids) throws Exception { + int i = 0; + for(int val : ids) { + Tuple t = tuples.get(i); + List tip = t.getMaps("group"); + int id = (int)tip.get(0).get("id"); + if(id != val) { + throw new Exception("Found value:"+id+" expecting:"+val); + } + ++i; + } + return true; + } + + + protected boolean assertFields(List tuples, String ... fields) throws Exception{ + for(Tuple tuple : tuples){ + for(String field : fields){ + if(!tuple.fields.containsKey(field)){ + throw new Exception(String.format(Locale.ROOT, "Expected field '%s' not found", field)); + } + } + } + return true; + } + protected boolean assertNotFields(List tuples, String ... fields) throws Exception{ + for(Tuple tuple : tuples){ + for(String field : fields){ + if(tuple.fields.containsKey(field)){ + throw new Exception(String.format(Locale.ROOT, "Unexpected field '%s' found", field)); + } + } + } + return true; + } + + protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception { + List group = (List)tuple.get("tuples"); + int i=0; + for(int val : ids) { + Map t = (Map)group.get(i); + Long tip = (Long)t.get("id"); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception { + long lv = (long)tuple.get(fieldName); + if(lv != l) { + throw new Exception("Longs not equal:"+l+" : "+lv); + } + + return true; + } + + public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception { + String actual = (String)tuple.get(fieldName); + + if( (null == expected && null != actual) || + (null != expected && null == actual) || + (null != expected && !expected.equals(actual))){ + throw new Exception("Longs not equal:"+expected+" : "+actual); + } + + return true; + } + + protected boolean assertMaps(List maps, int... ids) throws Exception { + if(maps.size() != ids.length) { + throw new Exception("Expected id count != actual map count:"+ids.length+":"+maps.size()); + } + + int i=0; + for(int val : ids) { + Map t = maps.get(i); + Long tip = (Long)t.get("id"); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + private boolean assertList(List list, Object... vals) throws Exception { + + if(list.size() != vals.length) { + throw new Exception("Lists are not the same size:"+list.size() +" : "+vals.length); + } + + for(int i=0; i tuples = null; + Set paths = null; + ShortestPathStream stream = null; + String zkHost = zkServer.getZkAddress(); + StreamContext context = new StreamContext(); + SolrClientCache cache = new SolrClientCache(); + context.setSolrClientCache(cache); + + Map params = new HashMap(); + params.put("fq", "predicate_s:knows"); + + stream = new ShortestPathStream(zkHost, + "collection1", + "jim", + "steve", + "from_s", + "to_s", + params, + 20, + 3, + 6); + + + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 2); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, dave, alex, steve]")); + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + //Test with batch size of 1 + + params.put("fq", "predicate_s:knows"); + + stream = new ShortestPathStream(zkHost, + "collection1", + "jim", + "steve", + "from_s", + "to_s", + params, + 1, + 3, + 6); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 2); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, dave, alex, steve]")); + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + //Test with bad predicate + + params.put("fq", "predicate_s:crap"); + + stream = new ShortestPathStream(zkHost, + "collection1", + "jim", + "steve", + "from_s", + "to_s", + params, + 1, + 3, + 6); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 0); + + //Test with depth 2 + + params.put("fq", "predicate_s:knows"); + + stream = new ShortestPathStream(zkHost, + "collection1", + "jim", + "steve", + "from_s", + "to_s", + params, + 1, + 3, + 2); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + + assertTrue(tuples.size() == 0); + + + + //Take out alex + params.put("fq", "predicate_s:knows NOT to_s:alex"); + + stream = new ShortestPathStream(zkHost, + "collection1", + "jim", + "steve", + "from_s", + "to_s", + params, + 10, + 3, + 6); + + stream.setStreamContext(context); + paths = new HashSet(); + tuples = getTuples(stream); + assertTrue(tuples.size() == 1); + + for(Tuple tuple : tuples) { + paths.add(tuple.getStrings("path").toString()); + } + + assertTrue(paths.contains("[jim, stan, mary, steve]")); + + cache.close(); + del("*:*"); + commit(); + } + + @Test + public void streamTests() throws Exception { + assertNotNull(cloudClient); + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForRecoveriesToFinish(false); + + del("*:*"); + + commit(); + + testShortestPathStream(); + + } + + protected Map mapParams(String... vals) { + Map params = new HashMap(); + String k = null; + for(String val : vals) { + if(k == null) { + k = val; + } else { + params.put(k, val); + k = null; + } + } + + return params; + } + + protected List getTuples(TupleStream tupleStream) throws IOException { + tupleStream.open(); + List tuples = new ArrayList(); + for(;;) { + Tuple t = tupleStream.read(); + if(t.EOF) { + break; + } else { + tuples.add(t); + } + } + tupleStream.close(); + return tuples; + } + + protected Tuple getTuple(TupleStream tupleStream) throws IOException { + tupleStream.open(); + Tuple t = tupleStream.read(); + tupleStream.close(); + return t; + } + + + protected boolean assertOrder(List tuples, int... ids) throws Exception { + int i = 0; + for(int val : ids) { + Tuple t = tuples.get(i); + Long tip = (Long)t.get("id"); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception { + List group = (List)tuple.get("tuples"); + int i=0; + for(int val : ids) { + Map t = (Map)group.get(i); + Long tip = (Long)t.get("id"); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + protected boolean assertMaps(List maps, int... ids) throws Exception { + if(maps.size() != ids.length) { + throw new Exception("Expected id count != actual map count:"+ids.length+":"+maps.size()); + } + + int i=0; + for(int val : ids) { + Map t = maps.get(i); + Long tip = (Long)t.get("id"); + if(tip.intValue() != val) { + throw new Exception("Found value:"+tip.intValue()+" expecting:"+val); + } + ++i; + } + return true; + } + + public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception { + long lv = (long)tuple.get(fieldName); + if(lv != l) { + throw new Exception("Longs not equal:"+l+" : "+lv); + } + + return true; + } + + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = getDoc(fields); + indexDoc(doc); + } + + private void attachStreamFactory(TupleStream tupleStream) { + StreamContext streamContext = new StreamContext(); + streamContext.setStreamFactory(streamFactory); + tupleStream.setStreamContext(streamContext); + } +} + From 586afc3db117eabb31e2572da9bd3b7665cdccc8 Mon Sep 17 00:00:00 2001 From: jbernste Date: Thu, 31 Mar 2016 20:56:44 -0400 Subject: [PATCH 0252/1113] SOLR-8888: Update CHANGES.txt --- solr/CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2c5fc28033d6..5224f1b7386d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -56,6 +56,11 @@ New Features * SOLR-8890: New static method in DistributedUpdateProcessorFactory to allow UpdateProcessorFactories to indicate request params that should be forwarded when DUP distributes updates. (hossman) +* SOLR-8888: Add shortestPath Streaming Expression. This expression performs a breadth first + graph traversal to find the shortest path(s) in an unweighted, directed graph stored in a + SolrCloud collection. (Joel Bernstein) + + Bug Fixes ---------------------- From 1153264782bdd8f86a9c381637c57003250b985e Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 31 Mar 2016 22:00:16 -0400 Subject: [PATCH 0253/1113] completely untested logic to try to make javadocs work across our rats nest of spatial modules ! --- lucene/sandbox/build.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lucene/sandbox/build.xml b/lucene/sandbox/build.xml index d53a6ac9ed14..4323241e2eda 100644 --- a/lucene/sandbox/build.xml +++ b/lucene/sandbox/build.xml @@ -46,6 +46,16 @@ + + + + + + + + + From 2c0a8ed418934732fafa4d2bca0c757cca1a42b0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 31 Mar 2016 22:28:46 -0400 Subject: [PATCH 0254/1113] LUCENE-7153: give GeoPointField and LatLonPoint full polygon support --- lucene/CHANGES.txt | 4 + .../apache/lucene/document/LatLonPoint.java | 15 +- .../document/LatLonPointInPolygonQuery.java | 163 +++++---- .../lucene/search/TestLatLonPointQueries.java | 5 +- .../search/GeoPointInPolygonQuery.java | 102 +++--- .../search/GeoPointInPolygonQueryImpl.java | 40 ++- .../lucene/spatial/util/GeoRelationUtils.java | 176 ---------- .../apache/lucene/spatial/util/GeoUtils.java | 49 --- .../apache/lucene/spatial/util/Polygon.java | 330 ++++++++++++++++++ .../geopoint/search/TestGeoPointQuery.java | 5 +- .../search/TestLegacyGeoPointQuery.java | 7 +- .../spatial/util/BaseGeoPointTestCase.java | 151 ++++---- .../lucene/spatial/util/GeoTestUtil.java | 17 +- .../lucene/spatial/util/TestGeoUtils.java | 59 ---- .../lucene/spatial/util/TestPolygon.java | 141 ++++++++ 15 files changed, 733 insertions(+), 531 deletions(-) create mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7b7c9d38daae..636b6422791c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -43,6 +43,10 @@ Optimizations * LUCENE-7147: Improve disjoint check for geo distance query traversal (Ryan Ernst, Robert Muir, Mike McCandless) +* LUCENE-7153: GeoPointField and LatLonPoint polygon queries now support + multiple polygons and holes, with memory usage independent of + polygon complexity. (Karl Wright, Mike McCandless, Robert Muir) + Bug Fixes * LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 60e83000083f..6e9e7154be54 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.spatial.util.Polygon; /** * An indexed location field. @@ -302,18 +303,14 @@ public static Query newDistanceQuery(String field, double latitude, double longi /** * Create a query for matching a polygon. *

    - * The supplied {@code polyLats}/{@code polyLons} must be clockwise or counter-clockwise. + * The supplied {@code polygon} must be clockwise or counter-clockwise. * @param field field name. must not be null. - * @param polyLats latitude values for points of the polygon: must be within standard +/-90 coordinate bounds. - * @param polyLons longitude values for points of the polygon: must be within standard +/-180 coordinate bounds. + * @param polygons array of polygons. must not be null or empty * @return query matching points within this polygon - * @throws IllegalArgumentException if {@code field} is null, {@code polyLats} is null or has invalid coordinates, - * {@code polyLons} is null or has invalid coordinates, if {@code polyLats} has a different - * length than {@code polyLons}, if the polygon has less than 4 points, or if polygon is - * not closed (first and last points should be the same) + * @throws IllegalArgumentException if {@code field} is null, {@code polygons} is null or empty */ - public static Query newPolygonQuery(String field, double[] polyLats, double[] polyLons) { - return new LatLonPointInPolygonQuery(field, polyLats, polyLons); + public static Query newPolygonQuery(String field, Polygon... polygons) { + return new LatLonPointInPolygonQuery(field, polygons); } /** diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 454a3ddf68b0..2875e2f04238 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -39,12 +39,13 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.lucene.util.StringHelper; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.spatial.util.Polygon; -/** Finds all previously indexed points that fall within the specified polygon. +/** Finds all previously indexed points that fall within the specified polygons. * *

    The field must be indexed with using {@link org.apache.lucene.document.LatLonPoint} added per document. * @@ -52,29 +53,27 @@ final class LatLonPointInPolygonQuery extends Query { final String field; - final double minLat; - final double maxLat; - final double minLon; - final double maxLon; - final double[] polyLats; - final double[] polyLons; + final Polygon[] polygons; /** The lats/lons must be clockwise or counter-clockwise. */ - public LatLonPointInPolygonQuery(String field, double[] polyLats, double[] polyLons) { + public LatLonPointInPolygonQuery(String field, Polygon[] polygons) { if (field == null) { throw new IllegalArgumentException("field must not be null"); } - GeoUtils.checkPolygon(polyLats, polyLons); + if (polygons == null) { + throw new IllegalArgumentException("polygons must not be null"); + } + if (polygons.length == 0) { + throw new IllegalArgumentException("polygons must not be empty"); + } + for (int i = 0; i < polygons.length; i++) { + if (polygons[i] == null) { + throw new IllegalArgumentException("polygon[" + i + "] must not be null"); + } + } this.field = field; - this.polyLats = polyLats; - this.polyLons = polyLons; - + this.polygons = polygons.clone(); // TODO: we could also compute the maximal inner bounding box, to make relations faster to compute? - GeoRect box = GeoUtils.polyToBBox(polyLats, polyLons); - this.minLon = box.minLon; - this.maxLon = box.maxLon; - this.minLat = box.minLat; - this.maxLat = box.maxLat; } @Override @@ -82,6 +81,25 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // I don't use RandomAccessWeight here: it's no good to approximate with "match all docs"; this is an inverted structure and should be // used in the first pass: + + // bounding box over all polygons, this can speed up tree intersection/cheaply improve approximation for complex multi-polygons + // these are pre-encoded with LatLonPoint's encoding + final GeoRect box = Polygon.getBoundingBox(polygons); + final byte minLat[] = new byte[Integer.BYTES]; + final byte maxLat[] = new byte[Integer.BYTES]; + final byte minLon[] = new byte[Integer.BYTES]; + final byte maxLon[] = new byte[Integer.BYTES]; + NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.minLat), minLat, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.maxLat), maxLat, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon, 0); + NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + + // TODO: make this fancier, but currently linear with number of vertices + float cumulativeCost = 0; + for (Polygon polygon : polygons) { + cumulativeCost += 20 * (polygon.getPolyLats().length + polygon.getHoles().length); + } + final float matchCost = cumulativeCost; return new ConstantScoreWeight(this) { @@ -120,27 +138,36 @@ public void visit(int docID) { @Override public void visit(int docID, byte[] packedValue) { - // TODO: range checks + // we bounds check individual values, as subtrees may cross, but we are being sent the values anyway: + // this reduces the amount of docvalues fetches (improves approximation) + if (StringHelper.compare(Integer.BYTES, packedValue, 0, maxLat, 0) > 0 || + StringHelper.compare(Integer.BYTES, packedValue, 0, minLat, 0) < 0 || + StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, maxLon, 0) > 0 || + StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon, 0) < 0) { + // outside of global bounding box range + return; + } result.add(docID); } @Override public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 || + StringHelper.compare(Integer.BYTES, maxPackedValue, 0, minLat, 0) < 0 || + StringHelper.compare(Integer.BYTES, minPackedValue, Integer.BYTES, maxLon, 0) > 0 || + StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon, 0) < 0) { + // outside of global bounding box range + return Relation.CELL_OUTSIDE_QUERY; + } + double cellMinLat = LatLonPoint.decodeLatitude(minPackedValue, 0); double cellMinLon = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES); double cellMaxLat = LatLonPoint.decodeLatitude(maxPackedValue, 0); double cellMaxLon = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - if (cellMinLat <= minLat && cellMaxLat >= maxLat && cellMinLon <= minLon && cellMaxLon >= maxLon) { - // Cell fully encloses the query - return Relation.CELL_CROSSES_QUERY; - } else if (GeoRelationUtils.rectWithinPolyPrecise(cellMinLat, cellMaxLat, cellMinLon, cellMaxLon, - polyLats, polyLons, - minLat, maxLat, minLon, maxLon)) { + if (Polygon.contains(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon)) { return Relation.CELL_INSIDE_QUERY; - } else if (GeoRelationUtils.rectCrossesPolyPrecise(cellMinLat, cellMaxLat, cellMinLon, cellMaxLon, - polyLats, polyLons, - minLat, maxLat, minLon, maxLon)) { + } else if (Polygon.crosses(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon)) { return Relation.CELL_CROSSES_QUERY; } else { return Relation.CELL_OUTSIDE_QUERY; @@ -156,6 +183,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { // return two-phase iterator using docvalues to postfilter candidates SortedNumericDocValues docValues = DocValues.getSortedNumeric(reader, field); + TwoPhaseIterator iterator = new TwoPhaseIterator(disi) { @Override public boolean matches() throws IOException { @@ -169,7 +197,7 @@ public boolean matches() throws IOException { long encoded = docValues.valueAt(i); double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - if (GeoRelationUtils.pointInPolygon(polyLats, polyLons, docLatitude, docLongitude)) { + if (Polygon.contains(polygons, docLatitude, docLongitude)) { return true; } } @@ -179,7 +207,7 @@ public boolean matches() throws IOException { @Override public float matchCost() { - return 20 * polyLons.length; // TODO: make this fancier, but currently linear with number of vertices + return matchCost; } }; return new ConstantScoreScorer(this, score(), iterator); @@ -187,64 +215,36 @@ public float matchCost() { }; } + /** Returns the query field */ public String getField() { return field; } - public double getMinLat() { - return minLat; - } - - public double getMaxLat() { - return maxLat; - } - - public double getMinLon() { - return minLon; - } - - public double getMaxLon() { - return maxLon; - } - - public double[] getPolyLats() { - return polyLats; - } - - public double[] getPolyLons() { - return polyLons; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - LatLonPointInPolygonQuery that = (LatLonPointInPolygonQuery) o; - - if (field.equals(that.field) == false) { - return false; - } - if (Arrays.equals(polyLons, that.polyLons) == false) { - return false; - } - if (Arrays.equals(polyLats, that.polyLats) == false) { - return false; - } - - return true; + /** Returns a copy of the internal polygon array */ + public Polygon[] getPolygons() { + return polygons.clone(); } @Override public int hashCode() { + final int prime = 31; int result = super.hashCode(); - result = 31 * result + field.hashCode(); - result = 31 * result + Arrays.hashCode(polyLons); - result = 31 * result + Arrays.hashCode(polyLats); + result = prime * result + field.hashCode(); + result = prime * result + Arrays.hashCode(polygons); return result; } + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + LatLonPointInPolygonQuery other = (LatLonPointInPolygonQuery) obj; + if (!field.equals(other.field)) return false; + if (!Arrays.equals(polygons, other.polygons)) return false; + return true; + } + @Override public String toString(String field) { final StringBuilder sb = new StringBuilder(); @@ -255,14 +255,7 @@ public String toString(String field) { sb.append(this.field); sb.append(':'); } - sb.append(" Points: "); - for (int i=0; i bMaxLon || aMaxLat < bMinLat || aMinLat > bMaxLat)); } - - ///////////////////////// - // Polygon relations - ///////////////////////// - - /** - * Convenience method for accurately computing whether a rectangle crosses a poly - */ - public static boolean rectCrossesPolyPrecise(final double rMinLat, final double rMaxLat, - final double rMinLon, final double rMaxLon, - final double[] shapeLat, final double[] shapeLon, - final double sMinLat, final double sMaxLat, - final double sMinLon, final double sMaxLon) { - // short-circuit: if the bounding boxes are disjoint then the shape does not cross - if (rectDisjoint(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon)) { - return false; - } - return rectCrossesPoly(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLat, shapeLon); - } - - /** - * Compute whether a rectangle crosses a shape. (touching not allowed) Includes a flag for approximating the - * relation. - */ - public static boolean rectCrossesPolyApprox(final double rMinLat, final double rMaxLat, - final double rMinLon, final double rMaxLon, - final double[] shapeLat, final double[] shapeLon, - final double sMinLat, final double sMaxLat, - final double sMinLon, final double sMaxLon) { - // short-circuit: if the bounding boxes are disjoint then the shape does not cross - if (rectDisjoint(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon)) { - return false; - } - - final int polyLength = shapeLon.length-1; - for (short p=0; p s || x01 < s || y00 > t || y01 < t || x10 > s || x11 < s || y10 > t || y11 < t)) { - return true; - } - } - } // for each poly edge - } // for each bbox edge - return false; - } - - private static boolean lineCrossesRect(double aLat1, double aLon1, - double aLat2, double aLon2, - final double rMinLat, final double rMaxLat, - final double rMinLon, final double rMaxLon) { - // short-circuit: if one point inside rect, other outside - if (pointInRectPrecise(aLat1, aLon1, rMinLat, rMaxLat, rMinLon, rMaxLon)) { - if (pointInRectPrecise(aLat2, aLon2, rMinLat, rMaxLat, rMinLon, rMaxLon) == false) { - return true; - } - } else if (pointInRectPrecise(aLat2, aLon2, rMinLat, rMaxLat, rMinLon, rMaxLon)) { - return true; - } - - return lineCrossesLine(aLat1, aLon1, aLat2, aLon2, rMinLat, rMinLon, rMaxLat, rMaxLon) - || lineCrossesLine(aLat1, aLon1, aLat2, aLon2, rMaxLat, rMinLon, rMinLat, rMaxLon); - } - - private static boolean lineCrossesLine(final double aLat1, final double aLon1, final double aLat2, final double aLon2, - final double bLat1, final double bLon1, final double bLat2, final double bLon2) { - // determine if three points are ccw (right-hand rule) by computing the determinate - final double aX2X1d = aLon2 - aLon1; - final double aY2Y1d = aLat2 - aLat1; - final double bX2X1d = bLon2 - bLon1; - final double bY2Y1d = bLat2 - bLat1; - - final double t1B = aX2X1d * (bLat2 - aLat1) - aY2Y1d * (bLon2 - aLon1); - final double test1 = (aX2X1d * (bLat1 - aLat1) - aY2Y1d * (bLon1 - aLon1)) * t1B; - final double t2B = bX2X1d * (aLat2 - bLat1) - bY2Y1d * (aLon2 - bLon1); - final double test2 = (bX2X1d * (aLat1 - bLat1) - bY2Y1d * (aLon1 - bLon1)) * t2B; - - if (test1 < 0 && test2 < 0) { - return true; - } - - if (test1 == 0 || test2 == 0) { - // vertically collinear - if (aLon1 == aLon2 || bLon1 == bLon2) { - final double minAy = Math.min(aLat1, aLat2); - final double maxAy = Math.max(aLat1, aLat2); - final double minBy = Math.min(bLat1, bLat2); - final double maxBy = Math.max(bLat1, bLat2); - - return !(minBy >= maxAy || maxBy <= minAy); - } - // horizontally collinear - final double minAx = Math.min(aLon1, aLon2); - final double maxAx = Math.max(aLon1, aLon2); - final double minBx = Math.min(bLon1, bLon2); - final double maxBx = Math.max(bLon1, bLon2); - - return !(minBx >= maxAx || maxBx <= minAx); - } - return false; - } - - /** - * Computes whether a rectangle is within a polygon (shared boundaries not allowed) with more rigor than the - * {@link GeoRelationUtils#rectWithinPolyApprox} counterpart - */ - public static boolean rectWithinPolyPrecise(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double[] shapeLats, final double[] shapeLons, final double sMinLat, - final double sMaxLat, final double sMinLon, final double sMaxLon) { - // check if rectangle crosses poly (to handle concave/pacman polys), then check that all 4 corners - // are contained - return !(rectCrossesPolyPrecise(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLats, shapeLons, sMinLat, sMaxLat, sMinLon, sMaxLon) || - !pointInPolygon(shapeLats, shapeLons, rMinLat, rMinLon) || !pointInPolygon(shapeLats, shapeLons, rMinLat, rMaxLon) || - !pointInPolygon(shapeLats, shapeLons, rMaxLat, rMaxLon) || !pointInPolygon(shapeLats, shapeLons, rMaxLat, rMinLon)); - } - - /** - * Computes whether a rectangle is within a given polygon (shared boundaries allowed) - */ - public static boolean rectWithinPolyApprox(final double rMinLat, final double rMaxLat, final double rMinLon, final double rMaxLon, - final double[] shapeLats, final double[] shapeLons, final double sMinLat, - final double sMaxLat, final double sMinLon, final double sMaxLon) { - // approximation: check if rectangle crosses poly (to handle concave/pacman polys), then check one of the corners - // are contained - - // short-cut: if bounding boxes cross, rect is not within - if (rectCrosses(rMinLat, rMaxLat, rMinLon, rMaxLon, sMinLat, sMaxLat, sMinLon, sMaxLon) == true) { - return false; - } - - return !(rectCrossesPolyApprox(rMinLat, rMaxLat, rMinLon, rMaxLon, shapeLats, shapeLons, sMinLat, sMaxLat, sMinLon, sMaxLon) - || !pointInPolygon(shapeLats, shapeLons, rMinLat, rMinLon)); - } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index 52e94052798f..6d7f615ba2f1 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -74,37 +74,6 @@ public static void checkLongitude(double longitude) { } } - /** validates polygon values are within standard +/-180 coordinate bounds, same - * number of latitude and longitude, and is closed - */ - public static void checkPolygon(double[] polyLats, double[] polyLons) { - if (polyLats == null) { - throw new IllegalArgumentException("polyLats must not be null"); - } - if (polyLons == null) { - throw new IllegalArgumentException("polyLons must not be null"); - } - if (polyLats.length != polyLons.length) { - throw new IllegalArgumentException("polyLats and polyLons must be equal length"); - } - if (polyLats.length != polyLons.length) { - throw new IllegalArgumentException("polyLats and polyLons must be equal length"); - } - if (polyLats.length < 4) { - throw new IllegalArgumentException("at least 4 polygon points required"); - } - if (polyLats[0] != polyLats[polyLats.length-1]) { - throw new IllegalArgumentException("first and last points of the polygon must be the same (it must close itself): polyLats[0]=" + polyLats[0] + " polyLats[" + (polyLats.length-1) + "]=" + polyLats[polyLats.length-1]); - } - if (polyLons[0] != polyLons[polyLons.length-1]) { - throw new IllegalArgumentException("first and last points of the polygon must be the same (it must close itself): polyLons[0]=" + polyLons[0] + " polyLons[" + (polyLons.length-1) + "]=" + polyLons[polyLons.length-1]); - } - for (int i = 0; i < polyLats.length; i++) { - checkLatitude(polyLats[i]); - checkLongitude(polyLons[i]); - } - } - /** Compute Bounding Box for a circle using WGS-84 parameters */ public static GeoRect circleToBBox(final double centerLat, final double centerLon, final double radiusMeters) { final double radLat = TO_RADIANS * centerLat; @@ -136,24 +105,6 @@ public static GeoRect circleToBBox(final double centerLat, final double centerLo return new GeoRect(TO_DEGREES * minLat, TO_DEGREES * maxLat, TO_DEGREES * minLon, TO_DEGREES * maxLon); } - - /** Compute Bounding Box for a polygon using WGS-84 parameters */ - public static GeoRect polyToBBox(double[] polyLats, double[] polyLons) { - checkPolygon(polyLats, polyLons); - - double minLon = Double.POSITIVE_INFINITY; - double maxLon = Double.NEGATIVE_INFINITY; - double minLat = Double.POSITIVE_INFINITY; - double maxLat = Double.NEGATIVE_INFINITY; - - for (int i=0;i maxLat || longitude < minLon || longitude > maxLon) { + return false; + } + /* + * simple even-odd point in polygon computation + * 1. Determine if point is contained in the longitudinal range + * 2. Determine whether point crosses the edge by computing the latitudinal delta + * between the end-point of a parallel vector (originating at the point) and the + * y-component of the edge sink + * + * NOTE: Requires polygon point (x,y) order either clockwise or counter-clockwise + */ + boolean inPoly = false; + /* + * Note: This is using a euclidean coordinate system which could result in + * upwards of 110KM error at the equator. + * TODO convert coordinates to cylindrical projection (e.g. mercator) + */ + for (int i = 1; i < polyLats.length; i++) { + if (polyLons[i] <= longitude && polyLons[i-1] >= longitude || polyLons[i-1] <= longitude && polyLons[i] >= longitude) { + if (polyLats[i] + (longitude - polyLons[i]) / (polyLons[i-1] - polyLons[i]) * (polyLats[i-1] - polyLats[i]) <= latitude) { + inPoly = !inPoly; + } + } + } + if (inPoly) { + for (Polygon hole : holes) { + if (hole.contains(latitude, longitude)) { + return false; + } + } + return true; + } else { + return false; + } + } + + /** + * Computes whether a rectangle is within a polygon (shared boundaries not allowed) + */ + public boolean contains(double minLat, double maxLat, double minLon, double maxLon) { + // check if rectangle crosses poly (to handle concave/pacman polys), then check that all 4 corners + // are contained + boolean contains = crosses(minLat, maxLat, minLon, maxLon) == false && + contains(minLat, minLon) && + contains(minLat, maxLon) && + contains(maxLat, maxLon) && + contains(maxLat, minLon); + + if (contains) { + // if we intersect with any hole, game over + for (Polygon hole : holes) { + if (hole.crosses(minLat, maxLat, minLon, maxLon) || hole.contains(minLat, maxLat, minLon, maxLon)) { + return false; + } + } + return true; + } else { + return false; + } + } + + /** + * Convenience method for accurately computing whether a rectangle crosses a poly. + */ + public boolean crosses(double minLat, double maxLat, final double minLon, final double maxLon) { + // if the bounding boxes are disjoint then the shape does not cross + if (maxLon < this.minLon || minLon > this.maxLon || maxLat < this.minLat || minLat > this.maxLat) { + return false; + } + // if the rectangle fully encloses us, we cross. + if (minLat <= this.minLat && maxLat >= this.maxLat && minLon <= this.minLon && maxLon >= this.maxLon) { + return true; + } + // if we cross any hole, we cross + for (Polygon hole : holes) { + if (hole.crosses(minLat, maxLat, minLon, maxLon)) { + return true; + } + } + + /* + * Accurately compute (within restrictions of cartesian decimal degrees) whether a rectangle crosses a polygon + */ + final double[][] bbox = new double[][] { {minLon, minLat}, {maxLon, minLat}, {maxLon, maxLat}, {minLon, maxLat}, {minLon, minLat} }; + final int polyLength = polyLons.length-1; + double d, s, t, a1, b1, c1, a2, b2, c2; + double x00, y00, x01, y01, x10, y10, x11, y11; + + // computes the intersection point between each bbox edge and the polygon edge + for (short b=0; b<4; ++b) { + a1 = bbox[b+1][1]-bbox[b][1]; + b1 = bbox[b][0]-bbox[b+1][0]; + c1 = a1*bbox[b+1][0] + b1*bbox[b+1][1]; + for (int p=0; p s || x01 < s || y00 > t || y01 < t || x10 > s || x11 < s || y10 > t || y11 < t)) { + return true; + } + } + } // for each poly edge + } // for each bbox edge + return false; + } + + /** Returns a copy of the internal latitude array */ + public double[] getPolyLats() { + return polyLats.clone(); + } + + /** Returns a copy of the internal longitude array */ + public double[] getPolyLons() { + return polyLons.clone(); + } + + /** Returns a copy of the internal holes array */ + public Polygon[] getHoles() { + return holes.clone(); + } + + /** Returns the bounding box over an array of polygons */ + public static GeoRect getBoundingBox(Polygon[] polygons) { + // compute bounding box + double minLat = Double.POSITIVE_INFINITY; + double maxLat = Double.NEGATIVE_INFINITY; + double minLon = Double.POSITIVE_INFINITY; + double maxLon = Double.NEGATIVE_INFINITY; + + for (int i = 0;i < polygons.length; i++) { + minLat = Math.min(polygons[i].minLat, minLat); + maxLat = Math.max(polygons[i].maxLat, maxLat); + minLon = Math.min(polygons[i].minLon, minLon); + maxLon = Math.max(polygons[i].maxLon, maxLon); + } + + return new GeoRect(minLat, maxLat, minLon, maxLon); + } + + /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the point */ + public static boolean contains(Polygon[] polygons, double latitude, double longitude) { + for (Polygon polygon : polygons) { + if (polygon.contains(latitude, longitude)) { + return true; + } + } + return false; + } + + /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the rectangle */ + public static boolean contains(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { + for (Polygon polygon : polygons) { + if (polygon.contains(minLat, maxLat, minLon, maxLon)) { + return true; + } + } + return false; + } + + /** Helper for multipolygon logic: returns true if any of the supplied polygons crosses the rectangle */ + public static boolean crosses(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { + for (Polygon polygon : polygons) { + if (polygon.crosses(minLat, maxLat, minLon, maxLon)) { + return true; + } + } + return false; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(holes); + result = prime * result + Arrays.hashCode(polyLats); + result = prime * result + Arrays.hashCode(polyLons); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + Polygon other = (Polygon) obj; + if (!Arrays.equals(holes, other.holes)) return false; + if (!Arrays.equals(polyLats, other.polyLats)) return false; + if (!Arrays.equals(polyLons, other.polyLons)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < polyLats.length; i++) { + sb.append("[") + .append(polyLats[i]) + .append(", ") + .append(polyLons[i]) + .append("] "); + } + if (holes.length > 0) { + sb.append(", holes="); + sb.append(Arrays.toString(holes)); + } + return sb.toString(); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 0f0eaeb555c6..91afe3fd7e2b 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.util.Polygon; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; @@ -56,8 +57,8 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo } @Override - protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, lats, lons); + protected Query newPolygonQuery(String field, Polygon... polygons) { + return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, polygons); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index 00cc279a00a7..cd15f6691a56 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.util.Polygon; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; @@ -56,8 +57,8 @@ protected Query newDistanceQuery(String field, double centerLat, double centerLo } @Override - protected Query newPolygonQuery(String field, double[] lats, double[] lons) { - return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, lats, lons); + protected Query newPolygonQuery(String field, Polygon... polygons) { + return new GeoPointInPolygonQuery(field, TermEncoding.NUMERIC, polygons); } // legacy encoding is too slow somehow for this random test, its not up to the task. @@ -75,4 +76,6 @@ public void testRandomDistanceHuge() throws Exception { public void testSamePointManyTimes() throws Exception { assumeTrue("legacy encoding goes OOM on this test", false); } + + } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index dd0e09be96d6..1b18a18fbc5c 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -285,73 +285,95 @@ public void testPolygonBasics() throws Exception { // search and verify we found our doc IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader); - assertEquals(1, searcher.count(newPolygonQuery("field", + assertEquals(1, searcher.count(newPolygonQuery("field", new Polygon( new double[] { 18, 18, 19, 19, 18 }, - new double[] { -66, -65, -65, -66, -66 }))); + new double[] { -66, -65, -65, -66, -66 })))); reader.close(); writer.close(); dir.close(); } - /** null field name not allowed */ - public void testPolygonNullField() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery(null, - new double[] { 18, 18, 19, 19, 18 }, - new double[] { -66, -65, -65, -66, -66 }); - }); - assertTrue(expected.getMessage().contains("field must not be null")); - } - - /** null polyLats not allowed */ - public void testPolygonNullPolyLats() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery("test", - null, - new double[] { -66, -65, -65, -66, -66 }); - }); - assertTrue(expected.getMessage().contains("polyLats must not be null")); - } - - /** null polyLons not allowed */ - public void testPolygonNullPolyLons() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery("test", - new double[] { 18, 18, 19, 19, 18 }, - null); - }); - assertTrue(expected.getMessage().contains("polyLons must not be null")); + /** test we can search for a polygon with a hole (but still includes the doc) */ + public void testPolygonHole() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + Polygon inner = new Polygon(new double[] { 18.5, 18.5, 18.7, 18.7, 18.5 }, + new double[] { -65.7, -65.4, -65.4, -65.7, -65.7 }); + Polygon outer = new Polygon(new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }, inner); + assertEquals(1, searcher.count(newPolygonQuery("field", outer))); + + reader.close(); + writer.close(); + dir.close(); } - /** polygon needs at least 3 vertices */ - public void testPolygonLine() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery("test", - new double[] { 18, 18, 18 }, - new double[] { -66, -65, -66 }); - }); - assertTrue(expected.getMessage().contains("at least 4 polygon points required")); + /** test we can search for a polygon with a hole (that excludes the doc) */ + public void testPolygonHoleExcludes() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + Polygon inner = new Polygon(new double[] { 18.2, 18.2, 18.4, 18.4, 18.2 }, + new double[] { -65.3, -65.2, -65.2, -65.3, -65.3 }); + Polygon outer = new Polygon(new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }, inner); + assertEquals(0, searcher.count(newPolygonQuery("field", outer))); + + reader.close(); + writer.close(); + dir.close(); } - /** polygon needs same number of latitudes as longitudes */ - public void testPolygonBogus() { - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery("test", - new double[] { 18, 18, 19, 19 }, - new double[] { -66, -65, -65, -66, -66 }); - }); - assertTrue(expected.getMessage().contains("must be equal length")); + /** test we can search for a multi-polygon */ + public void testMultiPolygonBasics() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + + // add a doc with a point + Document document = new Document(); + addPointToDoc("field", document, 18.313694, -65.227444); + writer.addDocument(document); + + // search and verify we found our doc + IndexReader reader = writer.getReader(); + IndexSearcher searcher = newSearcher(reader); + Polygon a = new Polygon(new double[] { 28, 28, 29, 29, 28 }, + new double[] { -56, -55, -55, -56, -56 }); + Polygon b = new Polygon(new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 }); + assertEquals(1, searcher.count(newPolygonQuery("field", a, b))); + + reader.close(); + writer.close(); + dir.close(); } - /** polygon must be closed */ - public void testPolygonNotClosed() { + /** null field name not allowed */ + public void testPolygonNullField() { IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { - newPolygonQuery("test", - new double[] { 18, 18, 19, 19, 19 }, - new double[] { -66, -65, -65, -66, -67 }); + newPolygonQuery(null, new Polygon( + new double[] { 18, 18, 19, 19, 18 }, + new double[] { -66, -65, -65, -66, -66 })); }); - assertTrue(expected.getMessage(), expected.getMessage().contains("it must close itself")); + assertTrue(expected.getMessage().contains("field must not be null")); } // A particularly tricky adversary for BKD tree: @@ -710,7 +732,7 @@ protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters); - protected abstract Query newPolygonQuery(String field, double[] lats, double[] lons); + protected abstract Query newPolygonQuery(String field, Polygon... polygon); static final boolean rectContainsPoint(GeoRect rect, double pointLat, double pointLon) { assert Double.isNaN(pointLat) == false; @@ -1072,16 +1094,14 @@ protected void verifyRandomPolygons(boolean small, double[] lats, double[] lons) } // Polygon - final double[][] polygon; + final Polygon polygon; if (small) { polygon = GeoTestUtil.nextPolygonNear(originLat, originLon); } else { polygon = GeoTestUtil.nextPolygon(); } - final double[] polyLats = polygon[0]; - final double[] polyLons = polygon[1]; - Query query = newPolygonQuery(FIELD_NAME, polyLats, polyLons); + Query query = newPolygonQuery(FIELD_NAME, polygon); if (VERBOSE) { System.out.println(" query=" + query); @@ -1118,7 +1138,7 @@ public void collect(int doc) { } else if (Double.isNaN(lats[id])) { expected = false; } else { - expected = GeoRelationUtils.pointInPolygon(polyLats, polyLons, lats[id], lons[id]); + expected = polygon.contains(lats[id], lons[id]); } if (hits.get(docID) != expected) { @@ -1132,8 +1152,7 @@ public void collect(int doc) { b.append(" query=" + query + " docID=" + docID + "\n"); b.append(" lat=" + lats[id] + " lon=" + lons[id] + "\n"); b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false)); - b.append(" polyLats=" + Arrays.toString(polyLats)); - b.append(" polyLons=" + Arrays.toString(polyLons)); + b.append(" polygon=" + polygon); if (true) { fail("wrong hit (first of possibly more):\n\n" + b); } else { @@ -1317,10 +1336,10 @@ public void testEquals() throws Exception { lons[3] = rect.maxLon; lats[4] = rect.minLat; lons[4] = rect.minLon; - q1 = newPolygonQuery("field", lats, lons); - q2 = newPolygonQuery("field", lats, lons); + q1 = newPolygonQuery("field", new Polygon(lats, lons)); + q2 = newPolygonQuery("field", new Polygon(lats, lons)); assertEquals(q1, q2); - assertFalse(q1.equals(newPolygonQuery("field2", lats, lons))); + assertFalse(q1.equals(newPolygonQuery("field2", new Polygon(lats, lons)))); } /** return topdocs over a small set of points in field "point" */ @@ -1406,18 +1425,20 @@ public void testSmallSetWholeMap() throws Exception { public void testSmallSetPoly() throws Exception { TopDocs td = searchSmallSet(newPolygonQuery("point", + new Polygon( new double[]{33.073130, 32.9942669, 32.938386, 33.0374494, 33.1369762, 33.1162747, 33.073130, 33.073130}, new double[]{-96.7682647, -96.8280029, -96.6288757, -96.4929199, - -96.6041564, -96.7449188, -96.76826477, -96.7682647}), + -96.6041564, -96.7449188, -96.76826477, -96.7682647})), 5); assertEquals(2, td.totalHits); } public void testSmallSetPolyWholeMap() throws Exception { TopDocs td = searchSmallSet(newPolygonQuery("point", + new Polygon( new double[] {GeoUtils.MIN_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MAX_LAT_INCL, GeoUtils.MIN_LAT_INCL, GeoUtils.MIN_LAT_INCL}, - new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL}), + new double[] {GeoUtils.MIN_LON_INCL, GeoUtils.MIN_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MAX_LON_INCL, GeoUtils.MIN_LON_INCL})), 20); assertEquals("testWholeMap failed", 24, td.totalHits); } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java index e52c8a587d9e..1785c12f4454 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java @@ -85,7 +85,7 @@ public static GeoRect nextBoxNear(double otherLatitude, double otherLongitude) { } /** returns next pseudorandom polygon */ - public static double[][] nextPolygon() { + public static Polygon nextPolygon() { if (random().nextBoolean()) { return surpriseMePolygon(null, null); } @@ -101,7 +101,7 @@ public static double[][] nextPolygon() { } /** returns next pseudorandom polygon, kinda close to {@code otherLatitude} and {@code otherLongitude} */ - public static double[][] nextPolygonNear(double otherLatitude, double otherLongitude) { + public static Polygon nextPolygonNear(double otherLatitude, double otherLongitude) { if (random().nextBoolean()) { return surpriseMePolygon(otherLatitude, otherLongitude); } @@ -133,7 +133,7 @@ private static GeoRect nextBoxInternal(double lat0, double lat1, double lon0, do return new GeoRect(lat0, lat1, lon0, lon1); } - private static double[][] boxPolygon(GeoRect box) { + private static Polygon boxPolygon(GeoRect box) { assert box.crossesDateline() == false; final double[] polyLats = new double[5]; final double[] polyLons = new double[5]; @@ -147,10 +147,10 @@ private static double[][] boxPolygon(GeoRect box) { polyLons[3] = box.maxLon; polyLats[4] = box.minLat; polyLons[4] = box.minLon; - return new double[][] { polyLats, polyLons }; + return new Polygon(polyLats, polyLons); } - private static double[][] trianglePolygon(GeoRect box) { + private static Polygon trianglePolygon(GeoRect box) { assert box.crossesDateline() == false; final double[] polyLats = new double[4]; final double[] polyLons = new double[4]; @@ -162,11 +162,10 @@ private static double[][] trianglePolygon(GeoRect box) { polyLons[2] = box.maxLon; polyLats[3] = box.minLat; polyLons[3] = box.minLon; - return new double[][] { polyLats, polyLons }; + return new Polygon(polyLats, polyLons); } - /** Returns {polyLats, polyLons} double[] array */ - private static double[][] surpriseMePolygon(Double otherLatitude, Double otherLongitude) { + private static Polygon surpriseMePolygon(Double otherLatitude, Double otherLongitude) { // repeat until we get a poly that doesn't cross dateline: newPoly: while (true) { @@ -232,7 +231,7 @@ private static double[][] surpriseMePolygon(Double otherLatitude, Double otherLo latsArray[i] = lats.get(i); lonsArray[i] = lons.get(i); } - return new double[][] {latsArray, lonsArray}; + return new Polygon(latsArray, lonsArray); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 9d6549a010c5..f3b9ad1a21cd 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -264,65 +264,6 @@ public void testInfiniteRect() { } } - public void testPacManPolyQuery() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // shape bbox - double xMinA = -10; - double xMaxA = 10; - double yMinA = -10; - double yMaxA = 10; - - // candidate crosses cell - double xMin = 2;//-5; - double xMax = 11;//0.000001; - double yMin = -1;//0; - double yMax = 1;//5; - - // test cell crossing poly - assertTrue(GeoRelationUtils.rectCrossesPolyApprox(yMin, yMax, xMin, yMax, py, px, yMinA, yMaxA, xMinA, xMaxA)); - assertFalse(GeoRelationUtils.rectCrossesPolyApprox(0, 5, -5, 0.000001, py, px, yMin, yMax, xMin, xMax)); - assertTrue(GeoRelationUtils.rectWithinPolyApprox(0, 5, -5, -2, py, px, yMin, yMax, xMin, xMax)); - } - - public void testPolyToBBox() throws Exception { - for (int i = 0; i < 1000; i++) { - double[][] polygon = GeoTestUtil.nextPolygon(); - GeoRect box = GeoUtils.polyToBBox(polygon[0], polygon[1]); - assertFalse(box.crossesDateline()); - - for (int j = 0; j < 1000; j++) { - double latitude = GeoTestUtil.nextLatitude(); - double longitude = GeoTestUtil.nextLongitude(); - // if the point is within poly, then it should be in our bounding box - if (GeoRelationUtils.pointInPolygon(polygon[0], polygon[1], latitude, longitude)) { - assertTrue(latitude >= box.minLat && latitude <= box.maxLat); - assertTrue(longitude >= box.minLon && longitude <= box.maxLon); - } - } - } - } - - public void testPolyToBBoxEdgeCases() throws Exception { - for (int i = 0; i < 1000; i++) { - double[][] polygon = GeoTestUtil.nextPolygon(); - GeoRect box = GeoUtils.polyToBBox(polygon[0], polygon[1]); - assertFalse(box.crossesDateline()); - - for (int j = 0; j < 1000; j++) { - double latitude = GeoTestUtil.nextLatitudeAround(box.minLat, box.maxLat); - double longitude = GeoTestUtil.nextLongitudeAround(box.minLon, box.maxLon); - // if the point is within poly, then it should be in our bounding box - if (GeoRelationUtils.pointInPolygon(polygon[0], polygon[1], latitude, longitude)) { - assertTrue(latitude >= box.minLat && latitude <= box.maxLat); - assertTrue(longitude >= box.minLon && longitude <= box.maxLon); - } - } - } - } - public void testAxisLat() { double earthCircumference = 2D * Math.PI * GeoUtils.SEMIMAJOR_AXIS; assertEquals(90, GeoUtils.axisLat(0, earthCircumference / 4), 0.0D); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java new file mode 100644 index 000000000000..4ccd8d423df7 --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.util; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestPolygon extends LuceneTestCase { + + /** null polyLats not allowed */ + public void testPolygonNullPolyLats() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + new Polygon(null, new double[] { -66, -65, -65, -66, -66 }); + }); + assertTrue(expected.getMessage().contains("polyLats must not be null")); + } + + /** null polyLons not allowed */ + public void testPolygonNullPolyLons() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + new Polygon(new double[] { 18, 18, 19, 19, 18 }, null); + }); + assertTrue(expected.getMessage().contains("polyLons must not be null")); + } + + /** polygon needs at least 3 vertices */ + public void testPolygonLine() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + new Polygon(new double[] { 18, 18, 18 }, new double[] { -66, -65, -66 }); + }); + assertTrue(expected.getMessage().contains("at least 4 polygon points required")); + } + + /** polygon needs same number of latitudes as longitudes */ + public void testPolygonBogus() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + new Polygon(new double[] { 18, 18, 19, 19 }, new double[] { -66, -65, -65, -66, -66 }); + }); + assertTrue(expected.getMessage().contains("must be equal length")); + } + + /** polygon must be closed */ + public void testPolygonNotClosed() { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + new Polygon(new double[] { 18, 18, 19, 19, 19 }, new double[] { -66, -65, -65, -66, -67 }); + }); + assertTrue(expected.getMessage(), expected.getMessage().contains("it must close itself")); + } + + /** Three boxes, an island inside a hole inside a shape */ + public void testMultiPolygon() { + Polygon hole = new Polygon(new double[] { -10, -10, 10, 10, -10 }, new double[] { -10, 10, 10, -10, -10 }); + Polygon outer = new Polygon(new double[] { -50, -50, 50, 50, -50 }, new double[] { -50, 50, 50, -50, -50 }, hole); + Polygon island = new Polygon(new double[] { -5, -5, 5, 5, -5 }, new double[] { -5, 5, 5, -5, -5 } ); + Polygon polygons[] = new Polygon[] { outer, island }; + + // contains(point) + assertTrue(Polygon.contains(polygons, -2, 2)); // on the island + assertFalse(Polygon.contains(polygons, -6, 6)); // in the hole + assertTrue(Polygon.contains(polygons, -25, 25)); // on the mainland + assertFalse(Polygon.contains(polygons, -51, 51)); // in the ocean + + // contains(box): this can conservatively return false + assertTrue(Polygon.contains(polygons, -2, 2, -2, 2)); // on the island + assertFalse(Polygon.contains(polygons, 6, 7, 6, 7)); // in the hole + assertTrue(Polygon.contains(polygons, 24, 25, 24, 25)); // on the mainland + assertFalse(Polygon.contains(polygons, 51, 52, 51, 52)); // in the ocean + assertFalse(Polygon.contains(polygons, -60, 60, -60, 60)); // enclosing us completely + assertFalse(Polygon.contains(polygons, 49, 51, 49, 51)); // overlapping the mainland + assertFalse(Polygon.contains(polygons, 9, 11, 9, 11)); // overlapping the hole + assertFalse(Polygon.contains(polygons, 5, 6, 5, 6)); // overlapping the island + + // crosses(box): this can conservatively return true + assertTrue(Polygon.crosses(polygons, -60, 60, -60, 60)); // enclosing us completely + assertTrue(Polygon.crosses(polygons, 49, 51, 49, 51)); // overlapping the mainland and ocean + assertTrue(Polygon.crosses(polygons, 9, 11, 9, 11)); // overlapping the hole and mainland + assertTrue(Polygon.crosses(polygons, 5, 6, 5, 6)); // overlapping the island + } + + public void testPacMan() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // candidate crosses cell + double xMin = 2;//-5; + double xMax = 11;//0.000001; + double yMin = -1;//0; + double yMax = 1;//5; + + // test cell crossing poly + Polygon polygon = new Polygon(py, px); + assertTrue(polygon.crosses(yMin, yMax, xMin, xMax)); + assertFalse(polygon.contains(yMin, yMax, xMin, xMax)); + } + + public void testBoundingBox() throws Exception { + for (int i = 0; i < 100; i++) { + Polygon polygon = GeoTestUtil.nextPolygon(); + + for (int j = 0; j < 100; j++) { + double latitude = GeoTestUtil.nextLatitude(); + double longitude = GeoTestUtil.nextLongitude(); + // if the point is within poly, then it should be in our bounding box + if (polygon.contains(latitude, longitude)) { + assertTrue(latitude >= polygon.minLat && latitude <= polygon.maxLat); + assertTrue(longitude >= polygon.minLon && longitude <= polygon.maxLon); + } + } + } + } + + public void testBoundingBoxEdgeCases() throws Exception { + for (int i = 0; i < 100; i++) { + Polygon polygon = GeoTestUtil.nextPolygon(); + + for (int j = 0; j < 100; j++) { + double latitude = GeoTestUtil.nextLatitudeAround(polygon.minLat, polygon.maxLat); + double longitude = GeoTestUtil.nextLongitudeAround(polygon.minLon, polygon.maxLon); + // if the point is within poly, then it should be in our bounding box + if (polygon.contains(latitude, longitude)) { + assertTrue(latitude >= polygon.minLat && latitude <= polygon.maxLat); + assertTrue(longitude >= polygon.minLon && longitude <= polygon.maxLon); + } + } + } + } +} From 50909527d038b7cff1f8dcb31d5343667956b790 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 1 Apr 2016 00:23:22 -0400 Subject: [PATCH 0255/1113] LUCENE-7153: remove extra PIP copy, this is in Polygon.java now --- .../search/GeoPointInPolygonQueryImpl.java | 2 +- .../lucene/spatial/util/GeoRelationUtils.java | 27 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java index d499a47e72d1..20c90785b634 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java @@ -77,7 +77,7 @@ protected boolean cellIntersectsShape(final double minLat, final double maxLat, * {@link org.apache.lucene.spatial.geopoint.search.GeoPointTermsEnum#accept} method is called to match * encoded terms that fall within the bounding box of the polygon. Those documents that pass the initial * bounding box filter are then compared to the provided polygon using the - * {@link org.apache.lucene.spatial.util.GeoRelationUtils#pointInPolygon} method. + * {@link Polygon#contains(Polygon[], double, double)} method. */ @Override protected boolean postFilter(final double lat, final double lon) { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java index 50f94465a802..a54a304de653 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRelationUtils.java @@ -36,33 +36,6 @@ public static boolean pointInRectPrecise(final double lat, final double lon, return lat >= minLat && lat <= maxLat && lon >= minLon && lon <= maxLon; } - /** - * simple even-odd point in polygon computation - * 1. Determine if point is contained in the longitudinal range - * 2. Determine whether point crosses the edge by computing the latitudinal delta - * between the end-point of a parallel vector (originating at the point) and the - * y-component of the edge sink - * - * NOTE: Requires polygon point (x,y) order either clockwise or counter-clockwise - */ - public static boolean pointInPolygon(double[] polyLats, double[] polyLons, double lat, double lon) { - assert polyLats.length == polyLons.length; - boolean inPoly = false; - /** - * Note: This is using a euclidean coordinate system which could result in - * upwards of 110KM error at the equator. - * TODO convert coordinates to cylindrical projection (e.g. mercator) - */ - for (int i = 1; i < polyLats.length; i++) { - if (polyLons[i] <= lon && polyLons[i-1] >= lon || polyLons[i-1] <= lon && polyLons[i] >= lon) { - if (polyLats[i] + (lon - polyLons[i]) / (polyLons[i-1] - polyLons[i]) * (polyLats[i-1] - polyLats[i]) <= lat) { - inPoly = !inPoly; - } - } - } - return inPoly; - } - ///////////////////////// // Rectangle relations ///////////////////////// From 4ecfa98bdbb3c00d6ab2b9fb9ebb27ec59031352 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 1 Apr 2016 05:40:50 -0400 Subject: [PATCH 0256/1113] LUCENE-7158: use the same value (from WGS84) for earth's mean radius when we approximate it as a sphere --- lucene/CHANGES.txt | 4 ++++ .../org/apache/lucene/util/SloppyMath.java | 6 ++++-- .../apache/lucene/util/TestSloppyMath.java | 20 +++++++++---------- .../demo/facet/DistanceFacetsExample.java | 5 +++-- .../expressions/TestDemoExpressions.java | 6 +++--- .../js/TestJavascriptFunction.java | 2 +- .../document/TestLatLonPointDistanceSort.java | 10 +++++----- .../apache/lucene/spatial/util/GeoUtils.java | 11 +++++----- .../spatial/util/BaseGeoPointTestCase.java | 2 +- .../lucene/spatial/util/TestGeoUtils.java | 2 +- .../apache/lucene/spatial3d/Geo3DPoint.java | 5 +---- .../lucene/spatial3d/geom/PlanetModel.java | 3 ++- .../lucene/spatial3d/TestGeo3DPoint.java | 2 +- .../lucene/spatial3d/geom/GeoCircleTest.java | 15 -------------- 14 files changed, 42 insertions(+), 51 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 636b6422791c..33bf3a596955 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -244,6 +244,10 @@ Bug Fixes * LUCENE-7126: Remove GeoPointDistanceRangeQuery. This query was implemented with boolean NOT, and incorrect for multi-valued documents. (Robert Muir) +* LUCENE-7158: Consistently use earth's WGS84 mean radius wherever our + geo search implementations approximate the earth as a sphere (Karl + Wright via Mike McCandless) + Other * LUCENE-7035: Upgrade icu4j to 56.1/unicode 8. (Robert Muir) diff --git a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java index e3ba489cfd94..d8b7056d077d 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java +++ b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java @@ -176,8 +176,10 @@ public static double asin(double a) { // TODO: remove these for java 9, they fixed Math.toDegrees()/toRadians() to work just like this. public static final double TO_RADIANS = Math.PI / 180D; public static final double TO_DEGREES = 180D / Math.PI; - private static final double TO_METERS = 6_378_137D; // equatorial radius - private static final double TO_KILOMETERS = 6_378.137D; // equatorial radius + + // Earth's mean radius, in meters and kilometers; see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf + private static final double TO_METERS = 6_371_008.7714D; // equatorial radius + private static final double TO_KILOMETERS = 6_371.0087714D; // equatorial radius // cos/asin private static final double ONE_DIV_F2 = 1/2.0; diff --git a/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java b/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java index 2ccf619534d5..f11d35fb95c3 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java @@ -99,8 +99,8 @@ public void testHaversin() { assertEquals(0, haversinMeters(90, -180, 90, 180), 0D); assertEquals(0, haversinMeters(90, 180, 90, 180), 0D); - // Test half a circle on the equator, using WGS84 equatorial earth radius - double earthRadiusMs = 6_378_137D; + // Test half a circle on the equator, using WGS84 mean earth radius in meters + double earthRadiusMs = 6_371_008.7714; double halfCircle = earthRadiusMs * Math.PI; assertEquals(halfCircle, haversinMeters(0, 0, 0, 180), 0D); @@ -111,17 +111,17 @@ public void testHaversin() { double randomLat2 = 40.65 + (r.nextInt(10) - 5) * 360; double randomLon2 = -73.95 + (r.nextInt(10) - 5) * 360; - assertEquals(8_581.7047, haversinMeters(randomLat1, randomLon1, randomLat2, randomLon2), 0.01D); + assertEquals(8_572.1137, haversinMeters(randomLat1, randomLon1, randomLat2, randomLon2), 0.01D); // from solr and ES tests (with their respective epsilons) assertEquals(0, haversinMeters(40.7143528, -74.0059731, 40.7143528, -74.0059731), 0D); - assertEquals(5_291.80, haversinMeters(40.7143528, -74.0059731, 40.759011, -73.9844722), 0.01D); - assertEquals(462.62, haversinMeters(40.7143528, -74.0059731, 40.718266, -74.007819), 0.01D); - assertEquals(1_056.16, haversinMeters(40.7143528, -74.0059731, 40.7051157, -74.0088305), 0.01D); - assertEquals(1_259.53, haversinMeters(40.7143528, -74.0059731, 40.7247222, -74), 0.01D); - assertEquals(2_030.79, haversinMeters(40.7143528, -74.0059731, 40.731033, -73.9962255), 0.01D); - assertEquals(8_581.70, haversinMeters(40.7143528, -74.0059731, 40.65, -73.95), 0.01D); + assertEquals(5_285.89, haversinMeters(40.7143528, -74.0059731, 40.759011, -73.9844722), 0.01D); + assertEquals(462.10, haversinMeters(40.7143528, -74.0059731, 40.718266, -74.007819), 0.01D); + assertEquals(1_054.98, haversinMeters(40.7143528, -74.0059731, 40.7051157, -74.0088305), 0.01D); + assertEquals(1_258.12, haversinMeters(40.7143528, -74.0059731, 40.7247222, -74), 0.01D); + assertEquals(2_028.52, haversinMeters(40.7143528, -74.0059731, 40.731033, -73.9962255), 0.01D); + assertEquals(8_572.11, haversinMeters(40.7143528, -74.0059731, 40.65, -73.95), 0.01D); } /** Test this method sorts the same way as real haversin */ @@ -164,6 +164,6 @@ private static double slowHaversin(double lat1, double lon1, double lat2, double double h1 = (1 - StrictMath.cos(StrictMath.toRadians(lat2) - StrictMath.toRadians(lat1))) / 2; double h2 = (1 - StrictMath.cos(StrictMath.toRadians(lon2) - StrictMath.toRadians(lon1))) / 2; double h = h1 + StrictMath.cos(StrictMath.toRadians(lat1)) * StrictMath.cos(StrictMath.toRadians(lat2)) * h2; - return 2 * 6378137 * StrictMath.asin(Math.min(1, Math.sqrt(h))); + return 2 * 6371008.7714 * StrictMath.asin(Math.min(1, Math.sqrt(h))); } } diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java index 8328f9711cb8..96ca57c683d9 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java @@ -70,12 +70,13 @@ public class DistanceFacetsExample implements Closeable { /** The "home" longitude. */ public final static double ORIGIN_LONGITUDE = -74.0059731; - /** Radius of the Earth in KM + /** Mean radius of the Earth in KM * * NOTE: this is approximate, because the earth is a bit * wider at the equator than the poles. See * http://en.wikipedia.org/wiki/Earth_radius */ - public final static double EARTH_RADIUS_KM = 6371.01; + // see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf + public final static double EARTH_RADIUS_KM = 6_371.0087714; /** Empty constructor */ public DistanceFacetsExample() {} diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java b/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java index 6b35942f6dd8..01b3394c75b0 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java @@ -224,13 +224,13 @@ public void testDistanceSort() throws Exception { TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(0.4626D, (Double)d.fields[0], 1E-4); + assertEquals(0.4621D, (Double)d.fields[0], 1E-4); d = (FieldDoc) td.scoreDocs[1]; - assertEquals(1.0562D, (Double)d.fields[0], 1E-4); + assertEquals(1.055D, (Double)d.fields[0], 1E-4); d = (FieldDoc) td.scoreDocs[2]; - assertEquals(5.2918D, (Double)d.fields[0], 1E-4); + assertEquals(5.2859D, (Double)d.fields[0], 1E-4); } public void testStaticExtendedVariableExample() throws Exception { diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java index ce38acd6330f..81362a620788 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptFunction.java @@ -158,7 +158,7 @@ public void testFloorMethod() throws Exception { } public void testHaversinMethod() throws Exception { - assertEvaluatesTo("haversin(40.7143528,-74.0059731,40.759011,-73.9844722)", 5.291799723323441); + assertEvaluatesTo("haversin(40.7143528,-74.0059731,40.759011,-73.9844722)", 5.285885589128259); } public void testLnMethod() throws Exception { diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index cd38b0ebd9fd..904532da7730 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -62,13 +62,13 @@ public void testDistanceSort() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.6174876948475D, (Double)d.fields[0], 0.0D); + assertEquals(462.1004647449412, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; - assertEquals(1056.163041670945D, (Double)d.fields[0], 0.0D); + assertEquals(1054.9826700985088, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[2]; - assertEquals(5291.798081190281D, (Double)d.fields[0], 0.0D); + assertEquals(5285.883948830351, (Double)d.fields[0], 0.0D); reader.close(); dir.close(); @@ -99,10 +99,10 @@ public void testMissingLast() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.6174876948475D, (Double)d.fields[0], 0.0D); + assertEquals(462.1004647449412D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; - assertEquals(1056.163041670945D, (Double)d.fields[0], 0.0D); + assertEquals(1054.9826700985088, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[2]; assertEquals(Double.POSITIVE_INFINITY, (Double)d.fields[0], 0.0D); diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java index 6d7f615ba2f1..0d510f37428b 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java @@ -53,8 +53,9 @@ public final class GeoUtils { public static final double MAX_LAT_RADIANS = TO_RADIANS * MAX_LAT_INCL; // WGS84 earth-ellipsoid parameters - /** major (a) axis in meters */ - public static final double SEMIMAJOR_AXIS = 6_378_137; // [m] + /** mean earth axis in meters */ + // see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf + public static final double EARTH_MEAN_RADIUS_METERS = 6_371_008.7714; // No instance: private GeoUtils() { @@ -79,7 +80,7 @@ public static GeoRect circleToBBox(final double centerLat, final double centerLo final double radLat = TO_RADIANS * centerLat; final double radLon = TO_RADIANS * centerLon; // LUCENE-7143 - double radDistance = (radiusMeters + 7E-2) / SEMIMAJOR_AXIS; + double radDistance = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; double minLat = radLat - radDistance; double maxLat = radLat + radDistance; double minLon; @@ -129,7 +130,7 @@ private static double sloppySin(double a) { } /** maximum error from {@link #axisLat(double, double)}. logic must be prepared to handle this */ - public static final double AXISLAT_ERROR = 0.1D / SEMIMAJOR_AXIS * TO_DEGREES; + public static final double AXISLAT_ERROR = 0.1D / EARTH_MEAN_RADIUS_METERS * TO_DEGREES; /** * Calculate the latitude of a circle's intersections with its bbox meridians. @@ -156,7 +157,7 @@ public static double axisLat(double centerLat, double radiusMeters) { // the argument to arc cosine, resulting in a range (0, PI/2]. double l1 = TO_RADIANS * centerLat; - double r = (radiusMeters + 7E-2) / SEMIMAJOR_AXIS; + double r = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; // if we are within radius range of a pole, the lat is the pole itself if (Math.abs(l1) + r >= MAX_LAT_RADIANS) { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 1b18a18fbc5c..2b5f885d9495 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -957,7 +957,7 @@ protected void verifyRandomDistances(boolean small, double[] lats, double[] lons radiusMeters = random().nextDouble() * 333000 + 1.0; } else { // So the query can cover at most 50% of the earth's surface: - radiusMeters = random().nextDouble() * GeoUtils.SEMIMAJOR_AXIS * Math.PI / 2.0 + 1.0; + radiusMeters = random().nextDouble() * GeoUtils.EARTH_MEAN_RADIUS_METERS * Math.PI / 2.0 + 1.0; } if (VERBOSE) { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index f3b9ad1a21cd..5cb9fc97365e 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -265,7 +265,7 @@ public void testInfiniteRect() { } public void testAxisLat() { - double earthCircumference = 2D * Math.PI * GeoUtils.SEMIMAJOR_AXIS; + double earthCircumference = 2D * Math.PI * GeoUtils.EARTH_MEAN_RADIUS_METERS; assertEquals(90, GeoUtils.axisLat(0, earthCircumference / 4), 0.0D); for (int i = 0; i < 100; ++i) { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index 6b34518a769b..45e17b72f3b3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -46,11 +46,8 @@ * @lucene.experimental */ public final class Geo3DPoint extends Field { - /** Mean radius of the earth, in meters */ - protected final static double MEAN_EARTH_RADIUS_METERS = 6371008.7714; - /** How many radians are in one earth surface meter */ - protected final static double RADIANS_PER_METER = 1.0 / MEAN_EARTH_RADIUS_METERS; + protected final static double RADIANS_PER_METER = 1.0 / PlanetModel.WGS84_MEAN; /** Indexing {@link FieldType}. */ public static final FieldType TYPE = new FieldType(); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java index 9f5385170cc8..c7d45a826c4f 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java @@ -26,7 +26,8 @@ public class PlanetModel { public static final PlanetModel SPHERE = new PlanetModel(1.0,1.0); /** Mean radius */ - public static final double WGS84_MEAN = 6371009.0; + // see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf + public static final double WGS84_MEAN = 6371008.7714; /** Polar radius */ public static final double WGS84_POLAR = 6356752.314245; /** Equatorial radius */ diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index ccaf0937f730..398458c907f4 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -796,7 +796,7 @@ public void collect(int doc) { public void testToString() { Geo3DPoint point = new Geo3DPoint("point", 44.244272, 7.769736); - assertEquals("Geo3DPoint ", point.toString()); + assertEquals("Geo3DPoint ", point.toString()); } public void testShapeQueryToString() { diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java index 186bf4c650ee..ca8b669ff185 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java @@ -211,21 +211,6 @@ public void testCircleBounds() { xyzb.getMinimumX(), xyzb.getMaximumX(), xyzb.getMinimumY(), xyzb.getMaximumY(), xyzb.getMinimumZ(), xyzb.getMaximumZ()); assertTrue(GeoArea.WITHIN == area.getRelationship(c) || GeoArea.OVERLAPS == area.getRelationship(c)); - // Yet another test case from BKD - c = GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, 0.006229478708446979, 0.005570196723795424, 3.840276763694387E-5); - xyzb = new XYZBounds(); - c.getBounds(xyzb); - area = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, - xyzb.getMinimumX(), xyzb.getMaximumX(), xyzb.getMinimumY(), xyzb.getMaximumY(), xyzb.getMinimumZ(), xyzb.getMaximumZ()); - p1 = new GeoPoint(PlanetModel.WGS84, 0.006224927111830945, 0.005597367237251763); - p2 = new GeoPoint(1.0010836083810235, 0.005603490759433942, 0.006231850560862502); - assertTrue(PlanetModel.WGS84.pointOnSurface(p1)); - //assertTrue(PlanetModel.WGS84.pointOnSurface(p2)); - assertTrue(c.isWithin(p1)); - assertTrue(c.isWithin(p2)); - assertTrue(area.isWithin(p1)); - assertTrue(area.isWithin(p2)); - // Another test case from BKD c = GeoCircleFactory.makeGeoCircle(PlanetModel.SPHERE, -0.005955031040627789, -0.0029274772647399153, 1.601488279374338E-5); xyzb = new XYZBounds(); From d9d88b8941108bf9d30c1e47ab881e94e6faabf2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 1 Apr 2016 08:32:16 -0400 Subject: [PATCH 0257/1113] LUCENE-7160: LatLonPoint quantization should use the same rounding mode as GeoPointField --- .../apache/lucene/document/LatLonPoint.java | 4 ++-- .../lucene/document/TestLatLonPoint.java | 19 ++++++++++++++++--- .../document/TestLatLonPointDistanceSort.java | 10 +++++----- .../lucene/spatial/util/TestGeoUtils.java | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 6e9e7154be54..70149b641a5f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -143,7 +143,7 @@ public static int encodeLatitude(double latitude) { if (latitude == 90.0D) { latitude = Math.nextDown(latitude); } - return Math.toIntExact((long) (latitude * LATITUDE_ENCODE)); + return (int) Math.floor(latitude * LATITUDE_ENCODE); } /** @@ -158,7 +158,7 @@ public static int encodeLongitude(double longitude) { if (longitude == 180.0D) { longitude = Math.nextDown(longitude); } - return Math.toIntExact((long) (longitude * LONGITUDE_ENCODE)); + return (int) Math.floor(longitude * LONGITUDE_ENCODE); } /** diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 55c8062a31d1..045f8baf8bef 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -23,10 +23,10 @@ public class TestLatLonPoint extends LuceneTestCase { public void testToString() throws Exception { // looks crazy due to lossiness - assertEquals("LatLonPoint ",(new LatLonPoint("field", 18.313694, -65.227444)).toString()); + assertEquals("LatLonPoint ",(new LatLonPoint("field", 18.313694, -65.227444)).toString()); // looks crazy due to lossiness - assertEquals("field:[17.99999997485429 TO 18.999999999068677],[-65.9999999217689 TO -64.99999998137355]", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); + assertEquals("field:[17.99999997485429 TO 18.999999999068677],[-66.00000000558794 TO -65.00000006519258]", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); // distance query does not quantize inputs assertEquals("field:18.0,19.0 +/- 25.0 meters", LatLonPoint.newDistanceQuery("field", 18, 19, 25).toString()); @@ -84,5 +84,18 @@ public void testEncodeDecodeIsStable() throws Exception { assertEquals(latEnc, latEnc2, 0.0); assertEquals(lonEnc, lonEnc2, 0.0); } - } + } + + /** make sure values always go down: this is important for edge case consistency */ + public void testEncodeDecodeRoundsDown() throws Exception { + int iters = atLeast(1000); + for(int iter=0;iter blockCacheFileTypes, boolean blockCacheReadEnabled, boolean blockCacheWriteEnabled) throws IOException { + this(dirName, directory, cache, blockCacheFileTypes, blockCacheReadEnabled, blockCacheWriteEnabled, true, true); + } + + public BlockDirectory(String dirName, Directory directory, Cache cache, + Set blockCacheFileTypes, boolean blockCacheReadEnabled, + boolean blockCacheWriteEnabled, boolean cacheMerges, boolean cacheReadOnce) throws IOException { super(directory); + this.cacheMerges = cacheMerges; + this.cacheReadOnce = cacheReadOnce; this.dirName = dirName; blockSize = BLOCK_SIZE; this.cache = cache; @@ -284,6 +295,17 @@ boolean useReadCache(String name, IOContext context) { return false; } switch (context.context) { + // depending on params, we don't cache on merges or when only reading once + case MERGE: { + return cacheMerges; + } + case READ: { + if (context.readOnce) { + return cacheReadOnce; + } else { + return true; + } + } default: { return true; } From 637dce83e29fe23735256f25f0c9881ed2546520 Mon Sep 17 00:00:00 2001 From: nknize Date: Fri, 1 Apr 2016 11:03:11 -0500 Subject: [PATCH 0259/1113] LUCENE-7152: Refactor GeoUtils from lucene-spatial to core module. --- lucene/CHANGES.txt | 3 + .../java/org/apache/lucene/geo/GeoUtils.java | 94 +++++++++ .../org/apache/lucene/geo/package-info.java | 21 ++ .../apache/lucene/document/LatLonPoint.java | 2 +- .../LatLonPointDistanceComparator.java | 4 +- .../document/LatLonPointDistanceQuery.java | 8 +- .../document/LatLonPointInPolygonQuery.java | 2 +- .../lucene/document/LatLonPointSortField.java | 2 +- .../geopoint/document/GeoPointField.java | 2 +- .../search/GeoPointDistanceQuery.java | 4 +- .../search/GeoPointDistanceQueryImpl.java | 5 +- .../geopoint/search/GeoPointInBBoxQuery.java | 2 +- .../search/GeoPointInPolygonQuery.java | 2 +- .../search/GeoPointMultiTermQuery.java | 2 +- .../lucene/spatial/util/GeoEncodingUtils.java | 4 +- .../apache/lucene/spatial/util/GeoRect.java | 121 ++++++++++++ .../apache/lucene/spatial/util/GeoUtils.java | 179 ------------------ .../apache/lucene/spatial/util/Polygon.java | 20 +- .../spatial/util/BaseGeoPointTestCase.java | 1 + .../lucene/spatial/util/GeoTestUtil.java | 5 +- .../lucene/spatial/util/TestGeoUtils.java | 31 +-- 21 files changed, 280 insertions(+), 234 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/geo/GeoUtils.java create mode 100644 lucene/core/src/java/org/apache/lucene/geo/package-info.java delete mode 100644 lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 33bf3a596955..a35f83cdce74 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -15,6 +15,9 @@ New Features API Changes +* LUCENE-7152: Refactor GeoUtils from lucene-spatial package to + core (Nick Knize) + * LUCENE-7141: Switch OfflineSorter's ByteSequencesReader to BytesRefIterator (Mike McCandless) diff --git a/lucene/core/src/java/org/apache/lucene/geo/GeoUtils.java b/lucene/core/src/java/org/apache/lucene/geo/GeoUtils.java new file mode 100644 index 000000000000..c11dfe185bd5 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/geo/GeoUtils.java @@ -0,0 +1,94 @@ +package org.apache.lucene.geo; + +import static org.apache.lucene.util.SloppyMath.TO_RADIANS; +import static org.apache.lucene.util.SloppyMath.cos; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Basic reusable geo-spatial utility methods + * + * @lucene.experimental + */ +public final class GeoUtils { + /** Minimum longitude value. */ + public static final double MIN_LON_INCL = -180.0D; + + /** Maximum longitude value. */ + public static final double MAX_LON_INCL = 180.0D; + + /** Minimum latitude value. */ + public static final double MIN_LAT_INCL = -90.0D; + + /** Maximum latitude value. */ + public static final double MAX_LAT_INCL = 90.0D; + + /** min longitude value in radians */ + public static final double MIN_LON_RADIANS = TO_RADIANS * MIN_LON_INCL; + /** min latitude value in radians */ + public static final double MIN_LAT_RADIANS = TO_RADIANS * MIN_LAT_INCL; + /** max longitude value in radians */ + public static final double MAX_LON_RADIANS = TO_RADIANS * MAX_LON_INCL; + /** max latitude value in radians */ + public static final double MAX_LAT_RADIANS = TO_RADIANS * MAX_LAT_INCL; + + // WGS84 earth-ellipsoid parameters + /** mean earth axis in meters */ + // see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf + public static final double EARTH_MEAN_RADIUS_METERS = 6_371_008.7714; + + // No instance: + private GeoUtils() { + } + + /** validates latitude value is within standard +/-90 coordinate bounds */ + public static void checkLatitude(double latitude) { + if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) { + throw new IllegalArgumentException("invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL); + } + } + + /** validates longitude value is within standard +/-180 coordinate bounds */ + public static void checkLongitude(double longitude) { + if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) { + throw new IllegalArgumentException("invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL); + } + } + + // some sloppyish stuff, do we really need this to be done in a sloppy way? + // unless it is performance sensitive, we should try to remove. + private static final double PIO2 = Math.PI / 2D; + + /** + * Returns the trigonometric sine of an angle converted as a cos operation. + *

    + * Note that this is not quite right... e.g. sin(0) != 0 + *

    + * Special cases: + *

      + *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. + *
    + * @param a an angle, in radians. + * @return the sine of the argument. + * @see Math#sin(double) + */ + // TODO: deprecate/remove this? at least its no longer public. + public static double sloppySin(double a) { + return cos(a - PIO2); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/geo/package-info.java b/lucene/core/src/java/org/apache/lucene/geo/package-info.java new file mode 100644 index 000000000000..8d9afc9e4d29 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/geo/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Geospatial Utility Implementations for Lucene Core + */ +package org.apache.lucene.geo; \ No newline at end of file diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 70149b641a5f..af014cba3c69 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -16,6 +16,7 @@ */ package org.apache.lucene.document; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.index.DocValuesType; @@ -28,7 +29,6 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.spatial.util.Polygon; /** diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index e818e1d2baf3..0ed85adf09ee 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -27,7 +27,7 @@ import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.SloppyMath; /** @@ -83,7 +83,7 @@ public void setBottom(int slot) { // sampling if we get called way too much: don't make gobs of bounding // boxes if comparator hits a worst case order (e.g. backwards distance order) if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { - GeoRect box = GeoUtils.circleToBBox(latitude, longitude, haversin2(bottom)); + GeoRect box = GeoRect.fromPointDistance(latitude, longitude, haversin2(bottom)); // pre-encode our box to our integer encoding, so we don't have to decode // to double values for uncompetitive hits. This has some cost! minLat = LatLonPoint.encodeLatitude(box.minLat); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 04d37233c956..29347189f8b6 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -36,7 +36,7 @@ import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; @@ -71,7 +71,7 @@ public LatLonPointDistanceQuery(String field, double latitude, double longitude, @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - GeoRect box = GeoUtils.circleToBBox(latitude, longitude, radiusMeters); + GeoRect box = GeoRect.fromPointDistance(latitude, longitude, radiusMeters); // create bounding box(es) for the distance range // these are pre-encoded with LatLonPoint's encoding final byte minLat[] = new byte[Integer.BYTES]; @@ -108,7 +108,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I maxPartialDistance = Double.POSITIVE_INFINITY; } - final double axisLat = GeoUtils.axisLat(latitude, radiusMeters); + final double axisLat = GeoRect.axisLat(latitude, radiusMeters); return new ConstantScoreWeight(this) { @@ -196,7 +196,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - if ((longitude < lonMin || longitude > lonMax) && (axisLat+GeoUtils.AXISLAT_ERROR < latMin || axisLat-GeoUtils.AXISLAT_ERROR > latMax)) { + if ((longitude < lonMin || longitude > lonMax) && (axisLat+GeoRect.AXISLAT_ERROR < latMin || axisLat-GeoRect.AXISLAT_ERROR > latMax)) { // circle not fully inside / crossing axis if (SloppyMath.haversinMeters(latitude, longitude, latMin, lonMin) > radiusMeters && SloppyMath.haversinMeters(latitude, longitude, latMin, lonMax) > radiusMeters && diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 2875e2f04238..56e906bc59de 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -84,7 +84,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // bounding box over all polygons, this can speed up tree intersection/cheaply improve approximation for complex multi-polygons // these are pre-encoded with LatLonPoint's encoding - final GeoRect box = Polygon.getBoundingBox(polygons); + final GeoRect box = GeoRect.fromPolygon(polygons); final byte minLat[] = new byte[Integer.BYTES]; final byte maxLat[] = new byte[Integer.BYTES]; final byte minLon[] = new byte[Integer.BYTES]; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java index 709122977d55..c88643847435 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointSortField.java @@ -20,7 +20,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; /** * Sorts by distance from an origin location. diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java index 378e513d8a19..8b1483abbae1 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/document/GeoPointField.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.spatial.util.GeoEncodingUtils; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; /** *

    diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java index cdd0d6d8e694..c7ac6eb91fcc 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java @@ -22,7 +22,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; /** Implements a simple point distance query on a GeoPoint field. This is based on * {@link GeoPointInBBoxQuery} and is implemented using a two phase approach. First, @@ -80,7 +80,7 @@ public GeoPointDistanceQuery(final String field, final double centerLat, final d * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter **/ public GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, final double radiusMeters) { - this(field, termEncoding, GeoUtils.circleToBBox(checkLatitude(centerLat), checkLongitude(centerLon), checkRadius(radiusMeters)), centerLat, centerLon, radiusMeters); + this(field, termEncoding, GeoRect.fromPointDistance(centerLat, centerLon, checkRadius(radiusMeters)), centerLat, centerLon, radiusMeters); } private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final GeoRect bbox, diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index dea2be80f81b..46dcce9553fb 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -19,7 +19,6 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.GeoUtils; import org.apache.lucene.util.SloppyMath; /** Package private implementation for the public facing GeoPointDistanceQuery delegate class. @@ -50,7 +49,7 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { } else { maxPartialDistance = Double.POSITIVE_INFINITY; } - axisLat = GeoUtils.axisLat(distanceQuery.centerLat, distanceQuery.radiusMeters); + axisLat = GeoRect.axisLat(distanceQuery.centerLat, distanceQuery.radiusMeters); } @Override @@ -76,7 +75,7 @@ protected boolean cellCrosses(final double minLat, final double maxLat, final do minLat > GeoPointDistanceQueryImpl.this.maxLat || minLon > GeoPointDistanceQueryImpl.this.maxLon) { return false; - } else if ((centerLon < minLon || centerLon > maxLon) && (axisLat+GeoUtils.AXISLAT_ERROR < minLat || axisLat-GeoUtils.AXISLAT_ERROR > maxLat)) { + } else if ((centerLon < minLon || centerLon > maxLon) && (axisLat+GeoRect.AXISLAT_ERROR < minLat || axisLat-GeoRect.AXISLAT_ERROR > maxLat)) { if (SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, minLon) > distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, maxLon) > distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, minLon) > distanceQuery.radiusMeters && diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java index 4229b28a637a..f30950e31258 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInBBoxQuery.java @@ -23,7 +23,7 @@ import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; /** Implements a simple bounding box query on a GeoPoint field. This is inspired by * {@link LegacyNumericRangeQuery} and is implemented using a diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java index 4e6bdcee5859..0d29d252b5a9 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java @@ -82,7 +82,7 @@ public GeoPointInPolygonQuery(final String field, final TermEncoding termEncodin * that fall within or on the boundary of the polygon defined by the input parameters. */ public GeoPointInPolygonQuery(String field, TermEncoding termEncoding, Polygon... polygons) { - this(field, termEncoding, Polygon.getBoundingBox(polygons), polygons); + this(field, termEncoding, GeoRect.fromPolygon(polygons), polygons); } // internal constructor diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java index ec08475035ec..f1f53e33f3fe 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointMultiTermQuery.java @@ -29,7 +29,7 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.spatial.util.GeoRelationUtils; -import org.apache.lucene.spatial.util.GeoUtils; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.SloppyMath; /** diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java index 7bed892bc81b..d2141d93c15e 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoEncodingUtils.java @@ -21,8 +21,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import static org.apache.lucene.spatial.util.GeoUtils.MIN_LON_INCL; -import static org.apache.lucene.spatial.util.GeoUtils.MIN_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LON_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL; /** * Basic reusable geopoint encoding methods diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java index 2dbba6d60119..391f6d78612e 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java @@ -16,6 +16,27 @@ */ package org.apache.lucene.spatial.util; +import org.apache.lucene.geo.GeoUtils; + +import static java.lang.Math.PI; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.Math.toDegrees; +import static java.lang.Math.toRadians; +import static org.apache.lucene.geo.GeoUtils.checkLatitude; +import static org.apache.lucene.geo.GeoUtils.checkLongitude; +import static org.apache.lucene.geo.GeoUtils.MAX_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MAX_LAT_RADIANS; +import static org.apache.lucene.geo.GeoUtils.MAX_LON_RADIANS; +import static org.apache.lucene.geo.GeoUtils.MIN_LAT_RADIANS; +import static org.apache.lucene.geo.GeoUtils.MIN_LON_RADIANS; +import static org.apache.lucene.geo.GeoUtils.EARTH_MEAN_RADIUS_METERS; +import static org.apache.lucene.geo.GeoUtils.sloppySin; +import static org.apache.lucene.util.SloppyMath.TO_DEGREES; +import static org.apache.lucene.util.SloppyMath.asin; +import static org.apache.lucene.util.SloppyMath.cos; + /** Represents a lat/lon rectangle. */ public class GeoRect { /** maximum longitude value (in degrees) */ @@ -67,4 +88,104 @@ public String toString() { public boolean crossesDateline() { return maxLon < minLon; } + + /** Compute Bounding Box for a circle using WGS-84 parameters */ + public static GeoRect fromPointDistance(final double centerLat, final double centerLon, final double radiusMeters) { + checkLatitude(centerLat); + checkLongitude(centerLon); + final double radLat = toRadians(centerLat); + final double radLon = toRadians(centerLon); + // LUCENE-7143 + double radDistance = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; + double minLat = radLat - radDistance; + double maxLat = radLat + radDistance; + double minLon; + double maxLon; + + if (minLat > MIN_LAT_RADIANS && maxLat < MAX_LAT_RADIANS) { + double deltaLon = asin(sloppySin(radDistance) / cos(radLat)); + minLon = radLon - deltaLon; + if (minLon < MIN_LON_RADIANS) { + minLon += 2d * PI; + } + maxLon = radLon + deltaLon; + if (maxLon > MAX_LON_RADIANS) { + maxLon -= 2d * PI; + } + } else { + // a pole is within the distance + minLat = max(minLat, MIN_LAT_RADIANS); + maxLat = min(maxLat, MAX_LAT_RADIANS); + minLon = MIN_LON_RADIANS; + maxLon = MAX_LON_RADIANS; + } + + return new GeoRect(toDegrees(minLat), toDegrees(maxLat), toDegrees(minLon), toDegrees(maxLon)); + } + + /** maximum error from {@link #axisLat(double, double)}. logic must be prepared to handle this */ + public static final double AXISLAT_ERROR = 0.1D / EARTH_MEAN_RADIUS_METERS * TO_DEGREES; + + /** + * Calculate the latitude of a circle's intersections with its bbox meridians. + *

    + * NOTE: the returned value will be +/- {@link #AXISLAT_ERROR} of the actual value. + * @param centerLat The latitude of the circle center + * @param radiusMeters The radius of the circle in meters + * @return A latitude + */ + public static double axisLat(double centerLat, double radiusMeters) { + // A spherical triangle with: + // r is the radius of the circle in radians + // l1 is the latitude of the circle center + // l2 is the latitude of the point at which the circle intersect's its bbox longitudes + // We know r is tangent to the bbox meridians at l2, therefore it is a right angle. + // So from the law of cosines, with the angle of l1 being 90, we have: + // cos(l1) = cos(r) * cos(l2) + sin(r) * sin(l2) * cos(90) + // The second part cancels out because cos(90) == 0, so we have: + // cos(l1) = cos(r) * cos(l2) + // Solving for l2, we get: + // l2 = acos( cos(l1) / cos(r) ) + // We ensure r is in the range (0, PI/2) and l1 in the range (0, PI/2]. This means we + // cannot divide by 0, and we will always get a positive value in the range [0, 1) as + // the argument to arc cosine, resulting in a range (0, PI/2]. + final double PIO2 = Math.PI / 2D; + double l1 = toRadians(centerLat); + double r = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; + + // if we are within radius range of a pole, the lat is the pole itself + if (Math.abs(l1) + r >= MAX_LAT_RADIANS) { + return centerLat >= 0 ? MAX_LAT_INCL : MIN_LAT_INCL; + } + + // adjust l1 as distance from closest pole, to form a right triangle with bbox meridians + // and ensure it is in the range (0, PI/2] + l1 = centerLat >= 0 ? PIO2 - l1 : l1 + PIO2; + + double l2 = Math.acos(Math.cos(l1) / Math.cos(r)); + assert !Double.isNaN(l2); + + // now adjust back to range [-pi/2, pi/2], ie latitude in radians + l2 = centerLat >= 0 ? PIO2 - l2 : l2 - PIO2; + + return toDegrees(l2); + } + + /** Returns the bounding box over an array of polygons */ + public static GeoRect fromPolygon(Polygon[] polygons) { + // compute bounding box + double minLat = Double.POSITIVE_INFINITY; + double maxLat = Double.NEGATIVE_INFINITY; + double minLon = Double.POSITIVE_INFINITY; + double maxLon = Double.NEGATIVE_INFINITY; + + for (int i = 0;i < polygons.length; i++) { + minLat = Math.min(polygons[i].minLat, minLat); + maxLat = Math.max(polygons[i].maxLat, maxLat); + minLon = Math.min(polygons[i].minLon, minLon); + maxLon = Math.max(polygons[i].maxLon, maxLon); + } + + return new GeoRect(minLat, maxLat, minLon, maxLon); + } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java deleted file mode 100644 index 0d510f37428b..000000000000 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoUtils.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.spatial.util; - -import static java.lang.Math.max; -import static java.lang.Math.min; -import static java.lang.Math.PI; - -import static org.apache.lucene.util.SloppyMath.asin; -import static org.apache.lucene.util.SloppyMath.cos; -import static org.apache.lucene.util.SloppyMath.TO_DEGREES; -import static org.apache.lucene.util.SloppyMath.TO_RADIANS; - -/** - * Basic reusable geo-spatial utility methods - * - * @lucene.experimental - */ -public final class GeoUtils { - /** Minimum longitude value. */ - public static final double MIN_LON_INCL = -180.0D; - - /** Maximum longitude value. */ - public static final double MAX_LON_INCL = 180.0D; - - /** Minimum latitude value. */ - public static final double MIN_LAT_INCL = -90.0D; - - /** Maximum latitude value. */ - public static final double MAX_LAT_INCL = 90.0D; - - /** min longitude value in radians */ - public static final double MIN_LON_RADIANS = TO_RADIANS * MIN_LON_INCL; - /** min latitude value in radians */ - public static final double MIN_LAT_RADIANS = TO_RADIANS * MIN_LAT_INCL; - /** max longitude value in radians */ - public static final double MAX_LON_RADIANS = TO_RADIANS * MAX_LON_INCL; - /** max latitude value in radians */ - public static final double MAX_LAT_RADIANS = TO_RADIANS * MAX_LAT_INCL; - - // WGS84 earth-ellipsoid parameters - /** mean earth axis in meters */ - // see http://earth-info.nga.mil/GandG/publications/tr8350.2/wgs84fin.pdf - public static final double EARTH_MEAN_RADIUS_METERS = 6_371_008.7714; - - // No instance: - private GeoUtils() { - } - - /** validates latitude value is within standard +/-90 coordinate bounds */ - public static void checkLatitude(double latitude) { - if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) { - throw new IllegalArgumentException("invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL); - } - } - - /** validates longitude value is within standard +/-180 coordinate bounds */ - public static void checkLongitude(double longitude) { - if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) { - throw new IllegalArgumentException("invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL); - } - } - - /** Compute Bounding Box for a circle using WGS-84 parameters */ - public static GeoRect circleToBBox(final double centerLat, final double centerLon, final double radiusMeters) { - final double radLat = TO_RADIANS * centerLat; - final double radLon = TO_RADIANS * centerLon; - // LUCENE-7143 - double radDistance = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; - double minLat = radLat - radDistance; - double maxLat = radLat + radDistance; - double minLon; - double maxLon; - - if (minLat > MIN_LAT_RADIANS && maxLat < MAX_LAT_RADIANS) { - double deltaLon = asin(sloppySin(radDistance) / cos(radLat)); - minLon = radLon - deltaLon; - if (minLon < MIN_LON_RADIANS) { - minLon += 2d * PI; - } - maxLon = radLon + deltaLon; - if (maxLon > MAX_LON_RADIANS) { - maxLon -= 2d * PI; - } - } else { - // a pole is within the distance - minLat = max(minLat, MIN_LAT_RADIANS); - maxLat = min(maxLat, MAX_LAT_RADIANS); - minLon = MIN_LON_RADIANS; - maxLon = MAX_LON_RADIANS; - } - - return new GeoRect(TO_DEGREES * minLat, TO_DEGREES * maxLat, TO_DEGREES * minLon, TO_DEGREES * maxLon); - } - - // some sloppyish stuff, do we really need this to be done in a sloppy way? - // unless it is performance sensitive, we should try to remove. - private static final double PIO2 = Math.PI / 2D; - - /** - * Returns the trigonometric sine of an angle converted as a cos operation. - *

    - * Note that this is not quite right... e.g. sin(0) != 0 - *

    - * Special cases: - *

      - *
    • If the argument is {@code NaN} or an infinity, then the result is {@code NaN}. - *
    - * @param a an angle, in radians. - * @return the sine of the argument. - * @see Math#sin(double) - */ - // TODO: deprecate/remove this? at least its no longer public. - private static double sloppySin(double a) { - return cos(a - PIO2); - } - - /** maximum error from {@link #axisLat(double, double)}. logic must be prepared to handle this */ - public static final double AXISLAT_ERROR = 0.1D / EARTH_MEAN_RADIUS_METERS * TO_DEGREES; - - /** - * Calculate the latitude of a circle's intersections with its bbox meridians. - *

    - * NOTE: the returned value will be +/- {@link #AXISLAT_ERROR} of the actual value. - * @param centerLat The latitude of the circle center - * @param radiusMeters The radius of the circle in meters - * @return A latitude - */ - public static double axisLat(double centerLat, double radiusMeters) { - // A spherical triangle with: - // r is the radius of the circle in radians - // l1 is the latitude of the circle center - // l2 is the latitude of the point at which the circle intersect's its bbox longitudes - // We know r is tangent to the bbox meridians at l2, therefore it is a right angle. - // So from the law of cosines, with the angle of l1 being 90, we have: - // cos(l1) = cos(r) * cos(l2) + sin(r) * sin(l2) * cos(90) - // The second part cancels out because cos(90) == 0, so we have: - // cos(l1) = cos(r) * cos(l2) - // Solving for l2, we get: - // l2 = acos( cos(l1) / cos(r) ) - // We ensure r is in the range (0, PI/2) and l1 in the range (0, PI/2]. This means we - // cannot divide by 0, and we will always get a positive value in the range [0, 1) as - // the argument to arc cosine, resulting in a range (0, PI/2]. - - double l1 = TO_RADIANS * centerLat; - double r = (radiusMeters + 7E-2) / EARTH_MEAN_RADIUS_METERS; - - // if we are within radius range of a pole, the lat is the pole itself - if (Math.abs(l1) + r >= MAX_LAT_RADIANS) { - return centerLat >= 0 ? MAX_LAT_INCL : MIN_LAT_INCL; - } - - // adjust l1 as distance from closest pole, to form a right triangle with bbox meridians - // and ensure it is in the range (0, PI/2] - l1 = centerLat >= 0 ? PIO2 - l1 : l1 + PIO2; - - double l2 = Math.acos(Math.cos(l1) / Math.cos(r)); - assert !Double.isNaN(l2); - - // now adjust back to range [-pi/2, pi/2], ie latitude in radians - l2 = centerLat >= 0 ? PIO2 - l2 : l2 - PIO2; - - return TO_DEGREES * l2; - } -} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java b/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java index 5dca23d35551..c0e23231de16 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java @@ -18,6 +18,8 @@ import java.util.Arrays; +import org.apache.lucene.geo.GeoUtils; + /** * Represents a closed polygon on the earth's surface. * @lucene.experimental @@ -241,24 +243,6 @@ public Polygon[] getHoles() { return holes.clone(); } - /** Returns the bounding box over an array of polygons */ - public static GeoRect getBoundingBox(Polygon[] polygons) { - // compute bounding box - double minLat = Double.POSITIVE_INFINITY; - double maxLat = Double.NEGATIVE_INFINITY; - double minLon = Double.POSITIVE_INFINITY; - double maxLon = Double.NEGATIVE_INFINITY; - - for (int i = 0;i < polygons.length; i++) { - minLat = Math.min(polygons[i].minLat, minLat); - maxLat = Math.max(polygons[i].maxLat, maxLat); - minLon = Math.min(polygons[i].minLon, minLon); - maxLon = Math.max(polygons[i].maxLon, maxLon); - } - - return new GeoRect(minLat, maxLat, minLon, maxLon); - } - /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the point */ public static boolean contains(Polygon[] polygons, double latitude, double longitude) { for (Polygon polygon : polygons) { diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 2b5f885d9495..da69b2452afb 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -37,6 +37,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java index 1785c12f4454..1e4fcf37bcea 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/GeoTestUtil.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Random; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.SloppyMath; import com.carrotsearch.randomizedtesting.RandomizedContext; @@ -398,7 +399,7 @@ public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, double rectMinLongitude, double rectMaxLongitude, double centerLatitude, double centerLongitude, double radiusMeters) { - GeoRect box = GeoUtils.circleToBBox(centerLatitude, centerLongitude, radiusMeters); + GeoRect box = GeoRect.fromPointDistance(centerLatitude, centerLongitude, radiusMeters); System.out.println(""); System.out.println(""); System.out.println(" "); @@ -420,7 +421,7 @@ public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, System.out.println(" }).addTo(earth);"); plotLatApproximatelyOnEarthSurface("lat0", "#ffffff", 4.68, 0.0, 360.0); plotLatApproximatelyOnEarthSurface("lat1", "#ffffff", 180-93.09, 0.0, 360.0); - plotLatApproximatelyOnEarthSurface("axisLat", "#00ff00", GeoUtils.axisLat(centerLatitude, radiusMeters), box.minLon, box.maxLon); + plotLatApproximatelyOnEarthSurface("axisLat", "#00ff00", GeoRect.axisLat(centerLatitude, radiusMeters), box.minLon, box.maxLon); plotLonApproximatelyOnEarthSurface("axisLon", "#00ff00", centerLongitude, box.minLat, box.maxLat); System.out.println(" }"); System.out.println(" "); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java index 7bd3fed44d2a..06ef48ca3787 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java @@ -18,6 +18,7 @@ import java.util.Locale; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -155,7 +156,7 @@ public void testRandomCircleToBBox() throws Exception { // TODO: randomly quantize radius too, to provoke exact math errors? - GeoRect bbox = GeoUtils.circleToBBox(centerLat, centerLon, radiusMeters); + GeoRect bbox = GeoRect.fromPointDistance(centerLat, centerLon, radiusMeters); int numPointsToTry = 1000; for(int i=0;i radius); } catch (AssertionError e) { GeoTestUtil.toWebGLEarth(latMin, latMax, lonMin, lonMax, centerLat, centerLon, radius); @@ -397,7 +398,7 @@ static double randomInRange(double min, double max) { } static boolean isDisjoint(double centerLat, double centerLon, double radius, double axisLat, double latMin, double latMax, double lonMin, double lonMax) { - if ((centerLon < lonMin || centerLon > lonMax) && (axisLat+GeoUtils.AXISLAT_ERROR < latMin || axisLat-GeoUtils.AXISLAT_ERROR > latMax)) { + if ((centerLon < lonMin || centerLon > lonMax) && (axisLat+GeoRect.AXISLAT_ERROR < latMin || axisLat-GeoRect.AXISLAT_ERROR > latMax)) { // circle not fully inside / crossing axis if (SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMin) > radius && SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMax) > radius && From ca9eca9de07328a198778d1dc1ab65625e9a6373 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 1 Apr 2016 13:28:11 -0400 Subject: [PATCH 0260/1113] LUCENE-7162: set SerialMergeScheduler in all spatial tests. We can test thread safety of e.g. BKD structures elsewhere in a simpler way. It is important for geo tests to be simpler and easier to reproduce and debug. --- .../document/TestLatLonPointDistanceSort.java | 7 ++++- .../spatial/util/BaseGeoPointTestCase.java | 29 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 43d3a269b209..ac402b6b6f21 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -20,7 +20,9 @@ import java.util.Arrays; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -176,7 +178,10 @@ public String toString() { private void doRandomTest(int numDocs, int numQueries) throws IOException { Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + IndexWriterConfig iwc = newIndexWriterConfig(); + // else seeds may not to reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index da69b2452afb..93b0b1cf8c82 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -498,10 +499,11 @@ public void testMultiValued() throws Exception { double[] lons = new double[2*numPoints]; Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); - initIndexWriterConfig(FIELD_NAME, iwc); // We rely on docID order: iwc.setMergePolicy(newLogMergePolicy()); + // and on seeds being able to reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); boolean small = random().nextBoolean(); @@ -724,9 +726,6 @@ protected GeoRect randomRect(boolean small) { } } - protected void initIndexWriterConfig(String field, IndexWriterConfig iwc) { - } - protected abstract void addPointToDoc(String field, Document doc, double lat, double lon); protected abstract Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon); @@ -767,6 +766,8 @@ private void verify(boolean small, double[] lats, double[] lons) throws Exceptio protected void verifyRandomRectangles(boolean small, double[] lats, double[] lons) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); // Else we can get O(N^2) merging: int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < lats.length/100) { @@ -873,6 +874,7 @@ public void collect(int doc) { } else { b.append("FAIL: id=" + id + " should not match but did\n"); } + b.append(" box=" + rect + "\n"); b.append(" query=" + query + " docID=" + docID + "\n"); b.append(" lat=" + lats[id] + " lon=" + lons[id] + "\n"); b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false)); @@ -894,6 +896,8 @@ public void collect(int doc) { protected void verifyRandomDistances(boolean small, double[] lats, double[] lons) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); // Else we can get O(N^2) merging: int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < lats.length/100) { @@ -1039,6 +1043,8 @@ public void collect(int doc) { protected void verifyRandomPolygons(boolean small, double[] lats, double[] lons) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); // Else we can get O(N^2) merging: int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < lats.length/100) { @@ -1181,6 +1187,8 @@ public void testRectBoundariesAreInclusive() throws Exception { } Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); for(int x=0;x<3;x++) { double lat; @@ -1235,6 +1243,8 @@ public void testRandomDistanceHuge() throws Exception { private void doRandomDistanceTest(int numDocs, int numQueries) throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); int pointsInLeaf = 2 + random().nextInt(4); iwc.setCodec(new FilterCodec("Lucene60", TestUtil.getDefaultCodec()) { @Override @@ -1367,10 +1377,13 @@ private TopDocs searchSmallSet(Query query, int size) throws Exception { Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random())) - .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) - .setMergePolicy(newLogMergePolicy())); + // TODO: must these simple tests really rely on docid order? + IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)); + iwc.setMergePolicy(newLogMergePolicy()); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc); for (double p[] : pts) { Document doc = new Document(); From d9c3c7e6f0692319ae7ef5e52bc47e76360d17d9 Mon Sep 17 00:00:00 2001 From: markrmiller Date: Fri, 1 Apr 2016 15:51:27 -0400 Subject: [PATCH 0261/1113] SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown. --- solr/CHANGES.txt | 2 ++ .../solr/core/CachingDirectoryFactory.java | 20 +++++-------- .../apache/solr/core/DirectoryFactory.java | 6 ++++ .../solr/core/HdfsDirectoryFactory.java | 1 + .../solr/core/ShutdownAwareDirectory.java | 30 +++++++++++++++++++ .../java/org/apache/solr/core/SolrCore.java | 4 +++ .../solr/store/blockcache/BlockDirectory.java | 10 ++++++- 7 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2aba27db6f70..17e949a93570 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -70,6 +70,8 @@ Bug Fixes * SOLR-8902: Make sure ReturnFields only returns the requested fields from (fl=) evn when DocumentTransformers ask for getExtraRequestFields() (ryan) +* SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown. (Mark Miller) + Optimizations ---------------------- * SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation. diff --git a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java index f7877c47c392..11cf479fc9da 100644 --- a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java @@ -25,27 +25,17 @@ import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.NRTCachingDirectory; -import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.lucene.store.NoLockFactory; -import org.apache.lucene.store.SimpleFSLockFactory; -import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.IOUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.ObjectReleaseTracker; -import org.apache.solr.store.blockcache.BlockDirectory; -import org.apache.solr.store.hdfs.HdfsDirectory; -import org.apache.solr.store.hdfs.HdfsLockFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -302,9 +292,15 @@ private boolean closeCacheValue(CacheValue cacheValue) { } private void close(CacheValue val) { + log.info("Closing directory, CoreContainer#isShutdown={}", coreContainer != null ? coreContainer.isShutDown() : "null"); try { - log.info("Closing directory: " + val.path); - val.directory.close(); + if (coreContainer != null && coreContainer.isShutDown() && val.directory instanceof ShutdownAwareDirectory) { + log.info("Closing directory on shutdown: " + val.path); + ((ShutdownAwareDirectory) val.directory).closeOnShutdown(); + } else { + log.info("Closing directory: " + val.path); + val.directory.close(); + } assert ObjectReleaseTracker.release(val.directory); } catch (Exception e) { SolrException.log(log, "Error closing directory", e); diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 99c35832ea71..8cc9e7d2073b 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -60,6 +60,8 @@ public enum DirContext {DEFAULT, META_DATA} public final static String LOCK_TYPE_SINGLE = "single"; public final static String LOCK_TYPE_NONE = "none"; public final static String LOCK_TYPE_HDFS = "hdfs"; + + protected volatile CoreContainer coreContainer; /** * Indicates a Directory will no longer be used, and when its ref count @@ -325,4 +327,8 @@ protected boolean deleteOldIndexDirectory(String oldDirPath) throws IOException FileUtils.deleteDirectory(dirToRm); return !dirToRm.isDirectory(); } + + public void initCoreContainer(CoreContainer cc) { + this.coreContainer = cc; + } } diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java index d22cbe3060b2..9d6797a4fb81 100644 --- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java @@ -143,6 +143,7 @@ private final static class LocalityHolder { @Override public void init(NamedList args) { + super.init(args); params = SolrParams.toSolrParams(args); this.hdfsDataDir = getConfig(HDFS_HOME, null); if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) { diff --git a/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java b/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java new file mode 100644 index 000000000000..effe6775adab --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/ShutdownAwareDirectory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + +import java.io.IOException; + +import org.apache.lucene.store.Directory; + +/** + * {@link CachingDirectoryFactory} will call this method + * rather than {@link Directory#close()} on shutdown if + * a Directory implements this interface. + */ +public interface ShutdownAwareDirectory { + public void closeOnShutdown() throws IOException; +} diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index c5e54d21a0da..e511de054c8d 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -125,6 +125,7 @@ * */ public final class SolrCore implements SolrInfoMBean, Closeable { + public static final String version="1.0"; private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -480,10 +481,13 @@ private DirectoryFactory initDirectoryFactory() { if (info != null) { log.info(info.className); dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class); + // allow DirectoryFactory instances to access the CoreContainer + dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer()); dirFactory.init(info.initArgs); } else { log.info("solr.NRTCachingDirectoryFactory"); dirFactory = new NRTCachingDirectoryFactory(); + dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer()); } return dirFactory; } diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java index d26bc44a8e41..209d6d157aef 100644 --- a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java +++ b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java @@ -29,6 +29,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.solr.core.ShutdownAwareDirectory; import org.apache.solr.store.hdfs.HdfsDirectory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +37,7 @@ /** * @lucene.experimental */ -public class BlockDirectory extends FilterDirectory { +public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block @@ -234,6 +235,13 @@ protected void closeInternal() throws IOException { } } + @Override + public void closeOnShutdown() throws IOException { + LOG.info("BlockDirectory closing on shutdown"); + // we are shutting down, no need to clean up cache + super.close(); + } + @Override public void close() throws IOException { try { From 99d16feb7e5b7be70c7bf172c19f82c4df4e6b0e Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sat, 2 Apr 2016 07:16:55 -0400 Subject: [PATCH 0262/1113] LUCENE-7166: fix quantization bugs in LatLonPoint and GeoPointField, remove test leniency Squashed commit of the following: commit 83c0f9b6158495b8b3d7108059a23bdf38e0f7f3 Author: Robert Muir Date: Fri Apr 1 23:33:29 2016 -0400 fix geopoint commit 97ebd2de516e61c236542fb2fb28e71cf6bdc403 Author: Robert Muir Date: Fri Apr 1 23:06:05 2016 -0400 fix test and LatLonPoint encoding/quantization/box queries --- lucene/CHANGES.txt | 3 + .../apache/lucene/document/LatLonPoint.java | 56 ++++++-- .../lucene/document/TestLatLonPoint.java | 136 ++++++++++++++++-- .../search/GeoPointMultiTermQuery.java | 12 +- .../apache/lucene/spatial/util/GeoRect.java | 12 +- .../spatial/util/BaseGeoPointTestCase.java | 33 ++++- .../lucene/spatial/util/TestGeoUtils.java | 15 -- 7 files changed, 212 insertions(+), 55 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a35f83cdce74..b6e56d704cfa 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -54,6 +54,9 @@ Bug Fixes * LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir) +* LUCENE-7166: Fix corner case bugs in LatLonPoint/GeoPointField bounding box + queries. (Robert Muir) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index af014cba3c69..507f543082ee 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -97,10 +97,10 @@ public LatLonPoint(String name, double latitude, double longitude) { } private static final int BITS = 32; - private static final double LONGITUDE_ENCODE = (0x1L<",(new LatLonPoint("field", 18.313694, -65.227444)).toString()); // looks crazy due to lossiness - assertEquals("field:[17.99999997485429 TO 18.999999999068677],[-66.00000000558794 TO -65.00000006519258]", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); + assertEquals("field:[18.000000016763806 TO 18.999999999068677],[-65.9999999217689 TO -65.00000006519258]", LatLonPoint.newBoxQuery("field", 18, 19, -66, -65).toString()); // distance query does not quantize inputs assertEquals("field:18.0,19.0 +/- 25.0 meters", LatLonPoint.newDistanceQuery("field", 18, 19, 25).toString()); @@ -41,11 +42,11 @@ public void testEncodeDecode() throws Exception { int iters = atLeast(10000); for(int iter=0;iter Date: Fri, 1 Apr 2016 13:12:34 -0500 Subject: [PATCH 0263/1113] LUCENE-7163: refactor GeoRect, Polygon, and GeoUtils tests to geo package in core. --- .../java/org/apache/lucene/geo}/Polygon.java | 62 +++++----- .../org/apache/lucene/geo/Rectangle.java} | 16 ++- .../org/apache/lucene/geo}/TestGeoUtils.java | 115 ++++-------------- .../apache/lucene/document/LatLonPoint.java | 2 +- .../LatLonPointDistanceComparator.java | 5 +- .../document/LatLonPointDistanceQuery.java | 8 +- .../document/LatLonPointInPolygonQuery.java | 6 +- .../document/TestLatLonPointDistanceSort.java | 2 +- .../lucene/search/TestLatLonPointQueries.java | 2 +- .../search/GeoPointDistanceQuery.java | 12 +- .../search/GeoPointDistanceQueryImpl.java | 8 +- .../search/GeoPointInPolygonQuery.java | 8 +- .../search/GeoPointInPolygonQueryImpl.java | 2 +- .../geopoint/search/TestGeoPointQuery.java | 2 +- .../search/TestLegacyGeoPointQuery.java | 2 +- .../spatial/util/BaseGeoPointTestCase.java | 17 +-- .../spatial/util/TestGeoEncodingUtils.java | 101 +++++++++++++++ .../lucene/spatial/util/TestPolygon.java | 19 ++- .../org/apache/lucene/geo}/GeoTestUtil.java | 71 ++++++----- .../org/apache/lucene/geo/package-info.java | 21 ++++ 20 files changed, 275 insertions(+), 206 deletions(-) rename lucene/{spatial/src/java/org/apache/lucene/spatial/util => core/src/java/org/apache/lucene/geo}/Polygon.java (92%) rename lucene/{spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java => core/src/java/org/apache/lucene/geo/Rectangle.java} (93%) rename lucene/{spatial/src/test/org/apache/lucene/spatial/util => core/src/test/org/apache/lucene/geo}/TestGeoUtils.java (76%) create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoEncodingUtils.java rename lucene/{spatial/src/test/org/apache/lucene/spatial/util => test-framework/src/java/org/apache/lucene/geo}/GeoTestUtil.java (94%) create mode 100644 lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java similarity index 92% rename from lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java rename to lucene/core/src/java/org/apache/lucene/geo/Polygon.java index c0e23231de16..3f32920e1480 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/Polygon.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java @@ -14,21 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.spatial.util; +package org.apache.lucene.geo; import java.util.Arrays; -import org.apache.lucene.geo.GeoUtils; - -/** +/** * Represents a closed polygon on the earth's surface. - * @lucene.experimental + * @lucene.experimental */ public final class Polygon { private final double[] polyLats; private final double[] polyLons; private final Polygon[] holes; - + /** minimum latitude of this polygon's bounding box area */ public final double minLat; /** maximum latitude of this polygon's bounding box area */ @@ -37,10 +35,13 @@ public final class Polygon { public final double minLon; /** maximum longitude of this polygon's bounding box area */ public final double maxLon; - + + // TODO: refactor to GeoUtils once LUCENE-7165 is complete + private static final double ENCODING_TOLERANCE = 1e-6; + // TODO: we could also compute the maximal inner bounding box, to make relations faster to compute? - - /** + + /** * Creates a new Polygon from the supplied latitude/longitude array, and optionally any holes. */ public Polygon(double[] polyLats, double[] polyLons, Polygon... holes) { @@ -99,14 +100,14 @@ public Polygon(double[] polyLats, double[] polyLons, Polygon... holes) { this.minLon = minLon; this.maxLon = maxLon; } - + /** Returns true if the point is contained within this polygon */ public boolean contains(double latitude, double longitude) { // check bounding box if (latitude < minLat || latitude > maxLat || longitude < minLon || longitude > maxLon) { return false; } - /* + /* * simple even-odd point in polygon computation * 1. Determine if point is contained in the longitudinal range * 2. Determine whether point crosses the edge by computing the latitudinal delta @@ -139,7 +140,7 @@ public boolean contains(double latitude, double longitude) { return false; } } - + /** * Computes whether a rectangle is within a polygon (shared boundaries not allowed) */ @@ -149,9 +150,9 @@ public boolean contains(double minLat, double maxLat, double minLon, double maxL boolean contains = crosses(minLat, maxLat, minLon, maxLon) == false && contains(minLat, minLon) && contains(minLat, maxLon) && - contains(maxLat, maxLon) && + contains(maxLat, maxLon) && contains(maxLat, minLon); - + if (contains) { // if we intersect with any hole, game over for (Polygon hole : holes) { @@ -164,7 +165,7 @@ public boolean contains(double minLat, double maxLat, double minLon, double maxL return false; } } - + /** * Convenience method for accurately computing whether a rectangle crosses a poly. */ @@ -207,14 +208,15 @@ public boolean crosses(double minLat, double maxLat, final double minLon, final c2 = a2*polyLons[p+1] + b2*polyLats[p+1]; s = (1/d)*(b2*c1 - b1*c2); t = (1/d)*(a1*c2 - a2*c1); - x00 = Math.min(bbox[b][0], bbox[b+1][0]) - GeoEncodingUtils.TOLERANCE; - x01 = Math.max(bbox[b][0], bbox[b+1][0]) + GeoEncodingUtils.TOLERANCE; - y00 = Math.min(bbox[b][1], bbox[b+1][1]) - GeoEncodingUtils.TOLERANCE; - y01 = Math.max(bbox[b][1], bbox[b+1][1]) + GeoEncodingUtils.TOLERANCE; - x10 = Math.min(polyLons[p], polyLons[p+1]) - GeoEncodingUtils.TOLERANCE; - x11 = Math.max(polyLons[p], polyLons[p+1]) + GeoEncodingUtils.TOLERANCE; - y10 = Math.min(polyLats[p], polyLats[p+1]) - GeoEncodingUtils.TOLERANCE; - y11 = Math.max(polyLats[p], polyLats[p+1]) + GeoEncodingUtils.TOLERANCE; + // todo TOLERANCE SHOULD MATCH EVERYWHERE this is currently blocked by LUCENE-7165 + x00 = Math.min(bbox[b][0], bbox[b+1][0]) - ENCODING_TOLERANCE; + x01 = Math.max(bbox[b][0], bbox[b+1][0]) + ENCODING_TOLERANCE; + y00 = Math.min(bbox[b][1], bbox[b+1][1]) - ENCODING_TOLERANCE; + y01 = Math.max(bbox[b][1], bbox[b+1][1]) + ENCODING_TOLERANCE; + x10 = Math.min(polyLons[p], polyLons[p+1]) - ENCODING_TOLERANCE; + x11 = Math.max(polyLons[p], polyLons[p+1]) + ENCODING_TOLERANCE; + y10 = Math.min(polyLats[p], polyLats[p+1]) - ENCODING_TOLERANCE; + y11 = Math.max(polyLats[p], polyLats[p+1]) + ENCODING_TOLERANCE; // check whether the intersection point is touching one of the line segments boolean touching = ((x00 == s && y00 == t) || (x01 == s && y01 == t)) || ((x10 == s && y10 == t) || (x11 == s && y11 == t)); @@ -227,22 +229,22 @@ public boolean crosses(double minLat, double maxLat, final double minLon, final } // for each bbox edge return false; } - + /** Returns a copy of the internal latitude array */ public double[] getPolyLats() { return polyLats.clone(); } - + /** Returns a copy of the internal longitude array */ public double[] getPolyLons() { return polyLons.clone(); } - + /** Returns a copy of the internal holes array */ public Polygon[] getHoles() { return holes.clone(); } - + /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the point */ public static boolean contains(Polygon[] polygons, double latitude, double longitude) { for (Polygon polygon : polygons) { @@ -252,7 +254,7 @@ public static boolean contains(Polygon[] polygons, double latitude, double longi } return false; } - + /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the rectangle */ public static boolean contains(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { for (Polygon polygon : polygons) { @@ -262,7 +264,7 @@ public static boolean contains(Polygon[] polygons, double minLat, double maxLat, } return false; } - + /** Helper for multipolygon logic: returns true if any of the supplied polygons crosses the rectangle */ public static boolean crosses(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { for (Polygon polygon : polygons) { @@ -272,7 +274,7 @@ public static boolean crosses(Polygon[] polygons, double minLat, double maxLat, } return false; } - + @Override public int hashCode() { final int prime = 31; diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java similarity index 93% rename from lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java rename to lucene/core/src/java/org/apache/lucene/geo/Rectangle.java index fdde3bf12c5b..527395ef3ddb 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/util/GeoRect.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java @@ -14,9 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.spatial.util; - -import org.apache.lucene.geo.GeoUtils; +package org.apache.lucene.geo; import static java.lang.Math.PI; import static java.lang.Math.max; @@ -38,7 +36,7 @@ import static org.apache.lucene.util.SloppyMath.cos; /** Represents a lat/lon rectangle. */ -public class GeoRect { +public class Rectangle { /** maximum longitude value (in degrees) */ public final double minLat; /** minimum longitude value (in degrees) */ @@ -51,7 +49,7 @@ public class GeoRect { /** * Constructs a bounding box by first validating the provided latitude and longitude coordinates */ - public GeoRect(double minLat, double maxLat, double minLon, double maxLon) { + public Rectangle(double minLat, double maxLat, double minLon, double maxLon) { GeoUtils.checkLatitude(minLat); GeoUtils.checkLatitude(maxLat); GeoUtils.checkLongitude(minLon); @@ -90,7 +88,7 @@ public boolean crossesDateline() { } /** Compute Bounding Box for a circle using WGS-84 parameters */ - public static GeoRect fromPointDistance(final double centerLat, final double centerLon, final double radiusMeters) { + public static Rectangle fromPointDistance(final double centerLat, final double centerLon, final double radiusMeters) { checkLatitude(centerLat); checkLongitude(centerLon); final double radLat = toRadians(centerLat); @@ -120,7 +118,7 @@ public static GeoRect fromPointDistance(final double centerLat, final double cen maxLon = MAX_LON_RADIANS; } - return new GeoRect(toDegrees(minLat), toDegrees(maxLat), toDegrees(minLon), toDegrees(maxLon)); + return new Rectangle(toDegrees(minLat), toDegrees(maxLat), toDegrees(minLon), toDegrees(maxLon)); } /** maximum error from {@link #axisLat(double, double)}. logic must be prepared to handle this */ @@ -172,7 +170,7 @@ public static double axisLat(double centerLat, double radiusMeters) { } /** Returns the bounding box over an array of polygons */ - public static GeoRect fromPolygon(Polygon[] polygons) { + public static Rectangle fromPolygon(Polygon[] polygons) { // compute bounding box double minLat = Double.POSITIVE_INFINITY; double maxLat = Double.NEGATIVE_INFINITY; @@ -186,6 +184,6 @@ public static GeoRect fromPolygon(Polygon[] polygons) { maxLon = Math.max(polygons[i].maxLon, maxLon); } - return new GeoRect(minLat, maxLat, minLon, maxLon); + return new Rectangle(minLat, maxLat, minLon, maxLon); } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java similarity index 76% rename from lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java rename to lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java index 28b885ae0b76..8727b42afa6b 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestGeoUtils.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java @@ -14,12 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.spatial.util; +package org.apache.lucene.geo; import java.util.Locale; -import org.apache.lucene.geo.GeoUtils; -import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; import org.junit.BeforeClass; @@ -61,65 +59,6 @@ public double randomLon(boolean small) { return result; } - /** - * Tests stability of {@link GeoEncodingUtils#geoCodedToPrefixCoded} - */ - public void testGeoPrefixCoding() throws Exception { - int numIters = atLeast(1000); - long hash; - long decodedHash; - BytesRefBuilder brb = new BytesRefBuilder(); - while (numIters-- >= 0) { - hash = GeoEncodingUtils.mortonHash(randomLat(false), randomLon(false)); - for (int i=32; i<64; ++i) { - GeoEncodingUtils.geoCodedToPrefixCoded(hash, i, brb); - decodedHash = GeoEncodingUtils.prefixCodedToGeoCoded(brb.get()); - assertEquals((hash >>> i) << i, decodedHash); - } - } - } - - public void testMortonEncoding() throws Exception { - long hash = GeoEncodingUtils.mortonHash(90, 180); - assertEquals(180.0, GeoEncodingUtils.mortonUnhashLon(hash), 0); - assertEquals(90.0, GeoEncodingUtils.mortonUnhashLat(hash), 0); - } - - public void testEncodeDecode() throws Exception { - int iters = atLeast(10000); - boolean small = random().nextBoolean(); - for(int iter=0;iter radius); } catch (AssertionError e) { GeoTestUtil.toWebGLEarth(latMin, latMax, lonMin, lonMax, centerLat, centerLon, radius); @@ -381,9 +320,9 @@ public void testCircleOpto() throws Exception { static double randomInRange(double min, double max) { return min + (max - min) * random().nextDouble(); } - + static boolean isDisjoint(double centerLat, double centerLon, double radius, double axisLat, double latMin, double latMax, double lonMin, double lonMax) { - if ((centerLon < lonMin || centerLon > lonMax) && (axisLat+GeoRect.AXISLAT_ERROR < latMin || axisLat-GeoRect.AXISLAT_ERROR > latMax)) { + if ((centerLon < lonMin || centerLon > lonMax) && (axisLat+ Rectangle.AXISLAT_ERROR < latMin || axisLat- Rectangle.AXISLAT_ERROR > latMax)) { // circle not fully inside / crossing axis if (SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMin) > radius && SloppyMath.haversinMeters(centerLat, centerLon, latMin, lonMax) > radius && @@ -393,7 +332,7 @@ static boolean isDisjoint(double centerLat, double centerLon, double radius, dou return true; } } - + return false; } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 507f543082ee..26bb04ad5b64 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -29,7 +29,7 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; /** * An indexed location field. diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index 0ed85adf09ee..58db5e57e67c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -26,8 +26,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; -import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.geo.GeoUtils; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.util.SloppyMath; /** @@ -83,7 +82,7 @@ public void setBottom(int slot) { // sampling if we get called way too much: don't make gobs of bounding // boxes if comparator hits a worst case order (e.g. backwards distance order) if (setBottomCounter < 1024 || (setBottomCounter & 0x3F) == 0x3F) { - GeoRect box = GeoRect.fromPointDistance(latitude, longitude, haversin2(bottom)); + Rectangle box = Rectangle.fromPointDistance(latitude, longitude, haversin2(bottom)); // pre-encode our box to our integer encoding, so we don't have to decode // to double values for uncompetitive hits. This has some cost! minLat = LatLonPoint.encodeLatitude(box.minLat); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 29347189f8b6..1b667dd25008 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -18,6 +18,7 @@ import java.io.IOException; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; @@ -35,7 +36,6 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; @@ -71,7 +71,7 @@ public LatLonPointDistanceQuery(String field, double latitude, double longitude, @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - GeoRect box = GeoRect.fromPointDistance(latitude, longitude, radiusMeters); + Rectangle box = Rectangle.fromPointDistance(latitude, longitude, radiusMeters); // create bounding box(es) for the distance range // these are pre-encoded with LatLonPoint's encoding final byte minLat[] = new byte[Integer.BYTES]; @@ -108,7 +108,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I maxPartialDistance = Double.POSITIVE_INFINITY; } - final double axisLat = GeoRect.axisLat(latitude, radiusMeters); + final double axisLat = Rectangle.axisLat(latitude, radiusMeters); return new ConstantScoreWeight(this) { @@ -196,7 +196,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - if ((longitude < lonMin || longitude > lonMax) && (axisLat+GeoRect.AXISLAT_ERROR < latMin || axisLat-GeoRect.AXISLAT_ERROR > latMax)) { + if ((longitude < lonMin || longitude > lonMax) && (axisLat+ Rectangle.AXISLAT_ERROR < latMin || axisLat- Rectangle.AXISLAT_ERROR > latMax)) { // circle not fully inside / crossing axis if (SloppyMath.haversinMeters(latitude, longitude, latMin, lonMin) > radiusMeters && SloppyMath.haversinMeters(latitude, longitude, latMin, lonMax) > radiusMeters && diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 56e906bc59de..54f5192074a8 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.search.ConstantScoreScorer; @@ -42,8 +43,7 @@ import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.SparseFixedBitSet; import org.apache.lucene.util.StringHelper; -import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; /** Finds all previously indexed points that fall within the specified polygons. * @@ -84,7 +84,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // bounding box over all polygons, this can speed up tree intersection/cheaply improve approximation for complex multi-polygons // these are pre-encoded with LatLonPoint's encoding - final GeoRect box = GeoRect.fromPolygon(polygons); + final Rectangle box = Rectangle.fromPolygon(polygons); final byte minLat[] = new byte[Integer.BYTES]; final byte maxLat[] = new byte[Integer.BYTES]; final byte minLon[] = new byte[Integer.BYTES]; diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index ac402b6b6f21..5ce819c98e93 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.geo.GeoTestUtil; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; @@ -29,7 +30,6 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.spatial.util.GeoTestUtil; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 3bde38971c3e..08b9ebf6d7c9 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -19,7 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; public class TestLatLonPointQueries extends BaseGeoPointTestCase { diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java index c7ac6eb91fcc..9e90a11af8c2 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQuery.java @@ -21,7 +21,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoRect; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.geo.GeoUtils; /** Implements a simple point distance query on a GeoPoint field. This is based on @@ -80,10 +80,10 @@ public GeoPointDistanceQuery(final String field, final double centerLat, final d * {@link org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding} parameter **/ public GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final double centerLat, final double centerLon, final double radiusMeters) { - this(field, termEncoding, GeoRect.fromPointDistance(centerLat, centerLon, checkRadius(radiusMeters)), centerLat, centerLon, radiusMeters); + this(field, termEncoding, Rectangle.fromPointDistance(centerLat, centerLon, checkRadius(radiusMeters)), centerLat, centerLon, radiusMeters); } - private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final GeoRect bbox, + private GeoPointDistanceQuery(final String field, final TermEncoding termEncoding, final Rectangle bbox, final double centerLat, final double centerLon, final double radiusMeters) { super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); @@ -105,7 +105,7 @@ public Query rewrite(IndexReader reader) { unwrappedLon += -360.0D; } GeoPointDistanceQueryImpl left = new GeoPointDistanceQueryImpl(field, termEncoding, this, unwrappedLon, - new GeoRect(minLat, maxLat, GeoUtils.MIN_LON_INCL, maxLon)); + new Rectangle(minLat, maxLat, GeoUtils.MIN_LON_INCL, maxLon)); bqb.add(new BooleanClause(left, BooleanClause.Occur.SHOULD)); if (unwrappedLon < maxLon) { @@ -113,13 +113,13 @@ public Query rewrite(IndexReader reader) { unwrappedLon += 360.0D; } GeoPointDistanceQueryImpl right = new GeoPointDistanceQueryImpl(field, termEncoding, this, unwrappedLon, - new GeoRect(minLat, maxLat, minLon, GeoUtils.MAX_LON_INCL)); + new Rectangle(minLat, maxLat, minLon, GeoUtils.MAX_LON_INCL)); bqb.add(new BooleanClause(right, BooleanClause.Occur.SHOULD)); return bqb.build(); } return new GeoPointDistanceQueryImpl(field, termEncoding, this, centerLon, - new GeoRect(this.minLat, this.maxLat, this.minLon, this.maxLon)); + new Rectangle(this.minLat, this.maxLat, this.minLon, this.maxLon)); } @Override diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java index 46dcce9553fb..a360fdbaf08a 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointDistanceQueryImpl.java @@ -16,9 +16,9 @@ */ package org.apache.lucene.spatial.geopoint.search; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.GeoRect; import org.apache.lucene.util.SloppyMath; /** Package private implementation for the public facing GeoPointDistanceQuery delegate class. @@ -36,7 +36,7 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { final double axisLat; GeoPointDistanceQueryImpl(final String field, final TermEncoding termEncoding, final GeoPointDistanceQuery q, - final double centerLonUnwrapped, final GeoRect bbox) { + final double centerLonUnwrapped, final Rectangle bbox) { super(field, termEncoding, bbox.minLat, bbox.maxLat, bbox.minLon, bbox.maxLon); distanceQuery = q; centerLon = centerLonUnwrapped; @@ -49,7 +49,7 @@ final class GeoPointDistanceQueryImpl extends GeoPointInBBoxQueryImpl { } else { maxPartialDistance = Double.POSITIVE_INFINITY; } - axisLat = GeoRect.axisLat(distanceQuery.centerLat, distanceQuery.radiusMeters); + axisLat = Rectangle.axisLat(distanceQuery.centerLat, distanceQuery.radiusMeters); } @Override @@ -75,7 +75,7 @@ protected boolean cellCrosses(final double minLat, final double maxLat, final do minLat > GeoPointDistanceQueryImpl.this.maxLat || minLon > GeoPointDistanceQueryImpl.this.maxLon) { return false; - } else if ((centerLon < minLon || centerLon > maxLon) && (axisLat+GeoRect.AXISLAT_ERROR < minLat || axisLat-GeoRect.AXISLAT_ERROR > maxLat)) { + } else if ((centerLon < minLon || centerLon > maxLon) && (axisLat+ Rectangle.AXISLAT_ERROR < minLat || axisLat- Rectangle.AXISLAT_ERROR > maxLat)) { if (SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, minLon) > distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, minLat, maxLon) > distanceQuery.radiusMeters && SloppyMath.haversinMeters(distanceQuery.centerLat, centerLon, maxLat, minLon) > distanceQuery.radiusMeters && diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java index 0d29d252b5a9..17ce54d437da 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java @@ -23,8 +23,8 @@ import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.GeoEncodingUtils; -import org.apache.lucene.spatial.util.GeoRect; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.geo.Polygon; /** Implements a simple point in polygon query on a GeoPoint field. This is based on * {@code GeoPointInBBoxQueryImpl} and is implemented using a @@ -82,11 +82,11 @@ public GeoPointInPolygonQuery(final String field, final TermEncoding termEncodin * that fall within or on the boundary of the polygon defined by the input parameters. */ public GeoPointInPolygonQuery(String field, TermEncoding termEncoding, Polygon... polygons) { - this(field, termEncoding, GeoRect.fromPolygon(polygons), polygons); + this(field, termEncoding, Rectangle.fromPolygon(polygons), polygons); } // internal constructor - private GeoPointInPolygonQuery(String field, TermEncoding termEncoding, GeoRect boundingBox, Polygon... polygons) { + private GeoPointInPolygonQuery(String field, TermEncoding termEncoding, Rectangle boundingBox, Polygon... polygons) { super(field, termEncoding, boundingBox.minLat, boundingBox.maxLat, boundingBox.minLon, boundingBox.maxLon); this.polygons = polygons.clone(); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java index 20c90785b634..1bb43c70ba47 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java @@ -20,7 +20,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; /** Package private implementation for the public facing GeoPointInPolygonQuery delegate class. * diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index 91afe3fd7e2b..ff8baca95c86 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -19,7 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index cd15f6691a56..4c8e3e32de3f 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -19,7 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; -import org.apache.lucene.spatial.util.Polygon; +import org.apache.lucene.geo.Polygon; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 182ec6f66d06..6ae4d20142eb 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -37,7 +37,10 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; +import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.geo.GeoTestUtil; import org.apache.lucene.geo.GeoUtils; +import org.apache.lucene.geo.Polygon; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -537,7 +540,7 @@ public void testMultiValued() throws Exception { int iters = atLeast(25); for (int iter=0;iter= 0) { + hash = GeoEncodingUtils.mortonHash(nextLatitude(), nextLongitude()); + for (int i=32; i<64; ++i) { + GeoEncodingUtils.geoCodedToPrefixCoded(hash, i, brb); + decodedHash = GeoEncodingUtils.prefixCodedToGeoCoded(brb.get()); + assertEquals((hash >>> i) << i, decodedHash); + } + } + } + + public void testMortonEncoding() throws Exception { + long hash = GeoEncodingUtils.mortonHash(90, 180); + assertEquals(180.0, GeoEncodingUtils.mortonUnhashLon(hash), 0); + assertEquals(90.0, GeoEncodingUtils.mortonUnhashLat(hash), 0); + } + + public void testEncodeDecode() throws Exception { + int iters = atLeast(10000); + for(int iter=0;iter 90) { @@ -299,7 +298,7 @@ private static double wrapLon(double lon) { return lon; } } - + private static void drawRectApproximatelyOnEarthSurface(String name, String color, double minLat, double maxLat, double minLon, double maxLon) { int steps = 20; System.out.println(" var " + name + " = WE.polygon(["); @@ -324,7 +323,7 @@ private static void drawRectApproximatelyOnEarthSurface(String name, String colo System.out.println(" ], {color: \"" + color + "\", fillColor: \"" + color + "\"});"); System.out.println(" " + name + ".addTo(earth);"); } - + private static void plotLatApproximatelyOnEarthSurface(String name, String color, double lat, double minLon, double maxLon) { System.out.println(" var " + name + " = WE.polygon(["); double lon; @@ -373,7 +372,7 @@ public static void polysToWebGLEarth(List polys) { double lon = poly[1][i]; System.out.println(" [" + lat + ", " + lon + "],"); } - System.out.println(" ], {color: '#00ff00'});"); + System.out.println(" ], {color: '#00ff00'});"); System.out.println(" poly" + count + ".addTo(earth);"); } @@ -399,7 +398,7 @@ public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, double rectMinLongitude, double rectMaxLongitude, double centerLatitude, double centerLongitude, double radiusMeters) { - GeoRect box = GeoRect.fromPointDistance(centerLatitude, centerLongitude, radiusMeters); + Rectangle box = Rectangle.fromPointDistance(centerLatitude, centerLongitude, radiusMeters); System.out.println(""); System.out.println(""); System.out.println(" "); @@ -413,7 +412,7 @@ public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, StringBuilder b = new StringBuilder(); inverseHaversin(b, centerLatitude, centerLongitude, radiusMeters); System.out.println(b); - System.out.println(" ], {color: '#00ff00'});"); + System.out.println(" ], {color: '#00ff00'});"); System.out.println(" polygonB.addTo(earth);"); drawRectApproximatelyOnEarthSurface("bbox", "#00ff00", box.minLat, box.maxLat, box.minLon, box.maxLon); System.out.println(" WE.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{"); @@ -421,7 +420,7 @@ public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, System.out.println(" }).addTo(earth);"); plotLatApproximatelyOnEarthSurface("lat0", "#ffffff", 4.68, 0.0, 360.0); plotLatApproximatelyOnEarthSurface("lat1", "#ffffff", 180-93.09, 0.0, 360.0); - plotLatApproximatelyOnEarthSurface("axisLat", "#00ff00", GeoRect.axisLat(centerLatitude, radiusMeters), box.minLon, box.maxLon); + plotLatApproximatelyOnEarthSurface("axisLat", "#00ff00", Rectangle.axisLat(centerLatitude, radiusMeters), box.minLon, box.maxLon); plotLonApproximatelyOnEarthSurface("axisLon", "#00ff00", centerLongitude, box.minLat, box.maxLat); System.out.println(" }"); System.out.println(" "); diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java b/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java new file mode 100644 index 000000000000..9f0340bd572a --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Reusable Geo test utilities. + */ +package org.apache.lucene.geo; \ No newline at end of file From 4e70702782b863aed9a340d8024ed2a33b03744d Mon Sep 17 00:00:00 2001 From: nknize Date: Sat, 2 Apr 2016 14:06:20 -0500 Subject: [PATCH 0264/1113] Fix missing import from refactor. Update change log --- lucene/CHANGES.txt | 3 +++ .../src/test/org/apache/lucene/document/TestLatLonPoint.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index b6e56d704cfa..edf222aa0569 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -15,6 +15,9 @@ New Features API Changes +* LUCENE-7163: refactor GeoRect, Polygon, and GeoUtils tests to geo + package in core (Nick Knize) + * LUCENE-7152: Refactor GeoUtils from lucene-spatial package to core (Nick Knize) diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index 73a368350bbd..6d50f068d32d 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -16,7 +16,7 @@ */ package org.apache.lucene.document; -import org.apache.lucene.spatial.util.GeoTestUtil; +import org.apache.lucene.geo.GeoTestUtil; import org.apache.lucene.util.LuceneTestCase; /** Simple tests for {@link LatLonPoint} */ From a70c3df7a6d4e66cba3d11785836fbd54342f8ba Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 3 Apr 2016 12:09:46 -0400 Subject: [PATCH 0265/1113] package-info.java -> package.html because this package is already defined in core/ --- .../org/apache/lucene/geo/package-info.java | 21 --------------- .../java/org/apache/lucene/geo/package.html | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+), 21 deletions(-) delete mode 100644 lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/geo/package.html diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java b/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java deleted file mode 100644 index 9f0340bd572a..000000000000 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Reusable Geo test utilities. - */ -package org.apache.lucene.geo; \ No newline at end of file diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/package.html b/lucene/test-framework/src/java/org/apache/lucene/geo/package.html new file mode 100644 index 000000000000..874d1a581d08 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/package.html @@ -0,0 +1,26 @@ + + + + + + + + +Reusable Geo test utilities. + + From 741a1e4eedbc88a684baecfcc35ed7cdabcea1b2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 3 Apr 2016 13:33:24 -0400 Subject: [PATCH 0266/1113] LUCENE-7163: move Polygon's test too --- .../src/test/org/apache/lucene/geo}/TestPolygon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lucene/{spatial/src/test/org/apache/lucene/spatial/util => core/src/test/org/apache/lucene/geo}/TestPolygon.java (99%) diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java similarity index 99% rename from lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java rename to lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java index 259aada18313..efd6d584bc5f 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/TestPolygon.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.spatial.util; +package org.apache.lucene.geo; import org.apache.lucene.geo.Polygon; import org.apache.lucene.util.LuceneTestCase; From bb2e01c3d913894062fff3f18bd5d0256f884185 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 3 Apr 2016 14:07:56 -0400 Subject: [PATCH 0267/1113] LUCENE-7159: improve testing of polygon tree methods --- .../org/apache/lucene/geo/TestPolygon.java | 108 ++++++++++++++++++ .../org/apache/lucene/geo/GeoTestUtil.java | 13 +++ 2 files changed, 121 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java index efd6d584bc5f..02025627eeda 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java @@ -145,4 +145,112 @@ public void testBoundingBoxEdgeCases() throws Exception { } } } + + /** If polygon.contains(box) returns true, then any point in that box should return true as well */ + public void testContainsRandom() throws Exception { + for (int i = 0; i < 1000; i++) { + Polygon polygon = nextPolygon(); + + for (int j = 0; j < 100; j++) { + Rectangle rectangle = GeoTestUtil.nextSimpleBox(); + // allowed to conservatively return false + if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon)) { + for (int k = 0; k < 1000; k++) { + // this tests in our range but sometimes outside! so we have to double-check its really in other box + double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); + double longitude = nextLongitudeAround(rectangle.minLon, rectangle.maxLon); + // check for sure its in our box + if (latitude >= rectangle.minLat && latitude <= rectangle.maxLat && longitude >= rectangle.minLon && longitude <= rectangle.maxLon) { + assertTrue(polygon.contains(latitude, longitude)); + } + } + } + } + } + } + + /** If polygon.contains(box) returns true, then any point in that box should return true as well */ + // different from testContainsRandom in that its not a purely random test. we iterate the vertices of the polygon + // and generate boxes near each one of those to try to be more efficient. + public void testContainsEdgeCases() throws Exception { + for (int i = 0; i < 1000; i++) { + Polygon polygon = nextPolygon(); + + double polyLats[] = polygon.getPolyLats(); + double polyLons[] = polygon.getPolyLons(); + + for (int vertex = 0; vertex < polyLats.length; vertex++) { + for (int j = 0; j < 10; j++) { + Rectangle rectangle = GeoTestUtil.nextSimpleBoxNear(polyLats[vertex], polyLons[vertex]); + // allowed to conservatively return false + if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon)) { + for (int k = 0; k < 100; k++) { + // this tests in our range but sometimes outside! so we have to double-check its really in other box + double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); + double longitude = nextLongitudeAround(rectangle.minLon, rectangle.maxLon); + // check for sure its in our box + if (latitude >= rectangle.minLat && latitude <= rectangle.maxLat && longitude >= rectangle.minLon && longitude <= rectangle.maxLon) { + assertTrue(polygon.contains(latitude, longitude)); + } + } + } + } + } + } + } + + /** If polygon.intersects(box) returns false, then any point in that box should return false as well */ + public void testIntersectRandom() { + for (int i = 0; i < 100; i++) { + Polygon polygon = nextPolygon(); + + for (int j = 0; j < 100; j++) { + Rectangle rectangle = GeoTestUtil.nextSimpleBox(); + // allowed to conservatively return true. + if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false && + polygon.crosses(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false) { + for (int k = 0; k < 1000; k++) { + // this tests in our range but sometimes outside! so we have to double-check its really in other box + double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); + double longitude = nextLongitudeAround(rectangle.minLon, rectangle.maxLon); + // check for sure its in our box + if (latitude >= rectangle.minLat && latitude <= rectangle.maxLat && longitude >= rectangle.minLon && longitude <= rectangle.maxLon) { + assertFalse(polygon.contains(latitude, longitude)); + } + } + } + } + } + } + + /** If polygon.intersects(box) returns false, then any point in that box should return false as well */ + // different from testIntersectsRandom in that its not a purely random test. we iterate the vertices of the polygon + // and generate boxes near each one of those to try to be more efficient. + public void testIntersectEdgeCases() { + for (int i = 0; i < 100; i++) { + Polygon polygon = nextPolygon(); + + double polyLats[] = polygon.getPolyLats(); + double polyLons[] = polygon.getPolyLons(); + + for (int vertex = 0; vertex < polyLats.length; vertex++) { + for (int j = 0; j < 10; j++) { + Rectangle rectangle = GeoTestUtil.nextSimpleBoxNear(polyLats[vertex], polyLons[vertex]); + // allowed to conservatively return true. + if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false && + polygon.crosses(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false) { + for (int k = 0; k < 100; k++) { + // this tests in our range but sometimes outside! so we have to double-check its really in other box + double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); + double longitude = nextLongitudeAround(rectangle.minLon, rectangle.maxLon); + // check for sure its in our box + if (latitude >= rectangle.minLat && latitude <= rectangle.maxLat && longitude >= rectangle.minLon && longitude <= rectangle.maxLon) { + assertFalse(polygon.contains(latitude, longitude)); + } + } + } + } + } + } + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java index bcb473eddfea..0d241f6dc66e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java @@ -75,6 +75,11 @@ public static double nextLongitudeAround(double minLongitude, double maxLongitud public static Rectangle nextBox() { return nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), true); } + + /** returns next pseudorandom box: will not cross the 180th meridian */ + public static Rectangle nextSimpleBox() { + return nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), false); + } /** returns next pseudorandom box, can cross the 180th meridian, kinda close to {@code otherLatitude} and {@code otherLongitude} */ public static Rectangle nextBoxNear(double otherLatitude, double otherLongitude) { @@ -83,6 +88,14 @@ public static Rectangle nextBoxNear(double otherLatitude, double otherLongitude) return nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), true); } + + /** returns next pseudorandom box, will not cross the 180th meridian, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static Rectangle nextSimpleBoxNear(double otherLatitude, double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + GeoUtils.checkLongitude(otherLongitude); + return nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), false); + } /** returns next pseudorandom polygon */ public static Polygon nextPolygon() { From 1113443fc683d80720efd42706cb932b00f2cba8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 4 Apr 2016 12:51:03 -0400 Subject: [PATCH 0268/1113] LUCENE-7159: Speed up LatLonPoint point-in-polygon performance --- lucene/CHANGES.txt | 3 + .../java/org/apache/lucene/geo/Polygon.java | 190 ++++++++++-------- .../org/apache/lucene/geo/TestPolygon.java | 38 ++-- .../apache/lucene/document/LatLonGrid.java | 155 ++++++++++++++ .../document/LatLonPointInPolygonQuery.java | 21 +- .../lucene/document/TestLatLonGrid.java | 50 +++++ .../search/GeoPointInPolygonQueryImpl.java | 8 +- 7 files changed, 348 insertions(+), 117 deletions(-) create mode 100644 lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java create mode 100644 lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index edf222aa0569..2688f44afb56 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -53,6 +53,9 @@ Optimizations multiple polygons and holes, with memory usage independent of polygon complexity. (Karl Wright, Mike McCandless, Robert Muir) +* LUCENE-7159: Speed up LatLonPoint polygon performance for complex + polygons. (Robert Muir) + Bug Fixes * LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir) diff --git a/lucene/core/src/java/org/apache/lucene/geo/Polygon.java b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java index 3f32920e1480..b643ca2bb902 100644 --- a/lucene/core/src/java/org/apache/lucene/geo/Polygon.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java @@ -18,6 +18,8 @@ import java.util.Arrays; +import org.apache.lucene.index.PointValues.Relation; + /** * Represents a closed polygon on the earth's surface. * @lucene.experimental @@ -140,90 +142,127 @@ public boolean contains(double latitude, double longitude) { return false; } } - - /** - * Computes whether a rectangle is within a polygon (shared boundaries not allowed) - */ - public boolean contains(double minLat, double maxLat, double minLon, double maxLon) { - // check if rectangle crosses poly (to handle concave/pacman polys), then check that all 4 corners - // are contained - boolean contains = crosses(minLat, maxLat, minLon, maxLon) == false && - contains(minLat, minLon) && - contains(minLat, maxLon) && - contains(maxLat, maxLon) && - contains(maxLat, minLon); - - if (contains) { - // if we intersect with any hole, game over - for (Polygon hole : holes) { - if (hole.crosses(minLat, maxLat, minLon, maxLon) || hole.contains(minLat, maxLat, minLon, maxLon)) { - return false; - } - } - return true; - } else { - return false; - } - } - - /** - * Convenience method for accurately computing whether a rectangle crosses a poly. - */ - public boolean crosses(double minLat, double maxLat, final double minLon, final double maxLon) { + + /** Returns relation to the provided rectangle */ + public Relation relate(double minLat, double maxLat, double minLon, double maxLon) { // if the bounding boxes are disjoint then the shape does not cross if (maxLon < this.minLon || minLon > this.maxLon || maxLat < this.minLat || minLat > this.maxLat) { - return false; + return Relation.CELL_OUTSIDE_QUERY; } // if the rectangle fully encloses us, we cross. if (minLat <= this.minLat && maxLat >= this.maxLat && minLon <= this.minLon && maxLon >= this.maxLon) { - return true; + return Relation.CELL_CROSSES_QUERY; } - // if we cross any hole, we cross + // check any holes for (Polygon hole : holes) { - if (hole.crosses(minLat, maxLat, minLon, maxLon)) { - return true; + Relation holeRelation = hole.relate(minLat, maxLat, minLon, maxLon); + if (holeRelation == Relation.CELL_CROSSES_QUERY) { + return Relation.CELL_CROSSES_QUERY; + } else if (holeRelation == Relation.CELL_INSIDE_QUERY) { + return Relation.CELL_OUTSIDE_QUERY; + } + } + // check each corner: if < 4 are present, its cheaper than crossesSlowly + int numCorners = numberOfCorners(minLat, maxLat, minLon, maxLon); + if (numCorners == 4) { + if (crossesSlowly(minLat, maxLat, minLon, maxLon)) { + return Relation.CELL_CROSSES_QUERY; } + return Relation.CELL_INSIDE_QUERY; + } else if (numCorners > 0) { + return Relation.CELL_CROSSES_QUERY; + } + + // we cross + if (crossesSlowly(minLat, maxLat, minLon, maxLon)) { + return Relation.CELL_CROSSES_QUERY; + } + + return Relation.CELL_OUTSIDE_QUERY; + } + + // returns 0, 4, or something in between + private int numberOfCorners(double minLat, double maxLat, double minLon, double maxLon) { + int containsCount = 0; + if (contains(minLat, minLon)) { + containsCount++; } + if (contains(minLat, maxLon)) { + containsCount++; + } + if (containsCount == 1) { + return containsCount; + } + if (contains(maxLat, maxLon)) { + containsCount++; + } + if (containsCount == 2) { + return containsCount; + } + if (contains(maxLat, minLon)) { + containsCount++; + } + return containsCount; + } + private boolean crossesSlowly(double minLat, double maxLat, final double minLon, final double maxLon) { /* * Accurately compute (within restrictions of cartesian decimal degrees) whether a rectangle crosses a polygon */ - final double[][] bbox = new double[][] { {minLon, minLat}, {maxLon, minLat}, {maxLon, maxLat}, {minLon, maxLat}, {minLon, minLat} }; - final int polyLength = polyLons.length-1; - double d, s, t, a1, b1, c1, a2, b2, c2; - double x00, y00, x01, y01, x10, y10, x11, y11; + final double[] boxLats = new double[] { minLat, minLat, maxLat, maxLat, minLat }; + final double[] boxLons = new double[] { minLon, maxLon, maxLon, minLon, minLon }; // computes the intersection point between each bbox edge and the polygon edge - for (short b=0; b<4; ++b) { - a1 = bbox[b+1][1]-bbox[b][1]; - b1 = bbox[b][0]-bbox[b+1][0]; - c1 = a1*bbox[b+1][0] + b1*bbox[b+1][1]; - for (int p=0; p s || x01 < s || y00 > t || y01 < t || x10 > s || x11 < s || y10 > t || y11 < t)) { - return true; + double x00 = Math.min(boxLons[b], boxLons[b+1]) - ENCODING_TOLERANCE; + if (x00 > s) { + continue; // out of range + } + double x01 = Math.max(boxLons[b], boxLons[b+1]) + ENCODING_TOLERANCE; + if (x01 < s) { + continue; // out of range + } + double x10 = Math.min(polyLons[p], polyLons[p+1]) - ENCODING_TOLERANCE; + if (x10 > s) { + continue; // out of range + } + double x11 = Math.max(polyLons[p], polyLons[p+1]) + ENCODING_TOLERANCE; + if (x11 < s) { + continue; // out of range + } + + double t = (1/d)*(a1*c2 - a2*c1); + double y00 = Math.min(boxLats[b], boxLats[b+1]) - ENCODING_TOLERANCE; + if (y00 > t || (x00 == s && y00 == t)) { + continue; // out of range or touching + } + double y01 = Math.max(boxLats[b], boxLats[b+1]) + ENCODING_TOLERANCE; + if (y01 < t || (x01 == s && y01 == t)) { + continue; // out of range or touching } + double y10 = Math.min(polyLats[p], polyLats[p+1]) - ENCODING_TOLERANCE; + if (y10 > t || (x10 == s && y10 == t)) { + continue; // out of range or touching + } + double y11 = Math.max(polyLats[p], polyLats[p+1]) + ENCODING_TOLERANCE; + if (y11 < t || (x11 == s && y11 == t)) { + continue; // out of range or touching + } + // if line segments are not touching and the intersection point is within the range of either segment + return true; } } // for each poly edge } // for each bbox edge @@ -255,24 +294,17 @@ public static boolean contains(Polygon[] polygons, double latitude, double longi return false; } - /** Helper for multipolygon logic: returns true if any of the supplied polygons contain the rectangle */ - public static boolean contains(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { - for (Polygon polygon : polygons) { - if (polygon.contains(minLat, maxLat, minLon, maxLon)) { - return true; - } - } - return false; - } - - /** Helper for multipolygon logic: returns true if any of the supplied polygons crosses the rectangle */ - public static boolean crosses(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { + /** Returns the multipolygon relation for the rectangle */ + public static Relation relate(Polygon[] polygons, double minLat, double maxLat, double minLon, double maxLon) { for (Polygon polygon : polygons) { - if (polygon.crosses(minLat, maxLat, minLon, maxLon)) { - return true; + Relation relation = polygon.relate(minLat, maxLat, minLon, maxLon); + if (relation != Relation.CELL_OUTSIDE_QUERY) { + // note: we optimize for non-overlapping multipolygons. so if we cross one, + // we won't keep iterating to try to find a contains. + return relation; } } - return false; + return Relation.CELL_OUTSIDE_QUERY; } @Override diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java index 02025627eeda..d2c1d67484ee 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestPolygon.java @@ -17,6 +17,7 @@ package org.apache.lucene.geo; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.util.LuceneTestCase; import static org.apache.lucene.geo.GeoTestUtil.nextLatitude; @@ -80,21 +81,15 @@ public void testMultiPolygon() { assertTrue(Polygon.contains(polygons, -25, 25)); // on the mainland assertFalse(Polygon.contains(polygons, -51, 51)); // in the ocean - // contains(box): this can conservatively return false - assertTrue(Polygon.contains(polygons, -2, 2, -2, 2)); // on the island - assertFalse(Polygon.contains(polygons, 6, 7, 6, 7)); // in the hole - assertTrue(Polygon.contains(polygons, 24, 25, 24, 25)); // on the mainland - assertFalse(Polygon.contains(polygons, 51, 52, 51, 52)); // in the ocean - assertFalse(Polygon.contains(polygons, -60, 60, -60, 60)); // enclosing us completely - assertFalse(Polygon.contains(polygons, 49, 51, 49, 51)); // overlapping the mainland - assertFalse(Polygon.contains(polygons, 9, 11, 9, 11)); // overlapping the hole - assertFalse(Polygon.contains(polygons, 5, 6, 5, 6)); // overlapping the island - - // crosses(box): this can conservatively return true - assertTrue(Polygon.crosses(polygons, -60, 60, -60, 60)); // enclosing us completely - assertTrue(Polygon.crosses(polygons, 49, 51, 49, 51)); // overlapping the mainland and ocean - assertTrue(Polygon.crosses(polygons, 9, 11, 9, 11)); // overlapping the hole and mainland - assertTrue(Polygon.crosses(polygons, 5, 6, 5, 6)); // overlapping the island + // relate(box): this can conservatively return CELL_CROSSES_QUERY + assertEquals(Relation.CELL_INSIDE_QUERY, Polygon.relate(polygons, -2, 2, -2, 2)); // on the island + assertEquals(Relation.CELL_OUTSIDE_QUERY, Polygon.relate(polygons, 6, 7, 6, 7)); // in the hole + assertEquals(Relation.CELL_INSIDE_QUERY, Polygon.relate(polygons, 24, 25, 24, 25)); // on the mainland + assertEquals(Relation.CELL_OUTSIDE_QUERY, Polygon.relate(polygons, 51, 52, 51, 52)); // in the ocean + assertEquals(Relation.CELL_CROSSES_QUERY, Polygon.relate(polygons, -60, 60, -60, 60)); // enclosing us completely + assertEquals(Relation.CELL_CROSSES_QUERY, Polygon.relate(polygons, 49, 51, 49, 51)); // overlapping the mainland + assertEquals(Relation.CELL_CROSSES_QUERY, Polygon.relate(polygons, 9, 11, 9, 11)); // overlapping the hole + assertEquals(Relation.CELL_CROSSES_QUERY, Polygon.relate(polygons, 5, 6, 5, 6)); // overlapping the island } public void testPacMan() throws Exception { @@ -110,8 +105,7 @@ public void testPacMan() throws Exception { // test cell crossing poly Polygon polygon = new Polygon(py, px); - assertTrue(polygon.crosses(yMin, yMax, xMin, xMax)); - assertFalse(polygon.contains(yMin, yMax, xMin, xMax)); + assertEquals(Relation.CELL_CROSSES_QUERY, polygon.relate(yMin, yMax, xMin, xMax)); } public void testBoundingBox() throws Exception { @@ -154,7 +148,7 @@ public void testContainsRandom() throws Exception { for (int j = 0; j < 100; j++) { Rectangle rectangle = GeoTestUtil.nextSimpleBox(); // allowed to conservatively return false - if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon)) { + if (polygon.relate(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == Relation.CELL_INSIDE_QUERY) { for (int k = 0; k < 1000; k++) { // this tests in our range but sometimes outside! so we have to double-check its really in other box double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); @@ -183,7 +177,7 @@ public void testContainsEdgeCases() throws Exception { for (int j = 0; j < 10; j++) { Rectangle rectangle = GeoTestUtil.nextSimpleBoxNear(polyLats[vertex], polyLons[vertex]); // allowed to conservatively return false - if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon)) { + if (polygon.relate(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == Relation.CELL_INSIDE_QUERY) { for (int k = 0; k < 100; k++) { // this tests in our range but sometimes outside! so we have to double-check its really in other box double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); @@ -207,8 +201,7 @@ public void testIntersectRandom() { for (int j = 0; j < 100; j++) { Rectangle rectangle = GeoTestUtil.nextSimpleBox(); // allowed to conservatively return true. - if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false && - polygon.crosses(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false) { + if (polygon.relate(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == Relation.CELL_OUTSIDE_QUERY) { for (int k = 0; k < 1000; k++) { // this tests in our range but sometimes outside! so we have to double-check its really in other box double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); @@ -237,8 +230,7 @@ public void testIntersectEdgeCases() { for (int j = 0; j < 10; j++) { Rectangle rectangle = GeoTestUtil.nextSimpleBoxNear(polyLats[vertex], polyLons[vertex]); // allowed to conservatively return true. - if (polygon.contains(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false && - polygon.crosses(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == false) { + if (polygon.relate(rectangle.minLat, rectangle.maxLat, rectangle.minLon, rectangle.maxLon) == Relation.CELL_OUTSIDE_QUERY) { for (int k = 0; k < 100; k++) { // this tests in our range but sometimes outside! so we have to double-check its really in other box double latitude = nextLatitudeAround(rectangle.minLat, rectangle.maxLat); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java new file mode 100644 index 000000000000..5d594c68723d --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.util.FixedBitSet; + +/** + * This is a temporary hack, until some polygon methods have better performance! + *

    + * When this file is removed then we have made good progress! In general we don't call + * the point-in-polygon algorithm that much, because of how BKD divides up the data. But + * today the method is very slow (general to all polygons, linear with the number of vertices). + * At the same time polygon-rectangle relation operations are also slow in the same way, this + * just really ensures they are the bottleneck by removing most of the point-in-polygon calls. + *

    + * See the "grid" algorithm description here: http://erich.realtimerendering.com/ptinpoly/ + * A few differences: + *

      + *
    • We work in an integer encoding, so edge cases are simpler. + *
    • We classify each grid cell as "contained", "not contained", or "don't know". + *
    • We form a grid over a potentially complex multipolygon with holes. + *
    • Construction is less efficient because we do not do anything "smart" such + * as following polygon edges. + *
    • Instead we construct a baby tree to reduce the number of relation operations, + * which are currently expensive. + *
    + */ +// TODO: just make a more proper tree (maybe in-ram BKD)? then we can answer most +// relational operations as rectangle <-> rectangle relations in integer space in log(n) time.. +final class LatLonGrid { + // must be a power of two! + static final int GRID_SIZE = 1<<5; + final int minLat; + final int maxLat; + final int minLon; + final int maxLon; + // TODO: something more efficient than parallel bitsets? maybe one bitset? + final FixedBitSet haveAnswer = new FixedBitSet(GRID_SIZE * GRID_SIZE); + final FixedBitSet answer = new FixedBitSet(GRID_SIZE * GRID_SIZE); + + final long latPerCell; + final long lonPerCell; + + final Polygon[] polygons; + + LatLonGrid(int minLat, int maxLat, int minLon, int maxLon, Polygon... polygons) { + this.minLat = minLat; + this.maxLat = maxLat; + this.minLon = minLon; + this.maxLon = maxLon; + this.polygons = polygons; + if (minLon > maxLon) { + // maybe make 2 grids if you want this? + throw new IllegalArgumentException("Grid cannot cross the dateline"); + } + if (minLat > maxLat) { + throw new IllegalArgumentException("bogus grid"); + } + long latitudeRange = maxLat - (long) minLat; + long longitudeRange = maxLon - (long) minLon; + // we spill over the edge of the bounding box in each direction a bit, + // but it prevents edge case bugs. + latPerCell = latitudeRange / (GRID_SIZE - 1); + lonPerCell = longitudeRange / (GRID_SIZE - 1); + fill(polygons, 0, GRID_SIZE, 0, GRID_SIZE); + } + + /** fills a 2D range of grid cells [minLatIndex .. maxLatIndex) X [minLonIndex .. maxLonIndex) */ + void fill(Polygon[] polygons, int minLatIndex, int maxLatIndex, int minLonIndex, int maxLonIndex) { + // grid cells at the edge of the bounding box are typically smaller than normal, because we spill over. + long cellMinLat = minLat + (minLatIndex * latPerCell); + long cellMaxLat = Math.min(maxLat, minLat + (maxLatIndex * latPerCell) - 1); + long cellMinLon = minLon + (minLonIndex * lonPerCell); + long cellMaxLon = Math.min(maxLon, minLon + (maxLonIndex * lonPerCell) - 1); + + assert cellMinLat <= maxLat && cellMinLon <= maxLon; + assert cellMaxLat >= cellMinLat; + assert cellMaxLon >= cellMinLon; + + Relation relation = Polygon.relate(polygons, LatLonPoint.decodeLatitude((int) cellMinLat), + LatLonPoint.decodeLatitude((int) cellMaxLat), + LatLonPoint.decodeLongitude((int) cellMinLon), + LatLonPoint.decodeLongitude((int) cellMaxLon)); + if (relation != Relation.CELL_CROSSES_QUERY) { + // we know the answer for this region, fill the cell range + for (int i = minLatIndex; i < maxLatIndex; i++) { + for (int j = minLonIndex; j < maxLonIndex; j++) { + int index = i * GRID_SIZE + j; + assert haveAnswer.get(index) == false; + haveAnswer.set(index); + if (relation == Relation.CELL_INSIDE_QUERY) { + answer.set(index); + } + } + } + } else if (minLatIndex == maxLatIndex - 1) { + // nothing more to do: this is a single grid cell (leaf node) and + // is an edge case for the polygon. + } else { + // grid range crosses our polygon, keep recursing. + int midLatIndex = (minLatIndex + maxLatIndex) >>> 1; + int midLonIndex = (minLonIndex + maxLonIndex) >>> 1; + fill(polygons, minLatIndex, midLatIndex, minLonIndex, midLonIndex); + fill(polygons, minLatIndex, midLatIndex, midLonIndex, maxLonIndex); + fill(polygons, midLatIndex, maxLatIndex, minLonIndex, midLonIndex); + fill(polygons, midLatIndex, maxLatIndex, midLonIndex, maxLonIndex); + } + } + + /** Returns true if inside one of our polygons, false otherwise */ + boolean contains(int latitude, int longitude) { + // first see if the grid knows the answer + int index = index(latitude, longitude); + if (index == -1) { + return false; // outside of bounding box range + } else if (haveAnswer.get(index)) { + return answer.get(index); + } + + // the grid is unsure (boundary): do a real test. + double docLatitude = LatLonPoint.decodeLatitude(latitude); + double docLongitude = LatLonPoint.decodeLongitude(longitude); + return Polygon.contains(polygons, docLatitude, docLongitude); + } + + /** Returns grid index of lat/lon, or -1 if the value is outside of the bounding box. */ + private int index(int latitude, int longitude) { + if (latitude < minLat || latitude > maxLat || longitude < minLon || longitude > maxLon) { + return -1; // outside of bounding box range + } + + long latRel = latitude - (long) minLat; + long lonRel = longitude - (long) minLon; + + int latIndex = (int) (latRel / latPerCell); + int lonIndex = (int) (lonRel / lonPerCell); + return latIndex * GRID_SIZE + lonIndex; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 54f5192074a8..375c83c60347 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -101,6 +101,11 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I } final float matchCost = cumulativeCost; + final LatLonGrid grid = new LatLonGrid(LatLonPoint.encodeLatitude(box.minLat), + LatLonPoint.encodeLatitude(box.maxLat), + LatLonPoint.encodeLongitude(box.minLon), + LatLonPoint.encodeLongitude(box.maxLon), polygons); + return new ConstantScoreWeight(this) { @Override @@ -128,7 +133,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } else { preApproved = new FixedBitSet(reader.maxDoc()); } - values.intersect(field, + values.intersect(field, new IntersectVisitor() { @Override public void visit(int docID) { @@ -165,13 +170,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { double cellMaxLat = LatLonPoint.decodeLatitude(maxPackedValue, 0); double cellMaxLon = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); - if (Polygon.contains(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon)) { - return Relation.CELL_INSIDE_QUERY; - } else if (Polygon.crosses(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon)) { - return Relation.CELL_CROSSES_QUERY; - } else { - return Relation.CELL_OUTSIDE_QUERY; - } + return Polygon.relate(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon); } }); @@ -195,9 +194,9 @@ public boolean matches() throws IOException { int count = docValues.count(); for (int i = 0; i < count; i++) { long encoded = docValues.valueAt(i); - double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); - double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); - if (Polygon.contains(polygons, docLatitude, docLongitude)) { + int latitudeBits = (int)(encoded >> 32); + int longitudeBits = (int)(encoded & 0xFFFFFFFF); + if (grid.contains(latitudeBits, longitudeBits)) { return true; } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java new file mode 100644 index 000000000000..2de5a433d6f1 --- /dev/null +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.geo.GeoTestUtil; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** tests against LatLonGrid (avoiding indexing/queries) */ +public class TestLatLonGrid extends LuceneTestCase { + + /** If the grid returns true, then any point in that cell should return true as well */ + public void testRandom() throws Exception { + for (int i = 0; i < 100; i++) { + Polygon polygon = GeoTestUtil.nextPolygon(); + Rectangle box = Rectangle.fromPolygon(new Polygon[] { polygon }); + int minLat = LatLonPoint.encodeLatitude(box.minLat); + int maxLat = LatLonPoint.encodeLatitude(box.maxLat); + int minLon = LatLonPoint.encodeLongitude(box.minLon); + int maxLon = LatLonPoint.encodeLongitude(box.maxLon); + LatLonGrid grid = new LatLonGrid(minLat, maxLat, minLon, maxLon, polygon); + // we are in integer space... but exhaustive testing is slow! + for (int j = 0; j < 10000; j++) { + int lat = TestUtil.nextInt(random(), minLat, maxLat); + int lon = TestUtil.nextInt(random(), minLon, maxLon); + + boolean expected = polygon.contains(LatLonPoint.decodeLatitude(lat), + LatLonPoint.decodeLongitude(lon)); + boolean actual = grid.contains(lat, lon); + assertEquals(expected, actual); + } + } + } +} diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java index 1bb43c70ba47..14b7cc7fa2b7 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQueryImpl.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.index.PointValues.Relation; /** Package private implementation for the public facing GeoPointInPolygonQuery delegate class. * @@ -58,18 +59,17 @@ private final class GeoPolygonCellComparator extends CellComparator { @Override protected boolean cellCrosses(final double minLat, final double maxLat, final double minLon, final double maxLon) { - return Polygon.crosses(polygons, minLat, maxLat, minLon, maxLon); + return Polygon.relate(polygons, minLat, maxLat, minLon, maxLon) == Relation.CELL_CROSSES_QUERY; } @Override protected boolean cellWithin(final double minLat, final double maxLat, final double minLon, final double maxLon) { - return Polygon.contains(polygons, minLat, maxLat, minLon, maxLon); + return Polygon.relate(polygons, minLat, maxLat, minLon, maxLon) == Relation.CELL_INSIDE_QUERY; } @Override protected boolean cellIntersectsShape(final double minLat, final double maxLat, final double minLon, final double maxLon) { - return cellContains(minLat, maxLat, minLon, maxLon) || cellWithin(minLat, maxLat, minLon, maxLon) - || cellCrosses(minLat, maxLat, minLon, maxLon); + return Polygon.relate(polygons, minLat, maxLat, minLon, maxLon) != Relation.CELL_OUTSIDE_QUERY; } /** From 0399eb4b671f57a7e98f32e5a263e48d2c91a746 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Fri, 1 Apr 2016 07:07:05 -0400 Subject: [PATCH 0269/1113] LUCENE-7157 Add and use clockwise/counterclockwise construction of geo polygons All code in place but isn't working Add GeoConcavePolygon, and make more efficient intersection code for GeoConvexPolygon Finish code, but it fails tests due to looping Fix some problems Fix typo Get one test to run at least Get it to work Turn off debugging Get rid of unused code Fix beasting failure Boundary edges have to use sided planes that are inverted in order to constrain the intersections right. Add test Use a more rigorous membership structure for bounding edges. Fix bounding issue Remove dependence on hash order Get everything working again Get everything working Remove System.out --- .../apache/lucene/spatial3d/Geo3DPoint.java | 8 +- .../spatial3d/geom/GeoConcavePolygon.java | 393 ++++++++ .../spatial3d/geom/GeoConvexPolygon.java | 175 +++- .../spatial3d/geom/GeoPolygonFactory.java | 913 ++++++++++++++++-- .../lucene/spatial3d/geom/SidedPlane.java | 20 +- .../lucene/spatial3d/geom/GeoPolygonTest.java | 176 +++- 6 files changed, 1530 insertions(+), 155 deletions(-) create mode 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index 45e17b72f3b3..f0f202049db4 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.PointValues; +import org.apache.lucene.spatial3d.geom.Vector; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; @@ -149,11 +150,8 @@ public static Query newPolygonQuery(final String field, final double[] polyLatit checkLongitude(longitude); polyPoints.add(new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude))); } - // We don't know what the sense of the polygon is without providing the index of one vertex we know to be convex. - // Since that doesn't fit with the "super-simple API" requirements, we just use the index of the first one, and people have to just - // know to do it that way. - final int convexPointIndex = 0; - final GeoShape shape = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, polyPoints, convexPointIndex); + // We use the polygon constructor that looks at point order. + final GeoShape shape = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, polyPoints, null); return newShapeQuery(field, shape); } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java new file mode 100644 index 000000000000..3250a80b6ec2 --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial3d.geom; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.HashMap; +import java.util.Map; + +/** + * GeoConcavePolygon objects are generic building blocks of more complex structures. + * The only restrictions on these objects are: (1) they must be concave; (2) they must have + * a maximum extent larger than PI. Violating either one of these limits will + * cause the logic to fail. + * + * @lucene.experimental + */ +public class GeoConcavePolygon extends GeoBasePolygon { + /** The list of polygon points */ + protected final List points; + /** A bitset describing, for each edge, whether it is internal or not */ + protected final BitSet isInternalEdges; + /** The list of holes. If a point is in the hole, it is *not* in the polygon */ + protected final List holes; + + /** A list of edges */ + protected SidedPlane[] edges = null; + /** A list of inverted edges */ + protected SidedPlane[] invertedEdges = null; + /** The set of notable points for each edge */ + protected GeoPoint[][] notableEdgePoints = null; + /** A point which is on the boundary of the polygon */ + protected GeoPoint[] edgePoints = null; + /** Tracking the maximum distance we go at any one time, so to be sure it's legal */ + protected double fullDistance = 0.0; + /** Set to true when the polygon is complete */ + protected boolean isDone = false; + /** A bounds object for each sided plane */ + protected Map eitherBounds = null; + + /** + * Create a concave polygon from a list of points. The first point must be on the + * external edge. + *@param planetModel is the planet model. + *@param pointList is the list of points to create the polygon from. + */ + public GeoConcavePolygon(final PlanetModel planetModel, final List pointList) { + this(planetModel, pointList, null); + } + + /** + * Create a concave polygon from a list of points. The first point must be on the + * external edge. + *@param planetModel is the planet model. + *@param pointList is the list of points to create the polygon from. + *@param holes is the list of GeoPolygon objects that describe holes in the concave polygon. Null == no holes. + */ + public GeoConcavePolygon(final PlanetModel planetModel, final List pointList, final List holes) { + super(planetModel); + this.points = pointList; + this.holes = holes; + this.isInternalEdges = new BitSet(); + done(false); + } + + /** + * Create a concave polygon from a list of points, keeping track of which boundaries + * are internal. This is used when creating a polygon as a building block for another shape. + *@param planetModel is the planet model. + *@param pointList is the set of points to create the polygon from. + *@param internalEdgeFlags is a bitset describing whether each edge is internal or not. + *@param returnEdgeInternal is true when the final return edge is an internal one. + */ + public GeoConcavePolygon(final PlanetModel planetModel, + final List pointList, + final BitSet internalEdgeFlags, + final boolean returnEdgeInternal) { + this(planetModel, pointList, null, internalEdgeFlags, returnEdgeInternal); + } + + /** + * Create a concave polygon from a list of points, keeping track of which boundaries + * are internal. This is used when creating a polygon as a building block for another shape. + *@param planetModel is the planet model. + *@param pointList is the set of points to create the polygon from. + *@param holes is the list of GeoPolygon objects that describe holes in the concave polygon. Null == no holes. + *@param internalEdgeFlags is a bitset describing whether each edge is internal or not. + *@param returnEdgeInternal is true when the final return edge is an internal one. + */ + public GeoConcavePolygon(final PlanetModel planetModel, + final List pointList, + final List holes, + final BitSet internalEdgeFlags, + final boolean returnEdgeInternal) { + super(planetModel); + this.points = pointList; + this.holes = holes; + this.isInternalEdges = internalEdgeFlags; + done(returnEdgeInternal); + } + + /** + * Create a concave polygon, with a starting latitude and longitude. + * Accepts only values in the following ranges: lat: {@code -PI/2 -> PI/2}, lon: {@code -PI -> PI} + *@param planetModel is the planet model. + *@param startLatitude is the latitude of the first point. + *@param startLongitude is the longitude of the first point. + */ + public GeoConcavePolygon(final PlanetModel planetModel, + final double startLatitude, + final double startLongitude) { + this(planetModel, startLatitude, startLongitude, null); + } + + /** + * Create a concave polygon, with a starting latitude and longitude. + * Accepts only values in the following ranges: lat: {@code -PI/2 -> PI/2}, lon: {@code -PI -> PI} + *@param planetModel is the planet model. + *@param startLatitude is the latitude of the first point. + *@param startLongitude is the longitude of the first point. + *@param holes is the list of GeoPolygon objects that describe holes in the concave polygon. Null == no holes. + */ + public GeoConcavePolygon(final PlanetModel planetModel, + final double startLatitude, + final double startLongitude, + final List holes) { + super(planetModel); + points = new ArrayList<>(); + this.holes = holes; + isInternalEdges = new BitSet(); + points.add(new GeoPoint(planetModel, startLatitude, startLongitude)); + } + + /** + * Add a point to the polygon. + * Accepts only values in the following ranges: lat: {@code -PI/2 -> PI/2}, lon: {@code -PI -> PI} + * + * @param latitude is the latitude of the next point. + * @param longitude is the longitude of the next point. + * @param isInternalEdge is true if the edge just added with this point should be considered "internal", and not + * intersected as part of the intersects() operation. + */ + public void addPoint(final double latitude, final double longitude, final boolean isInternalEdge) { + if (isDone) + throw new IllegalStateException("Can't call addPoint() if done() already called"); + if (isInternalEdge) + isInternalEdges.set(points.size() - 1); + points.add(new GeoPoint(planetModel, latitude, longitude)); + } + + /** + * Finish the polygon, by connecting the last added point with the starting point. + *@param isInternalReturnEdge is true if the return edge (back to start) is an internal one. + */ + public void done(final boolean isInternalReturnEdge) { + if (isDone) + throw new IllegalStateException("Can't call done() more than once"); + // If fewer than 3 points, can't do it. + if (points.size() < 3) + throw new IllegalArgumentException("Polygon needs at least three points."); + + if (isInternalReturnEdge) + isInternalEdges.set(points.size() - 1); + + isDone = true; + + // Time to construct the planes. If the polygon is truly concave then any adjacent point + // to a segment can provide an exterior measurement. Note: We build the true planes + // here and use the logic to return what *isn't* inside all of them. + edges = new SidedPlane[points.size()]; + invertedEdges = new SidedPlane[points.size()]; + notableEdgePoints = new GeoPoint[points.size()][]; + + for (int i = 0; i < points.size(); i++) { + final GeoPoint start = points.get(i); + final GeoPoint end = points.get(legalIndex(i + 1)); + final double distance = start.arcDistance(end); + if (distance > fullDistance) + fullDistance = distance; + final GeoPoint check = points.get(legalIndex(i + 2)); + // Here note the flip of the sense of the sided plane!! + final SidedPlane sp = new SidedPlane(check, false, start, end); + //System.out.println("Created edge "+sp+" using start="+start+" end="+end+" check="+check); + edges[i] = sp; + invertedEdges[i] = new SidedPlane(sp); + notableEdgePoints[i] = new GeoPoint[]{start, end}; + } + // In order to naively confirm that the polygon is concave, I would need to + // check every edge, and verify that every point (other than the edge endpoints) + // is within the edge's sided plane. This is an order n^2 operation. That's still + // not wrong, though, because everything else about polygons has a similar cost. + for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { + final SidedPlane edge = edges[edgeIndex]; + for (int pointIndex = 0; pointIndex < points.size(); pointIndex++) { + if (pointIndex != edgeIndex && pointIndex != legalIndex(edgeIndex + 1)) { + if (edge.isWithin(points.get(pointIndex))) + throw new IllegalArgumentException("Polygon is not concave: Point " + points.get(pointIndex) + " Edge " + edge); + } + } + } + + // For each edge, create a bounds object. + eitherBounds = new HashMap<>(edges.length); + for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { + eitherBounds.put(edges[edgeIndex], new EitherBound(invertedEdges[edgeIndex])); + } + + // Pick an edge point arbitrarily + edgePoints = new GeoPoint[]{points.get(0)}; + } + + /** Compute a legal point index from a possibly illegal one, that may have wrapped. + *@param index is the index. + *@return the normalized index. + */ + protected int legalIndex(int index) { + while (index >= points.size()) + index -= points.size(); + return index; + } + + @Override + public boolean isWithin(final double x, final double y, final double z) { + // If present within *any* plane, then it is a member, except where there are holes. + boolean isMember = false; + for (final SidedPlane edge : edges) { + if (edge.isWithin(x, y, z)) { + isMember = true; + break; + } + } + if (isMember == false) { + return false; + } + if (holes != null) { + for (final GeoPolygon polygon : holes) { + if (polygon.isWithin(x, y, z)) { + return false; + } + } + } + return true; + } + + @Override + public GeoPoint[] getEdgePoints() { + return edgePoints; + } + + @Override + public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final Membership... bounds) { + // The bounding planes are inverted and complementary. For intersection computation, we + // cannot use them as bounds. They are independent hemispheres. + for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { + final SidedPlane edge = edges[edgeIndex]; + final GeoPoint[] points = this.notableEdgePoints[edgeIndex]; + if (!isInternalEdges.get(edgeIndex)) { + if (edge.intersects(planetModel, p, notablePoints, points, bounds, eitherBounds.get(edge))) { + //System.err.println(" intersects!"); + return true; + } + } + } + if (holes != null) { + // Each hole needs to be looked at for intersection too, since a shape can be entirely within the hole + for (final GeoPolygon hole : holes) { + if (hole.intersects(p, notablePoints, bounds)) { + return true; + } + } + } + //System.err.println(" no intersection"); + return false; + } + + /** A membership implementation representing polygon edges that all must apply. + */ + protected class EitherBound implements Membership { + + protected final SidedPlane exception; + + /** Constructor. + */ + public EitherBound(final SidedPlane exception) { + this.exception = exception; + } + + @Override + public boolean isWithin(final Vector v) { + for (final SidedPlane edge : invertedEdges) { + if (edge != exception && !edge.isWithin(v)) { + return false; + } + } + return true; + } + + @Override + public boolean isWithin(final double x, final double y, final double z) { + for (final SidedPlane edge : invertedEdges) { + if (edge != exception && !edge.isWithin(x, y, z)) { + return false; + } + } + return true; + } + } + + @Override + public void getBounds(Bounds bounds) { + super.getBounds(bounds); + bounds.isWide(); + + // Add all the points + for (final GeoPoint point : points) { + bounds.addPoint(point); + } + + // Add planes with membership. + for (final SidedPlane edge : edges) { + bounds.addPlane(planetModel, edge, eitherBounds.get(edge)); + } + } + + @Override + protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { + double minimumDistance = Double.MAX_VALUE; + for (final GeoPoint edgePoint : points) { + final double newDist = distanceStyle.computeDistance(edgePoint, x,y,z); + if (newDist < minimumDistance) { + minimumDistance = newDist; + } + } + for (final SidedPlane edgePlane : edges) { + final double newDist = distanceStyle.computeDistance(planetModel, edgePlane, x, y, z, eitherBounds.get(edgePlane)); + if (newDist < minimumDistance) { + minimumDistance = newDist; + } + } + return minimumDistance; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof GeoConcavePolygon)) + return false; + GeoConcavePolygon other = (GeoConcavePolygon) o; + if (!super.equals(other)) + return false; + if (!other.isInternalEdges.equals(isInternalEdges)) + return false; + if (other.holes != null || holes != null) { + if (other.holes == null || holes == null) { + return false; + } + if (!other.holes.equals(holes)) { + return false; + } + } + return (other.points.equals(points)); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + points.hashCode(); + if (holes != null) { + result = 31 * result + holes.hashCode(); + } + return result; + } + + @Override + public String toString() { + return "GeoConcavePolygon: {planetmodel=" + planetModel + ", points=" + points + ", internalEdges=" + isInternalEdges + ((holes== null)?"":", holes=" + holes) + "}"; + } +} + diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index fb024b6efee2..749971f2f8f6 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -19,6 +19,8 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.HashMap; +import java.util.Map; /** * GeoConvexPolygon objects are generic building blocks of more complex structures. @@ -33,6 +35,8 @@ public class GeoConvexPolygon extends GeoBasePolygon { protected final List points; /** A bitset describing, for each edge, whether it is internal or not */ protected final BitSet isInternalEdges; + /** The list of holes. If a point is in the hole, it is *not* in the polygon */ + protected final List holes; /** A list of edges */ protected SidedPlane[] edges = null; @@ -44,6 +48,8 @@ public class GeoConvexPolygon extends GeoBasePolygon { protected double fullDistance = 0.0; /** Set to true when the polygon is complete */ protected boolean isDone = false; + /** A bounds object for each sided plane */ + protected Map eitherBounds = null; /** * Create a convex polygon from a list of points. The first point must be on the @@ -52,8 +58,20 @@ public class GeoConvexPolygon extends GeoBasePolygon { *@param pointList is the list of points to create the polygon from. */ public GeoConvexPolygon(final PlanetModel planetModel, final List pointList) { + this(planetModel, pointList, null); + } + + /** + * Create a convex polygon from a list of points. The first point must be on the + * external edge. + *@param planetModel is the planet model. + *@param pointList is the list of points to create the polygon from. + *@param holes is the list of GeoPolygon objects that describe holes in the complex polygon. Null == no holes. + */ + public GeoConvexPolygon(final PlanetModel planetModel, final List pointList, final List holes) { super(planetModel); this.points = pointList; + this.holes = holes; this.isInternalEdges = new BitSet(); done(false); } @@ -66,10 +84,30 @@ public GeoConvexPolygon(final PlanetModel planetModel, final List poin *@param internalEdgeFlags is a bitset describing whether each edge is internal or not. *@param returnEdgeInternal is true when the final return edge is an internal one. */ - public GeoConvexPolygon(final PlanetModel planetModel, final List pointList, final BitSet internalEdgeFlags, - final boolean returnEdgeInternal) { + public GeoConvexPolygon(final PlanetModel planetModel, + final List pointList, + final BitSet internalEdgeFlags, + final boolean returnEdgeInternal) { + this(planetModel, pointList, null, internalEdgeFlags, returnEdgeInternal); + } + + /** + * Create a convex polygon from a list of points, keeping track of which boundaries + * are internal. This is used when creating a polygon as a building block for another shape. + *@param planetModel is the planet model. + *@param pointList is the set of points to create the polygon from. + *@param holes is the list of GeoPolygon objects that describe holes in the complex polygon. Null == no holes. + *@param internalEdgeFlags is a bitset describing whether each edge is internal or not. + *@param returnEdgeInternal is true when the final return edge is an internal one. + */ + public GeoConvexPolygon(final PlanetModel planetModel, + final List pointList, + final List holes, + final BitSet internalEdgeFlags, + final boolean returnEdgeInternal) { super(planetModel); this.points = pointList; + this.holes = holes; this.isInternalEdges = internalEdgeFlags; done(returnEdgeInternal); } @@ -81,9 +119,27 @@ public GeoConvexPolygon(final PlanetModel planetModel, final List poin *@param startLatitude is the latitude of the first point. *@param startLongitude is the longitude of the first point. */ - public GeoConvexPolygon(final PlanetModel planetModel, final double startLatitude, final double startLongitude) { + public GeoConvexPolygon(final PlanetModel planetModel, + final double startLatitude, + final double startLongitude) { + this(planetModel, startLatitude, startLongitude, null); + } + + /** + * Create a convex polygon, with a starting latitude and longitude. + * Accepts only values in the following ranges: lat: {@code -PI/2 -> PI/2}, lon: {@code -PI -> PI} + *@param planetModel is the planet model. + *@param startLatitude is the latitude of the first point. + *@param startLongitude is the longitude of the first point. + *@param holes is the list of GeoPolygon objects that describe holes in the complex polygon. Null == no holes. + */ + public GeoConvexPolygon(final PlanetModel planetModel, + final double startLatitude, + final double startLongitude, + final List holes) { super(planetModel); points = new ArrayList<>(); + this.holes = holes; isInternalEdges = new BitSet(); points.add(new GeoPoint(planetModel, startLatitude, startLongitude)); } @@ -138,12 +194,6 @@ public void done(final boolean isInternalReturnEdge) { edges[i] = sp; notableEdgePoints[i] = new GeoPoint[]{start, end}; } - createCenterPoint(); - } - - /** Compute a reasonable center point. - */ - protected void createCenterPoint() { // In order to naively confirm that the polygon is convex, I would need to // check every edge, and verify that every point (other than the edge endpoints) // is within the edge's sided plane. This is an order n^2 operation. That's still @@ -157,6 +207,14 @@ protected void createCenterPoint() { } } } + + // For each edge, create a bounds object. + eitherBounds = new HashMap<>(edges.length); + for (final SidedPlane edge : edges) { + eitherBounds.put(edge, new EitherBound(edge)); + } + + // Pick an edge point arbitrarily edgePoints = new GeoPoint[]{points.get(0)}; } @@ -176,6 +234,13 @@ public boolean isWithin(final double x, final double y, final double z) { if (!edge.isWithin(x, y, z)) return false; } + if (holes != null) { + for (final GeoPolygon polygon : holes) { + if (polygon.isWithin(x, y, z)) { + return false; + } + } + } return true; } @@ -191,26 +256,58 @@ public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final M final SidedPlane edge = edges[edgeIndex]; final GeoPoint[] points = this.notableEdgePoints[edgeIndex]; if (!isInternalEdges.get(edgeIndex)) { - //System.err.println(" non-internal edge "+edge); - // Edges flagged as 'internal only' are excluded from the matching - // Construct boundaries - final Membership[] membershipBounds = new Membership[edges.length - 1]; - int count = 0; - for (int otherIndex = 0; otherIndex < edges.length; otherIndex++) { - if (otherIndex != edgeIndex) { - membershipBounds[count++] = edges[otherIndex]; - } - } - if (edge.intersects(planetModel, p, notablePoints, points, bounds, membershipBounds)) { + if (edge.intersects(planetModel, p, notablePoints, points, bounds, eitherBounds.get(edge))) { //System.err.println(" intersects!"); return true; } } } + if (holes != null) { + // Each hole needs to be looked at for intersection too, since a shape can be entirely within the hole + for (final GeoPolygon hole : holes) { + if (hole.intersects(p, notablePoints, bounds)) { + return true; + } + } + } //System.err.println(" no intersection"); return false; } + /** A membership implementation representing polygon edges that all must apply. + */ + protected class EitherBound implements Membership { + + protected final SidedPlane exception; + + /** Constructor. + */ + public EitherBound(final SidedPlane exception) { + this.exception = exception; + } + + @Override + public boolean isWithin(final Vector v) { + for (final SidedPlane edge : edges) { + if (edge != exception && !edge.isWithin(v)) { + return false; + } + } + return true; + } + + @Override + public boolean isWithin(final double x, final double y, final double z) { + for (final SidedPlane edge : edges) { + if (edge != exception && !edge.isWithin(x, y, z)) { + return false; + } + } + return true; + } + } + + @Override public void getBounds(Bounds bounds) { super.getBounds(bounds); @@ -221,17 +318,8 @@ public void getBounds(Bounds bounds) { } // Add planes with membership. - for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { - final SidedPlane edge = edges[edgeIndex]; - // Construct boundaries - final Membership[] membershipBounds = new Membership[edges.length - 1]; - int count = 0; - for (int otherIndex = 0; otherIndex < edges.length; otherIndex++) { - if (otherIndex != edgeIndex) { - membershipBounds[count++] = edges[otherIndex]; - } - } - bounds.addPlane(planetModel, edge, membershipBounds); + for (final SidedPlane edge : edges) { + bounds.addPlane(planetModel, edge, eitherBounds.get(edge)); } } @@ -244,16 +332,8 @@ protected double outsideDistance(final DistanceStyle distanceStyle, final double minimumDistance = newDist; } } - for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { - final Plane edgePlane = edges[edgeIndex]; - final Membership[] membershipBounds = new Membership[edges.length - 1]; - int count = 0; - for (int otherIndex = 0; otherIndex < edges.length; otherIndex++) { - if (otherIndex != edgeIndex) { - membershipBounds[count++] = edges[otherIndex]; - } - } - final double newDist = distanceStyle.computeDistance(planetModel, edgePlane, x, y, z, membershipBounds); + for (final SidedPlane edgePlane : edges) { + final double newDist = distanceStyle.computeDistance(planetModel, edgePlane, x, y, z, eitherBounds.get(edgePlane)); if (newDist < minimumDistance) { minimumDistance = newDist; } @@ -270,6 +350,14 @@ public boolean equals(Object o) { return false; if (!other.isInternalEdges.equals(isInternalEdges)) return false; + if (other.holes != null || holes != null) { + if (other.holes == null || holes == null) { + return false; + } + if (!other.holes.equals(holes)) { + return false; + } + } return (other.points.equals(points)); } @@ -277,12 +365,15 @@ public boolean equals(Object o) { public int hashCode() { int result = super.hashCode(); result = 31 * result + points.hashCode(); + if (holes != null) { + result = 31 * result + holes.hashCode(); + } return result; } @Override public String toString() { - return "GeoConvexPolygon: {planetmodel=" + planetModel + ", points=" + points + "}"; + return "GeoConvexPolygon: {planetmodel=" + planetModel + ", points=" + points + ", internalEdges="+isInternalEdges+((holes== null)?"":", holes=" + holes) + "}"; } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 8ee4290df3f6..d8040de64e88 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -19,6 +19,12 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.Random; +import java.util.Iterator; +import java.util.Set; +import java.util.HashSet; +import java.util.Map; +import java.util.HashMap; /** * Class which constructs a GeoMembershipShape representing an arbitrary polygon. @@ -37,138 +43,620 @@ private GeoPolygonFactory() { * its neighbors determines inside/outside for the entire polygon. * @return a GeoPolygon corresponding to what was specified. */ - public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final List pointList, final int convexPointIndex) { + public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, + final List pointList, + final int convexPointIndex) { + return makeGeoPolygon(planetModel, pointList, convexPointIndex, null); + } + + /** + * Create a GeoMembershipShape of the right kind given the specified bounds. + * + * @param pointList is a list of the GeoPoints to build an arbitrary polygon out of. + * @param convexPointIndex is the index of a single convex point whose conformation with + * its neighbors determines inside/outside for the entire polygon. + * @param holes is a list of polygons representing "holes" in the outside polygon. Null == none. + * @return a GeoPolygon corresponding to what was specified. + */ + public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, + final List pointList, + final int convexPointIndex, + final List holes) { // The basic operation uses a set of points, two points determining one particular edge, and a sided plane // describing membership. return buildPolygonShape(planetModel, pointList, convexPointIndex, getLegalIndex(convexPointIndex + 1, pointList.size()), new SidedPlane(pointList.get(getLegalIndex(convexPointIndex - 1, pointList.size())), pointList.get(convexPointIndex), pointList.get(getLegalIndex(convexPointIndex + 1, pointList.size()))), - false); + false, + holes, + null); } - /** Build a GeoMembershipShape given points, starting edge, and whether starting edge is internal or not. + /** Create a GeoPolygon using the specified points and holes, using order to determine + * siding of the polygon. Much like ESRI, this method uses clockwise to indicate the space + * on the same side of the shape as being inside, and counter-clockwise to indicate the + * space on the opposite side as being inside. + * @param pointList is a list of the GeoPoints to build an arbitrary polygon out of. If points go + * clockwise from a given pole, then that pole should be within the polygon. If points go + * counter-clockwise, then that pole should be outside the polygon. + * @return a GeoPolygon corresponding to what was specified. + */ + public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, + final List pointList) { + return makeGeoPolygon(planetModel, pointList, null); + } + + /** Create a GeoPolygon using the specified points and holes, using order to determine + * siding of the polygon. Much like ESRI, this method uses clockwise to indicate the space + * on the same side of the shape as being inside, and counter-clockwise to indicate the + * space on the opposite side as being inside. + * @param pointList is a list of the GeoPoints to build an arbitrary polygon out of. If points go + * clockwise from a given pole, then that pole should be within the polygon. If points go + * counter-clockwise, then that pole should be outside the polygon. + * @param holes is a list of polygons representing "holes" in the outside polygon. Null == none. + * @return a GeoPolygon corresponding to what was specified. + */ + public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, + final List pointList, + final List holes) { + // Create a random number generator. Effectively this furnishes us with a repeatable sequence + // of points to use for poles. + final Random generator = new Random(1234); + while (true) { + // Pick the next random pole + final double poleLat = generator.nextDouble() * Math.PI - Math.PI * 0.5; + final double poleLon = generator.nextDouble() * Math.PI * 2.0 - Math.PI; + final GeoPoint pole = new GeoPoint(planetModel, poleLat, poleLon); + // Is it inside or outside? + final Boolean isPoleInside = isInsidePolygon(pole, pointList); + if (isPoleInside != null) { + // Legal pole + return makeGeoPolygon(planetModel, pointList, holes, pole, isPoleInside); + } + // If pole choice was illegal, try another one + } + } + + /** + * Create a GeoPolygon using the specified points and holes and a test point. + * + * @param pointList is a list of the GeoPoints to build an arbitrary polygon out of. + * @param holes is a list of polygons representing "holes" in the outside polygon. Null == none. + * @param testPoint is a test point that is either known to be within the polygon area, or not. + * @param testPointInside is true if the test point is within the area, false otherwise. + * @return a GeoPolygon corresponding to what was specified. + */ + public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, + final List pointList, + final List holes, + final GeoPoint testPoint, + final boolean testPointInside) { + // We will be trying twice to find the right GeoPolygon, using alternate siding choices for the first polygon + // side. While this looks like it might be 2x as expensive as it could be, there's really no other choice I can + // find. + final SidedPlane initialPlane = new SidedPlane(testPoint, pointList.get(0), pointList.get(1)); + // We don't know if this is the correct siding choice. We will only know as we build the complex polygon. + // So we need to be prepared to try both possibilities. + final GeoPolygon trial = buildPolygonShape(planetModel, pointList, 0, 1, initialPlane, false, holes, testPoint); + if (trial == null) { + // The testPoint was within the shape. Was that intended? + if (testPointInside) { + // Yes: build it for real + return buildPolygonShape(planetModel, pointList, 0, 1, initialPlane, false, holes, null); + } + // No: do the complement and return that. + return buildPolygonShape(planetModel, pointList, 0, 1, new SidedPlane(initialPlane), false, holes, null); + } else { + // The testPoint was outside the shape. Was that intended? + if (!testPointInside) { + // Yes: return what we just built + return trial; + } + // No: return the complement + return buildPolygonShape(planetModel, pointList, 0, 1, new SidedPlane(initialPlane), false, holes, null); + } + } + + /** For a specified point and a list of poly points, determine based on point order whether the + * point should be considered in or out of the polygon. + * @param point is the point to check. + * @param polyPoints is the list of points comprising the polygon. + * @return null if the point is illegal, otherwise false if the point is inside and true if the point is outside + * of the polygon. + */ + protected static Boolean isInsidePolygon(final GeoPoint point, final List polyPoints) { + // First, compute sine and cosine of pole point latitude and longitude + final double norm = 1.0 / point.magnitude(); + final double xyDenom = Math.sqrt(point.x * point.x + point.y * point.y); + final double sinLatitude = point.z * norm; + final double cosLatitude = xyDenom * norm; + final double sinLongitude; + final double cosLongitude; + if (Math.abs(xyDenom) < Vector.MINIMUM_RESOLUTION) { + sinLongitude = 0.0; + cosLongitude = 1.0; + } else { + final double xyNorm = 1.0 / xyDenom; + sinLongitude = point.y * xyNorm; + cosLongitude = point.x * xyNorm; + } + + // Now, compute the incremental arc distance around the points of the polygon + double arcDistance = 0.0; + Double prevAngle = null; + for (final GeoPoint polyPoint : polyPoints) { + final Double angle = computeAngle(polyPoint, sinLatitude, cosLatitude, sinLongitude, cosLongitude); + if (angle == null) { + return null; + } + //System.out.println("Computed angle: "+angle); + if (prevAngle != null) { + // Figure out delta between prevAngle and current angle, and add it to arcDistance + double angleDelta = angle - prevAngle; + if (angleDelta < -Math.PI) { + angleDelta += Math.PI * 2.0; + } + if (angleDelta > Math.PI) { + angleDelta -= Math.PI * 2.0; + } + if (Math.abs(angleDelta - Math.PI) < Vector.MINIMUM_RESOLUTION) { + return null; + } + //System.out.println(" angle delta = "+angleDelta); + arcDistance += angleDelta; + } + prevAngle = angle; + } + if (prevAngle != null) { + final Double lastAngle = computeAngle(polyPoints.get(0), sinLatitude, cosLatitude, sinLongitude, cosLongitude); + if (lastAngle == null) { + return null; + } + //System.out.println("Computed last angle: "+lastAngle); + // Figure out delta and add it + double angleDelta = lastAngle - prevAngle; + if (angleDelta < -Math.PI) { + angleDelta += Math.PI * 2.0; + } + if (angleDelta > Math.PI) { + angleDelta -= Math.PI * 2.0; + } + if (Math.abs(angleDelta - Math.PI) < Vector.MINIMUM_RESOLUTION) { + return null; + } + //System.out.println(" angle delta = "+angleDelta); + arcDistance += angleDelta; + } + // Clockwise == inside == negative + //System.out.println("Arcdistance = "+arcDistance); + if (Math.abs(arcDistance) < Vector.MINIMUM_RESOLUTION) { + // No idea what direction, so try another pole. + return null; + } + return arcDistance > 0.0; + } + + protected static Double computeAngle(final GeoPoint point, + final double sinLatitude, + final double cosLatitude, + final double sinLongitude, + final double cosLongitude) { + // Coordinate rotation formula: + // x1 = x0 cos T - y0 sin T + // y1 = x0 sin T + y0 cos T + // We need to rotate the point in question into the coordinate frame specified by + // the lat and lon trig functions. + // To do this we need to do two rotations on it. First rotation is in x/y. Second rotation is in x/z. + // So: + // x1 = x0 cos az - y0 sin az + // y1 = x0 sin az + y0 cos az + // z1 = z0 + // x2 = x1 cos al - z1 sin al + // y2 = y1 + // z2 = x1 sin al + z1 cos al + + final double x1 = point.x * cosLongitude - point.y * sinLongitude; + final double y1 = point.x * sinLongitude + point.y * cosLongitude; + final double z1 = point.z; + //final double x2 = x1 * cosLatitude - z1 * sinLatitude; + final double y2 = y1; + final double z2 = x1 * sinLatitude + z1 * cosLatitude; + + // Now we should be looking down the X axis; the original point has rotated coordinates (N, 0, 0). + // So we can just compute the angle using y2 and z2. (If Math.sqrt(y2*y2 + z2 * z2) is 0.0, then the point is on the pole and we need another one). + if (Math.sqrt(y2*y2 + z2*z2) < Vector.MINIMUM_RESOLUTION) { + return null; + } + + return Math.atan2(z2, y2); + } + + /** Build a GeoPolygon out of one concave part and multiple convex parts given points, starting edge, and whether starting edge is internal or not. * @param pointsList is a list of the GeoPoints to build an arbitrary polygon out of. - * @param startPointIndex is one of the points constituting the starting edge. - * @param endPointIndex is another of the points constituting the starting edge. + * @param startPointIndex is the first of the points, constituting the starting edge. * @param startingEdge is the plane describing the starting edge. * @param isInternalEdge is true if the specified edge is an internal one. - * @return a GeoMembershipShape corresponding to what was specified. + * @param holes is the list of holes in the polygon, or null if none. + * @param testPoint is an (optional) test point, which will be used to determine if we are generating + * a shape with the proper sidedness. It is passed in only when the test point is supposed to be outside + * of the generated polygon. In this case, if the generated polygon is found to contain the point, the + * method exits early with a null return value. + * This only makes sense in the context of evaluating both possible choices and using logic to determine + * which result to use. If the test point is supposed to be within the shape, then it must be outside of the + * complement shape. If the test point is supposed to be outside the shape, then it must be outside of the + * original shape. Either way, we can figure out the right thing to use. + * @return a GeoMembershipShape corresponding to what was specified, or null if what was specified + * was inconsistent with what we generated. Specifically, if we specify an exterior point that is + * found in the interior of the shape we create here we return null, which is a signal that we chose + * our initial plane sidedness backwards. */ - public static GeoPolygon buildPolygonShape(final PlanetModel planetModel, final List pointsList, final int startPointIndex, final int endPointIndex, final SidedPlane startingEdge, final boolean isInternalEdge) { - // Algorithm as follows: - // Start with sided edge. Go through all points in some order. For each new point, determine if the point is within all edges considered so far. - // If not, put it into a list of points for recursion. If it is within, add new edge and keep going. - // Once we detect a point that is within, if there are points put aside for recursion, then call recursively. + public static GeoPolygon buildPolygonShape( + final PlanetModel planetModel, + final List pointsList, + final int startPointIndex, + final int endPointIndex, + final SidedPlane startingEdge, + final boolean isInternalEdge, + final List holes, + final GeoPoint testPoint) { + + // It could be the case that we need a concave polygon. So we need to try and look for that case + // as part of the general code for constructing complex polygons. - // Current composite. This is what we'll actually be returning. + // Note that there can be only one concave polygon. + + // The code here must keep track of two lists of sided planes. The first list contains the planes consistent with + // a concave polygon. This list will grow and shrink. The second list is built starting at the current edge that + // was last consistent with the concave polygon, and contains all edges consistent with a convex polygon. + // When that sequence of edges is done, then an internal edge is created and the identified points are converted to a + // convex polygon. That internal edge is used to extend the list of edges in the concave polygon edge list. + + // The edge buffer. + final EdgeBuffer edgeBuffer = new EdgeBuffer(pointsList, startPointIndex, endPointIndex, startingEdge, isInternalEdge); + + // Current composite. This is what we'll actually be returning. This will have a number of convex polygons, and + // maybe a single concave one too. final GeoCompositePolygon rval = new GeoCompositePolygon(); - final List recursionList = new ArrayList(); - final List currentList = new ArrayList(); - final BitSet internalEdgeList = new BitSet(); - final List currentPlanes = new ArrayList(); - - // Initialize the current list and current planes - currentList.add(pointsList.get(startPointIndex)); - currentList.add(pointsList.get(endPointIndex)); - internalEdgeList.set(currentPlanes.size(), isInternalEdge); - currentPlanes.add(startingEdge); - - // Now, scan all remaining points, in order. We'll use an index and just add to it. - for (int i = 0; i < pointsList.size() - 2; i++) { - GeoPoint newPoint = pointsList.get(getLegalIndex(i + endPointIndex + 1, pointsList.size())); - if (isWithin(newPoint, currentPlanes)) { - // Construct a sided plane based on the last two points, and the previous point - SidedPlane newBoundary = new SidedPlane(currentList.get(currentList.size() - 2), newPoint, currentList.get(currentList.size() - 1)); - // Construct a sided plane based on the return trip - SidedPlane returnBoundary = new SidedPlane(currentList.get(currentList.size() - 1), currentList.get(0), newPoint); - // Verify that none of the points beyond the new point in the list are inside the polygon we'd - // be creating if we stopped making the current polygon right now. - boolean pointInside = false; - for (int j = i + 1; j < pointsList.size() - 2; j++) { - GeoPoint checkPoint = pointsList.get(getLegalIndex(j + endPointIndex + 1, pointsList.size())); - boolean isInside = true; - if (isInside && !newBoundary.isWithin(checkPoint)) - isInside = false; - if (isInside && !returnBoundary.isWithin(checkPoint)) - isInside = false; - if (isInside) { - for (SidedPlane plane : currentPlanes) { - if (!plane.isWithin(checkPoint)) { - isInside = false; + // Starting state: + // The stopping point + Edge stoppingPoint = edgeBuffer.pickOne(); + // The current edge + Edge currentEdge = stoppingPoint; + + // Progressively look for convex sections. If we find one, we emit it and replace it. + // Keep going until we have been around once and nothing needed to change, and then + // do the concave polygon, if necessary. + while (true) { + + if (currentEdge == null) { + // We're done! + break; + } + + // Find convexity around the current edge, if any + final Boolean foundIt = findConvexPolygon(planetModel, currentEdge, rval, edgeBuffer, holes, testPoint); + if (foundIt == null) { + return null; + } + + if (foundIt) { + // New start point + stoppingPoint = edgeBuffer.pickOne(); + currentEdge = stoppingPoint; + // back around + continue; + } + + // Otherwise, go on to the next + currentEdge = edgeBuffer.getNext(currentEdge); + if (currentEdge == stoppingPoint) { + break; + } + } + + // If there's anything left in the edge buffer, convert to concave polygon. + if (makeConcavePolygon(planetModel, rval, edgeBuffer, holes, testPoint) == false) { + return null; + } + + return rval; + } + + /** Look for a concave polygon in the remainder of the edgebuffer. + * By this point, if there are any edges in the edgebuffer, they represent a concave polygon. + * @param planetModel is the planet model. + * @param rval is the composite polygon we're building. + * @param edgeBuffer is the edge buffer. + * @param holes is the optional list of holes. + * @param testPoint is the optional test point. + * @return true unless the testPoint caused failure. + */ + protected static boolean makeConcavePolygon(final PlanetModel planetModel, + final GeoCompositePolygon rval, + final EdgeBuffer edgeBuffer, + final List holes, + final GeoPoint testPoint) { + if (edgeBuffer.size() == 0) { + return true; + } + + // If there are less than three edges, something got messed up somehow. Don't know how this + // can happen but check. + if (edgeBuffer.size() < 3) { + throw new IllegalStateException("Ending edge buffer had only "+edgeBuffer.size()+" edges"); + } + + // Create the list of points + final List points = new ArrayList(edgeBuffer.size()); + final BitSet internalEdges = new BitSet(edgeBuffer.size()-1); + + Edge edge = edgeBuffer.pickOne(); + boolean isInternal = false; + for (int i = 0; i < edgeBuffer.size(); i++) { + points.add(edge.startPoint); + if (i < edgeBuffer.size() - 1) { + internalEdges.set(i, edge.isInternal); + } else { + isInternal = edge.isInternal; + } + edge = edgeBuffer.getNext(edge); + } + + if (testPoint != null && holes != null && holes.size() > 0) { + // No holes, for test + final GeoPolygon testPolygon = new GeoConcavePolygon(planetModel, points, null, internalEdges, isInternal); + if (testPolygon.isWithin(testPoint)) { + return false; + } + } + + final GeoPolygon realPolygon = new GeoConcavePolygon(planetModel, points, holes, internalEdges, isInternal); + if (testPoint != null && (holes == null || holes.size() == 0)) { + if (realPolygon.isWithin(testPoint)) { + return false; + } + } + + rval.addShape(realPolygon); + return true; + } + + /** Look for a convex polygon at the specified edge. If we find it, create one and adjust the edge buffer. + * @param planetModel is the planet model. + * @param currentEdge is the current edge to use starting the search. + * @param rval is the composite polygon to build. + * @param edgeBuffer is the edge buffer. + * @param holes is the optional list of holes. + * @param testPoint is the optional test point. + * @return null if the testPoint is within any polygon detected, otherwise true if a convex polygon was created. + */ + protected static Boolean findConvexPolygon(final PlanetModel planetModel, + final Edge currentEdge, + final GeoCompositePolygon rval, + final EdgeBuffer edgeBuffer, + final List holes, + final GeoPoint testPoint) { + + //System.out.println("Looking at edge "+currentEdge+" with startpoint "+currentEdge.startPoint+" endpoint "+currentEdge.endPoint); + + // Initialize the structure. + // We don't keep track of order here; we just care about membership. + // The only exception is the head and tail pointers. + final Set includedEdges = new HashSet<>(); + includedEdges.add(currentEdge); + Edge firstEdge = currentEdge; + Edge lastEdge = currentEdge; + + // First, walk towards the end until we need to stop + while (true) { + if (firstEdge.startPoint == lastEdge.endPoint) { + break; + } + final Edge newLastEdge = edgeBuffer.getNext(lastEdge); + if (isWithin(newLastEdge.endPoint, includedEdges)) { + //System.out.println(" maybe can extend to next edge"); + // Found a candidate for extension. But do some other checks first. Basically, we need to know if we construct a polygon + // here will overlap with other remaining points? + final SidedPlane returnBoundary; + if (firstEdge.startPoint != newLastEdge.endPoint) { + returnBoundary = new SidedPlane(firstEdge.endPoint, firstEdge.startPoint, newLastEdge.endPoint); + } else { + returnBoundary = null; + } + // The complete set of sided planes for the tentative new polygon include the ones in includedEdges, plus the one from newLastEdge, + // plus the new tentative return boundary. We have to make sure there are no points from elsewhere within the tentative convex polygon. + boolean foundPointInside = false; + final Iterator edgeIterator = edgeBuffer.iterator(); + while (edgeIterator.hasNext()) { + final Edge edge = edgeIterator.next(); + if (!includedEdges.contains(edge) && edge != newLastEdge) { + // This edge has a point to check + if (edge.startPoint != newLastEdge.endPoint) { + // look at edge.startPoint + if (isWithin(edge.startPoint, includedEdges, newLastEdge, returnBoundary)) { + //System.out.println(" nope; point within found: "+edge.startPoint); + foundPointInside = true; + break; + } + } + if (edge.endPoint != firstEdge.startPoint) { + // look at edge.endPoint + if (isWithin(edge.endPoint, includedEdges, newLastEdge, returnBoundary)) { + //System.out.println(" nope; point within found: "+edge.endPoint); + foundPointInside = true; break; } } } - if (isInside) { - pointInside = true; - break; - } } - if (!pointInside) { - // Any excluded points? - boolean isInternalBoundary = recursionList.size() > 0; - if (isInternalBoundary) { - // Handle exclusion - recursionList.add(newPoint); - recursionList.add(currentList.get(currentList.size() - 1)); - if (recursionList.size() == pointsList.size()) { - // We are trying to recurse with a list the same size as the one we started with. - // Clearly, the polygon cannot be constructed - throw new IllegalArgumentException("Polygon is illegal; cannot be decomposed into convex parts"); + + if (!foundPointInside) { + //System.out.println(" extending!"); + // Extend the polygon by the new last edge + includedEdges.add(newLastEdge); + lastEdge = newLastEdge; + // continue extending in this direction + continue; + } + } + // We can't extend any more in this direction, so break from the loop. + break; + } + + // Now, walk towards the beginning until we need to stop + while (true) { + if (firstEdge.startPoint == lastEdge.endPoint) { + break; + } + final Edge newFirstEdge = edgeBuffer.getPrevious(firstEdge); + if (isWithin(newFirstEdge.startPoint, includedEdges)) { + //System.out.println(" maybe can extend to previous edge"); + // Found a candidate for extension. But do some other checks first. Basically, we need to know if we construct a polygon + // here will overlap with other remaining points? + final SidedPlane returnBoundary; + if (newFirstEdge.startPoint != lastEdge.endPoint) { + returnBoundary = new SidedPlane(lastEdge.startPoint, lastEdge.endPoint, newFirstEdge.startPoint); + } else { + returnBoundary = null; + } + // The complete set of sided planes for the tentative new polygon include the ones in includedEdges, plus the one from newLastEdge, + // plus the new tentative return boundary. We have to make sure there are no points from elsewhere within the tentative convex polygon. + boolean foundPointInside = false; + final Iterator edgeIterator = edgeBuffer.iterator(); + while (edgeIterator.hasNext()) { + final Edge edge = edgeIterator.next(); + if (!includedEdges.contains(edge) && edge != newFirstEdge) { + // This edge has a point to check + if (edge.startPoint != lastEdge.endPoint) { + // look at edge.startPoint + if (isWithin(edge.startPoint, includedEdges, newFirstEdge, returnBoundary)) { + //System.out.println(" nope; point within found: "+edge.startPoint); + foundPointInside = true; + break; + } + } + if (edge.endPoint != newFirstEdge.startPoint) { + // look at edge.endPoint + if (isWithin(edge.endPoint, includedEdges, newFirstEdge, returnBoundary)) { + //System.out.println(" nope; point within found: "+edge.endPoint); + foundPointInside = true; + break; + } } - // We want the other side for the recursion - SidedPlane otherSideNewBoundary = new SidedPlane(newBoundary); - rval.addShape(buildPolygonShape(planetModel, recursionList, recursionList.size() - 2, recursionList.size() - 1, otherSideNewBoundary, true)); - recursionList.clear(); } - currentList.add(newPoint); - internalEdgeList.set(currentPlanes.size(), isInternalBoundary); - currentPlanes.add(newBoundary); - } else { - recursionList.add(newPoint); } - } else { - recursionList.add(newPoint); - } - } - - boolean returnEdgeInternalBoundary = recursionList.size() > 0; - if (returnEdgeInternalBoundary) { - // The last step back to the start point had a recursion, so take care of that before we complete our work - recursionList.add(currentList.get(0)); - recursionList.add(currentList.get(currentList.size() - 1)); - if (recursionList.size() == pointsList.size()) { - // We are trying to recurse with a list the same size as the one we started with. - // Clearly, the polygon cannot be constructed - throw new IllegalArgumentException("Polygon is illegal; cannot be decomposed into convex parts"); - } - // Construct a sided plane based on these two points, and the previous point - SidedPlane newBoundary = new SidedPlane(currentList.get(currentList.size() - 2), currentList.get(0), currentList.get(currentList.size() - 1)); - // We want the other side for the recursion - SidedPlane otherSideNewBoundary = new SidedPlane(newBoundary); - rval.addShape(buildPolygonShape(planetModel, recursionList, recursionList.size() - 2, recursionList.size() - 1, otherSideNewBoundary, true)); - recursionList.clear(); - } - // Now, add in the current shape. - rval.addShape(new GeoConvexPolygon(planetModel, currentList, internalEdgeList, returnEdgeInternalBoundary)); - //System.out.println("Done creating polygon"); - return rval; - } + + if (!foundPointInside) { + //System.out.println(" extending!"); + // Extend the polygon by the new last edge + includedEdges.add(newFirstEdge); + firstEdge = newFirstEdge; + // continue extending in this direction + continue; + } + } + // We can't extend any more in this direction, so break from the loop. + break; + } - /** Check if a point is within a described list of planes. - *@param newPoint is the point. - *@param currentPlanes is the list of planes. - *@return true if within. - */ - protected static boolean isWithin(final GeoPoint newPoint, final List currentPlanes) { - for (SidedPlane p : currentPlanes) { - if (!p.isWithin(newPoint)) + // Ok, figure out what we've accumulated. If it is enough for a polygon, build it. + if (includedEdges.size() < 2) { + //System.out.println("Done edge "+currentEdge+": no poly found"); + return false; + } + + // It's enough to build a convex polygon + //System.out.println("Edge "+currentEdge+": Found complex poly"); + + // Create the point list and edge list, starting with the first edge and going to the last. The return edge will be between + // the start point of the first edge and the end point of the last edge. If the first edge start point is the same as the last edge end point, + // it's a degenerate case and we want to just clean out the edge buffer entirely. + + final List points = new ArrayList(includedEdges.size()); + final BitSet internalEdges = new BitSet(includedEdges.size()-1); + final boolean returnIsInternal; + + if (firstEdge.startPoint == lastEdge.endPoint) { + // Degenerate case!! There is no return edge -- or rather, we already have it. + Edge edge = firstEdge; + points.add(edge.startPoint); + int i = 0; + while (true) { + if (edge == lastEdge) { + break; + } + points.add(edge.endPoint); + internalEdges.set(i++, edge.isInternal); + edge = edgeBuffer.getNext(edge); + } + returnIsInternal = lastEdge.isInternal; + edgeBuffer.clear(); + } else { + // Build the return edge (internal, of course) + final SidedPlane returnSidedPlane = new SidedPlane(firstEdge.endPoint, false, firstEdge.startPoint, lastEdge.endPoint); + final Edge returnEdge = new Edge(firstEdge.startPoint, lastEdge.endPoint, returnSidedPlane, true); + + // Build point list and edge list + final List edges = new ArrayList(includedEdges.size()); + returnIsInternal = true; + + Edge edge = firstEdge; + points.add(edge.startPoint); + int i = 0; + while (true) { + points.add(edge.endPoint); + internalEdges.set(i++, edge.isInternal); + edges.add(edge); + if (edge == lastEdge) { + break; + } + edge = edgeBuffer.getNext(edge); + } + + // Modify the edge buffer + edgeBuffer.replace(edges, returnEdge); + } + + // Now, construct the polygon + if (testPoint != null && holes != null && holes.size() > 0) { + // No holes, for test + final GeoPolygon testPolygon = new GeoConvexPolygon(planetModel, points, null, internalEdges, returnIsInternal); + if (testPolygon.isWithin(testPoint)) { + return null; + } + } + + final GeoPolygon realPolygon = new GeoConvexPolygon(planetModel, points, holes, internalEdges, returnIsInternal); + if (testPoint != null && (holes == null || holes.size() == 0)) { + if (realPolygon.isWithin(testPoint)) { + return null; + } + } + + rval.addShape(realPolygon); + return true; + } + + protected static boolean isWithin(final GeoPoint point, final Set edgeSet, final Edge extension, final SidedPlane returnBoundary) { + if (!extension.plane.isWithin(point)) { + return false; + } + if (returnBoundary != null && !returnBoundary.isWithin(point)) { + return false; + } + return isWithin(point, edgeSet); + } + + protected static boolean isWithin(final GeoPoint point, final Set edgeSet) { + for (final Edge edge : edgeSet) { + if (!edge.plane.isWithin(point)) { return false; + } } return true; } - + /** Convert raw point index into valid array position. *@param index is the array index. *@param size is the array size. @@ -184,4 +672,219 @@ protected static int getLegalIndex(int index, int size) { return index; } + /** Class representing a single (unused) edge. + */ + protected static class Edge { + public final SidedPlane plane; + public final GeoPoint startPoint; + public final GeoPoint endPoint; + public final boolean isInternal; + + public Edge(final GeoPoint startPoint, final GeoPoint endPoint, final SidedPlane plane, final boolean isInternal) { + this.startPoint = startPoint; + this.endPoint = endPoint; + this.plane = plane; + this.isInternal = isInternal; + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + @Override + public boolean equals(final Object o) { + return o == this; + } + } + + /** Class representing an iterator over an EdgeBuffer. + */ + protected static class EdgeBufferIterator implements Iterator { + protected final EdgeBuffer edgeBuffer; + protected final Edge firstEdge; + protected Edge currentEdge; + + public EdgeBufferIterator(final EdgeBuffer edgeBuffer) { + this.edgeBuffer = edgeBuffer; + this.currentEdge = edgeBuffer.pickOne(); + this.firstEdge = currentEdge; + } + + @Override + public boolean hasNext() { + return currentEdge != null; + } + + @Override + public Edge next() { + final Edge rval = currentEdge; + if (currentEdge != null) { + currentEdge = edgeBuffer.getNext(currentEdge); + if (currentEdge == firstEdge) { + currentEdge = null; + } + } + return rval; + } + + @Override + public void remove() { + throw new RuntimeException("Unsupported operation"); + } + } + + /** Class representing a pool of unused edges, all linked together by vertices. + */ + protected static class EdgeBuffer { + protected Edge oneEdge; + protected final Set edges = new HashSet<>(); + protected final Map previousEdges = new HashMap<>(); + protected final Map nextEdges = new HashMap<>(); + + public EdgeBuffer(final List pointList, final int startPlaneStartIndex, final int startPlaneEndIndex, final SidedPlane startPlane, final boolean startPlaneIsInternal) { + /* + System.out.println("Initial points:"); + for (final GeoPoint p : pointList) { + System.out.println(" "+p); + } + System.out.println("For start plane, the following points are in/out:"); + for (final GeoPoint p: pointList) { + System.out.println(" "+p+" is: "+(startPlane.isWithin(p)?"in":"out")); + } + */ + + final Edge startEdge = new Edge(pointList.get(startPlaneStartIndex), pointList.get(startPlaneEndIndex), startPlane, startPlaneIsInternal); + // Fill in the EdgeBuffer by walking around creating more stuff + Edge currentEdge = startEdge; + int startIndex = startPlaneStartIndex; + int endIndex = startPlaneEndIndex; + boolean isInternal = startPlaneIsInternal; + while (true) { + // Compute the next edge + startIndex = endIndex; + endIndex++; + if (endIndex >= pointList.size()) { + endIndex -= pointList.size(); + } + // Get the next point + final GeoPoint newPoint = pointList.get(endIndex); + // Build the new edge + final boolean isNewPointWithin = currentEdge.plane.isWithin(newPoint); + final SidedPlane newPlane = new SidedPlane(currentEdge.startPoint, isNewPointWithin, pointList.get(startIndex), newPoint); + /* + System.out.println("For next plane, the following points are in/out:"); + for (final GeoPoint p: pointList) { + System.out.println(" "+p+" is: "+(newPlane.isWithin(p)?"in":"out")); + } + */ + final Edge newEdge = new Edge(pointList.get(startIndex), pointList.get(endIndex), newPlane, false); + + // Link it in + previousEdges.put(newEdge, currentEdge); + nextEdges.put(currentEdge, newEdge); + edges.add(newEdge); + currentEdge = newEdge; + + if (currentEdge.endPoint == startEdge.startPoint) { + // We finish here. Link the current edge to the start edge, and exit + previousEdges.put(startEdge, currentEdge); + nextEdges.put(currentEdge, startEdge); + edges.add(startEdge); + break; + } + } + + oneEdge = startEdge; + + // Verify the structure. + //verify(); + } + + /* + protected void verify() { + if (edges.size() != previousEdges.size() || edges.size() != nextEdges.size()) { + throw new IllegalStateException("broken structure"); + } + // Confirm each edge + for (final Edge e : edges) { + final Edge previousEdge = getPrevious(e); + final Edge nextEdge = getNext(e); + if (e.endPoint != nextEdge.startPoint) { + throw new IllegalStateException("broken structure"); + } + if (e.startPoint != previousEdge.endPoint) { + throw new IllegalStateException("broken structure"); + } + if (getNext(previousEdge) != e) { + throw new IllegalStateException("broken structure"); + } + if (getPrevious(nextEdge) != e) { + throw new IllegalStateException("broken structure"); + } + } + if (oneEdge != null && !edges.contains(oneEdge)) { + throw new IllegalStateException("broken structure"); + } + if (oneEdge == null && edges.size() > 0) { + throw new IllegalStateException("broken structure"); + } + } + */ + + public Edge getPrevious(final Edge currentEdge) { + return previousEdges.get(currentEdge); + } + + public Edge getNext(final Edge currentEdge) { + return nextEdges.get(currentEdge); + } + + public void replace(final List removeList, final Edge newEdge) { + /* + System.out.println("Replacing: "); + for (final Edge e : removeList) { + System.out.println(" "+e.startPoint+"-->"+e.endPoint); + } + System.out.println("...with: "+newEdge.startPoint+"-->"+newEdge.endPoint); + */ + final Edge previous = previousEdges.get(removeList.get(0)); + final Edge next = nextEdges.get(removeList.get(removeList.size()-1)); + edges.add(newEdge); + previousEdges.put(newEdge, previous); + nextEdges.put(previous, newEdge); + previousEdges.put(next, newEdge); + nextEdges.put(newEdge, next); + for (final Edge edge : removeList) { + if (edge == oneEdge) { + oneEdge = newEdge; + } + edges.remove(edge); + previousEdges.remove(edge); + nextEdges.remove(edge); + } + //verify(); + } + + public void clear() { + edges.clear(); + previousEdges.clear(); + nextEdges.clear(); + oneEdge = null; + } + + public int size() { + return edges.size(); + } + + public Iterator iterator() { + return new EdgeBufferIterator(this); + } + + public Edge pickOne() { + return oneEdge; + } + + } + } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java index e080bc04f40e..0c19a9eadf4a 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/SidedPlane.java @@ -31,7 +31,7 @@ public class SidedPlane extends Plane implements Membership { * * @param sidedPlane is the existing plane. */ - public SidedPlane(SidedPlane sidedPlane) { + public SidedPlane(final SidedPlane sidedPlane) { super(sidedPlane, sidedPlane.D); this.sigNum = -sidedPlane.sigNum; } @@ -44,13 +44,29 @@ public SidedPlane(SidedPlane sidedPlane) { * @param A is the first in-plane point * @param B is the second in-plane point */ - public SidedPlane(Vector p, Vector A, Vector B) { + public SidedPlane(final Vector p, final Vector A, final Vector B) { super(A, B); sigNum = Math.signum(evaluate(p)); if (sigNum == 0.0) throw new IllegalArgumentException("Cannot determine sidedness because check point is on plane."); } + /** + * Construct a sided plane from a pair of vectors describing points, and including + * origin, plus a point p which describes the side. + * + * @param p point to evaluate + * @param onSide is true if the point is on the correct side of the plane, false otherwise. + * @param A is the first in-plane point + * @param B is the second in-plane point + */ + public SidedPlane(final Vector p, final boolean onSide, final Vector A, final Vector B) { + super(A, B); + sigNum = onSide?Math.signum(evaluate(p)):-Math.signum(evaluate(p)); + if (sigNum == 0.0) + throw new IllegalArgumentException("Cannot determine sidedness because check point is on plane."); + } + /** * Construct a sided plane from a point and a Z coordinate. * diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java index d9b220de9252..9f2bea41b484 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java @@ -18,8 +18,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.BitSet; import org.junit.Test; +import org.junit.Ignore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -27,10 +29,45 @@ public class GeoPolygonTest { + @Test + public void testPolygonClockwise() { + GeoPolygon c; + GeoPoint gp; + List points; + + // Points go counterclockwise, so + points = new ArrayList(); + points.add(new GeoPoint(PlanetModel.SPHERE, -0.1, -0.5)); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.0, -0.6)); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.1, -0.5)); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.0, -0.4)); + + c = GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points); + //System.out.println(c); + + // Middle point should NOT be within!! + gp = new GeoPoint(PlanetModel.SPHERE, 0.0, -0.5); + assertTrue(!c.isWithin(gp)); + + // Now, go clockwise + points = new ArrayList(); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.0, -0.4)); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.1, -0.5)); + points.add(new GeoPoint(PlanetModel.SPHERE, 0.0, -0.6)); + points.add(new GeoPoint(PlanetModel.SPHERE, -0.1, -0.5)); + + c = GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points); + //System.out.println(c); + + // Middle point should be within!! + gp = new GeoPoint(PlanetModel.SPHERE, 0.0, -0.5); + assertTrue(c.isWithin(gp)); + + } @Test public void testPolygonPointWithin() { - GeoMembershipShape c; + GeoPolygon c; GeoPoint gp; List points; @@ -162,4 +199,141 @@ public void testPolygonBounds() { assertEquals(0.1, b.getMaxLatitude(), 0.000001); } + @Test + public void testPolygonBoundsCase1() { + GeoPolygon c; + LatLonBounds b; + List points; + XYZBounds xyzb; + GeoPoint point1; + GeoPoint point2; + GeoArea area; + + // Build the polygon + points = new ArrayList<>(); + points.add(new GeoPoint(PlanetModel.WGS84, 0.7769776943105245, -2.157536559188766)); + points.add(new GeoPoint(PlanetModel.WGS84, -0.9796549195552824, -0.25078026625235256)); + points.add(new GeoPoint(PlanetModel.WGS84, 0.17644522781457245, 2.4225312555674967)); + points.add(new GeoPoint(PlanetModel.WGS84, -1.4459804612164617, -1.2970934639728127)); + c = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, points, 3); + // GeoCompositeMembershipShape: {[GeoConvexPolygon: {planetmodel=PlanetModel.WGS84, points= + // [[lat=0.17644522781457245, lon=2.4225312555674967], + // [lat=-1.4459804612164617, lon=-1.2970934639728127], + // [lat=0.7769776943105245, lon=-2.157536559188766]]}, + // GeoConcavePolygon: {planetmodel=PlanetModel.WGS84, points= + // [[lat=-0.9796549195552824, lon=-0.25078026625235256], + // [lat=0.17644522781457245, lon=2.4225312555674967], + // [lat=0.7769776943105245, lon=-2.157536559188766]]}]} + point1 = new GeoPoint(PlanetModel.WGS84, -1.2013743680763862, 0.48458963747230094); + point2 = new GeoPoint(0.3189285805649921, 0.16790264636909197, -0.9308557496413026); + + assertTrue(c.isWithin(point1)); + assertTrue(c.isWithin(point2)); + + // Now try bounds + xyzb = new XYZBounds(); + c.getBounds(xyzb); + area = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, + xyzb.getMinimumX(), xyzb.getMaximumX(), xyzb.getMinimumY(), xyzb.getMaximumY(), xyzb.getMinimumZ(), xyzb.getMaximumZ()); + + assertTrue(area.isWithin(point1)); + assertTrue(area.isWithin(point2)); + } + + @Test + public void testGeoPolygonBoundsCase2() { + // [junit4] 1> TEST: iter=23 shape=GeoCompositeMembershipShape: {[GeoConvexPolygon: {planetmodel=PlanetModel(ab=0.7563871189161702 c=1.2436128810838298), points= + // [[lat=0.014071770744627236, lon=0.011030818292803128], + // [lat=0.006772117088906782, lon=-0.0012531892445234592], + // [lat=0.0022201615609504792, lon=0.005941293187389326]]}, GeoConcavePolygon: {planetmodel=PlanetModel(ab=0.7563871189161702 c=1.2436128810838298), points= + // [[lat=-0.005507100238396111, lon=-0.008487706131259667], + // [lat=0.014071770744627236, lon=0.011030818292803128], + // [lat=0.0022201615609504792, lon=0.005941293187389326]]}]} + + PlanetModel pm = new PlanetModel(0.7563871189161702, 1.2436128810838298); + // Build the polygon + GeoCompositeMembershipShape c = new GeoCompositeMembershipShape(); + List points1 = new ArrayList<>(); + points1.add(new GeoPoint(pm, 0.014071770744627236, 0.011030818292803128)); + points1.add(new GeoPoint(pm, 0.006772117088906782, -0.0012531892445234592)); + points1.add(new GeoPoint(pm, 0.0022201615609504792, 0.005941293187389326)); + BitSet p1bits = new BitSet(); + c.addShape(new GeoConvexPolygon(pm, points1, p1bits, true)); + List points2 = new ArrayList<>(); + points2.add(new GeoPoint(pm, -0.005507100238396111, -0.008487706131259667)); + points2.add(new GeoPoint(pm, 0.014071770744627236, 0.011030818292803128)); + points2.add(new GeoPoint(pm, 0.0022201615609504792, 0.005941293187389326)); + BitSet p2bits = new BitSet(); + p2bits.set(1, true); + c.addShape(new GeoConcavePolygon(pm, points2, p2bits, false)); + //System.out.println(c); + + // [junit4] 1> point=[lat=0.003540694517552105, lon=-9.99517927901697E-4] + // [junit4] 1> quantized=[X=0.7563849869428783, Y=-7.560204674780763E-4, Z=0.0026781405884151086] + GeoPoint point = new GeoPoint(pm, 0.003540694517552105, -9.99517927901697E-4); + GeoPoint pointQuantized = new GeoPoint(0.7563849869428783, -7.560204674780763E-4, 0.0026781405884151086); + + // Now try bounds + XYZBounds xyzb = new XYZBounds(); + c.getBounds(xyzb); + GeoArea area = GeoAreaFactory.makeGeoArea(pm, + xyzb.getMinimumX(), xyzb.getMaximumX(), xyzb.getMinimumY(), xyzb.getMaximumY(), xyzb.getMinimumZ(), xyzb.getMaximumZ()); + + assertTrue(c.isWithin(point)); + assertTrue(c.isWithin(pointQuantized)); + // This fails!! + assertTrue(area.isWithin(point)); + assertTrue(area.isWithin(pointQuantized)); + } + + @Test + public void testGeoConcaveRelationshipCase1() { + /* + [junit4] 1> doc=906 matched but should not + [junit4] 1> point=[lat=-0.9825762558001477, lon=2.4832136904725273] + [junit4] 1> quantized=[X=-0.4505446160475436, Y=0.34850109186970535, Z=-0.8539966368663765] + +doc=906 added here: + + [junit4] 1> cycle: cell=107836 parentCellID=107835 x: -1147288468 TO -742350917, y: -1609508490 TO 1609508490, z: -2147483647 TO 2147483647, splits: 3 queue.size()=1 + [junit4] 1> minx=-0.6107484000858642 maxx=-0.39518364125756916 miny=-0.8568069517709872 maxy=0.8568069517709872 minz=-1.1431930485939341 maxz=1.1431930485939341 + [junit4] 1> GeoArea.CONTAINS: now addAll + +shape: + [junit4] 1> TEST: iter=18 shape=GeoCompositeMembershipShape: {[GeoConvexPolygon: { + planetmodel=PlanetModel(ab=0.8568069516722363 c=1.1431930483277637), points= + [[lat=1.1577814487635816, lon=1.6283601832010004], + [lat=0.6664570999069251, lon=2.0855825542851574], + [lat=-0.23953537010974632, lon=1.8498724094352876]]}, GeoConcavePolygon: {planetmodel=PlanetModel(ab=0.8568069516722363 c=1.1431930483277637), points= + [[lat=1.1577814487635816, lon=1.6283601832010004], + [lat=-0.23953537010974632, lon=1.8498724094352876], + [lat=-1.1766904875978805, lon=-2.1346828411344436]]}]} + */ + PlanetModel pm = new PlanetModel(0.8568069516722363, 1.1431930483277637); + // Build the polygon + GeoCompositeMembershipShape c = new GeoCompositeMembershipShape(); + List points1 = new ArrayList<>(); + points1.add(new GeoPoint(pm, 1.1577814487635816, 1.6283601832010004)); + points1.add(new GeoPoint(pm, 0.6664570999069251, 2.0855825542851574)); + points1.add(new GeoPoint(pm, -0.23953537010974632, 1.8498724094352876)); + BitSet p1bits = new BitSet(); + c.addShape(new GeoConvexPolygon(pm, points1, p1bits, true)); + List points2 = new ArrayList<>(); + points2.add(new GeoPoint(pm, 1.1577814487635816, 1.6283601832010004)); + points2.add(new GeoPoint(pm, -0.23953537010974632, 1.8498724094352876)); + points2.add(new GeoPoint(pm, -1.1766904875978805, -2.1346828411344436)); + BitSet p2bits = new BitSet(); + p2bits.set(1, true); + c.addShape(new GeoConcavePolygon(pm, points2, p2bits, false)); + //System.out.println(c); + + GeoPoint point = new GeoPoint(pm, -0.9825762558001477, 2.4832136904725273); + GeoPoint quantizedPoint = new GeoPoint(-0.4505446160475436, 0.34850109186970535, -0.8539966368663765); + + GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(pm, + -0.6107484000858642, -0.39518364125756916, -0.8568069517709872, 0.8568069517709872, -1.1431930485939341, 1.1431930485939341); + //System.out.println("relationship = "+xyzSolid.getRelationship(c)); + assertTrue(xyzSolid.getRelationship(c) == GeoArea.OVERLAPS); + } + } From c698928b0675b1a7060abc0411fe22bbf51fc5dc Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Sun, 3 Apr 2016 09:43:11 -0400 Subject: [PATCH 0270/1113] LUCENE-7167: Make solid variants package private More public->package private --- .../java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java | 2 +- .../java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java | 2 +- .../java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java | 2 +- .../java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java | 4 ++-- .../org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java | 4 ++-- .../org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java | 4 ++-- .../lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java | 2 +- .../lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java | 2 +- .../lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java | 2 +- .../apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java | 2 +- .../java/org/apache/lucene/spatial3d/geom/GeoRectangle.java | 2 +- .../apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoStandardCircle.java | 2 +- .../spatial3d/geom/GeoWideDegenerateHorizontalLine.java | 2 +- .../apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java | 2 +- .../apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java | 2 +- .../org/apache/lucene/spatial3d/geom/GeoWideRectangle.java | 2 +- .../apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java | 2 +- .../org/apache/lucene/spatial3d/geom/StandardXYZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java | 2 +- .../src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java | 2 +- .../java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java | 2 +- 32 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java index 16b52ccee75d..25aa130b5d9b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/BaseXYZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public abstract class BaseXYZSolid extends BasePlanetObject implements XYZSolid { +abstract class BaseXYZSolid extends BasePlanetObject implements XYZSolid { /** Unit vector in x */ protected static final Vector xUnitVector = new Vector(1.0, 0.0, 0.0); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java index 7190cdccff1b..ba92cc2242eb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseBBox.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public abstract class GeoBaseBBox extends GeoBaseMembershipShape implements GeoBBox { +abstract class GeoBaseBBox extends GeoBaseMembershipShape implements GeoBBox { /** Construct, given planet model. *@param planetModel is the planet model. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java index 75219fd3d32a..b34313a495be 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java @@ -21,7 +21,7 @@ * * @lucene.experimental */ -public abstract class GeoBaseCircle extends GeoBaseDistanceShape implements GeoCircle { +abstract class GeoBaseCircle extends GeoBaseDistanceShape implements GeoCircle { /** Constructor. *@param planetModel is the planet model to use. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java index ba221ae88530..08c0634cefef 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePolygon.java @@ -19,9 +19,9 @@ /** * GeoBasePolygon objects are the base class of most GeoPolygon objects. * - * @lucene.experimental + * @lucene.internal */ -public abstract class GeoBasePolygon extends GeoBaseMembershipShape implements GeoPolygon { +abstract class GeoBasePolygon extends GeoBaseMembershipShape implements GeoPolygon { /** Constructor. *@param planetModel is the planet model to use. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 3250a80b6ec2..7069e380fba9 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -28,9 +28,9 @@ * a maximum extent larger than PI. Violating either one of these limits will * cause the logic to fail. * - * @lucene.experimental + * @lucene.internal */ -public class GeoConcavePolygon extends GeoBasePolygon { +class GeoConcavePolygon extends GeoBasePolygon { /** The list of polygon points */ protected final List points; /** A bitset describing, for each edge, whether it is internal or not */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index 749971f2f8f6..502af3f372d9 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -28,9 +28,9 @@ * a maximum extent no larger than PI. Violating either one of these limits will * cause the logic to fail. * - * @lucene.experimental + * @lucene.internal */ -public class GeoConvexPolygon extends GeoBasePolygon { +class GeoConvexPolygon extends GeoBasePolygon { /** The list of polygon points */ protected final List points; /** A bitset describing, for each edge, whether it is internal or not */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java index b7de0c2106ae..927fbdc0f2e0 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateHorizontalLine.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class GeoDegenerateHorizontalLine extends GeoBaseBBox { +class GeoDegenerateHorizontalLine extends GeoBaseBBox { /** Latitude of horizontal line */ protected final double latitude; /** Left bounding longitude of line */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java index e79412312b50..ea6bb5734533 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLatitudeZone.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoDegenerateLatitudeZone extends GeoBaseBBox { +class GeoDegenerateLatitudeZone extends GeoBaseBBox { /** The latitude */ protected final double latitude; /** Sine of the latitude */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java index 0bb7b90dfc2f..09f468cc27e3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegenerateLongitudeSlice.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoDegenerateLongitudeSlice extends GeoBaseBBox { +class GeoDegenerateLongitudeSlice extends GeoBaseBBox { /** The longitude of the slice */ protected final double longitude; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java index fcd20371004b..aa4732d4eece 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoDegeneratePoint extends GeoPoint implements GeoBBox, GeoCircle { +class GeoDegeneratePoint extends GeoPoint implements GeoBBox, GeoCircle { /** Current planet model, since we don't extend BasePlanetObject */ protected final PlanetModel planetModel; /** Edge point is an area containing just this */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java index 912ca3205f84..14618f53f9cc 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLatitudeZone.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoLatitudeZone extends GeoBaseBBox { +class GeoLatitudeZone extends GeoBaseBBox { /** The top latitude of the zone */ protected final double topLat; /** The bottom latitude of the zone */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java index 458cf8bef5bb..263ff47fd44d 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoLongitudeSlice.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class GeoLongitudeSlice extends GeoBaseBBox { +class GeoLongitudeSlice extends GeoBaseBBox { /** The left longitude of the slice */ protected final double leftLon; /** The right longitude of the slice */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java index 2c940618f3ff..2acfc4ed8e5c 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthLatitudeZone.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoNorthLatitudeZone extends GeoBaseBBox { +class GeoNorthLatitudeZone extends GeoBaseBBox { /** The bottom latitude of the zone */ protected final double bottomLat; /** Cosine of the bottom latitude of the zone */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java index a2b6f1b9c8cb..6015e5bc91bb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoNorthRectangle.java @@ -24,7 +24,7 @@ * * @lucene.internal */ -public class GeoNorthRectangle extends GeoBaseBBox { +class GeoNorthRectangle extends GeoBaseBBox { /** The bottom latitude of the rectangle */ protected final double bottomLat; /** The left longitude */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java index 1420c1116d20..4be5d84fa865 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoRectangle.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class GeoRectangle extends GeoBaseBBox { +class GeoRectangle extends GeoBaseBBox { /** The top latitude of the rect */ protected final double topLat; /** The bottom latitude of the rect */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java index a1d896740a72..c915c24f7582 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthLatitudeZone.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoSouthLatitudeZone extends GeoBaseBBox { +class GeoSouthLatitudeZone extends GeoBaseBBox { /** The top latitude of the zone */ protected final double topLat; /** The cosine of the top latitude of the zone */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java index 806535e72783..2eb071cc166d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoSouthRectangle.java @@ -24,7 +24,7 @@ * * @lucene.internal */ -public class GeoSouthRectangle extends GeoBaseBBox { +class GeoSouthRectangle extends GeoBaseBBox { /** The top latitude of the rect */ protected final double topLat; /** The left longitude of the rect */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java index bbf50466b8a6..1bc25bef8a68 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardCircle.java @@ -21,7 +21,7 @@ * * @lucene.experimental */ -public class GeoStandardCircle extends GeoBaseCircle { +class GeoStandardCircle extends GeoBaseCircle { /** Center of circle */ protected final GeoPoint center; /** Cutoff angle of circle (not quite the same thing as radius) */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java index 48a73afec72c..fe934731c2ae 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideDegenerateHorizontalLine.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoWideDegenerateHorizontalLine extends GeoBaseBBox { +class GeoWideDegenerateHorizontalLine extends GeoBaseBBox { /** The latitude of the line */ protected final double latitude; /** The left longitude cutoff of the line */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java index 1d6187649f3c..2ef3c8b5c940 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoWideLongitudeSlice extends GeoBaseBBox { +class GeoWideLongitudeSlice extends GeoBaseBBox { /** The left longitude */ protected final double leftLon; /** The right longitude */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java index 9f9dd49bab86..1a5603fc69db 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideNorthRectangle.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoWideNorthRectangle extends GeoBaseBBox { +class GeoWideNorthRectangle extends GeoBaseBBox { /** Bottom latitude */ protected final double bottomLat; /** Left longitude */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java index c561747c57d1..031dcaa32ee7 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideRectangle.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoWideRectangle extends GeoBaseBBox { +class GeoWideRectangle extends GeoBaseBBox { /** The top latitude */ protected final double topLat; /** The bottom latitude */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java index da9799ab0c34..3f1d2329b1d2 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideSouthRectangle.java @@ -22,7 +22,7 @@ * * @lucene.internal */ -public class GeoWideSouthRectangle extends GeoBaseBBox { +class GeoWideSouthRectangle extends GeoBaseBBox { /** Top latitude of rect */ protected final double topLat; /** Left longitude of rect */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java index 25bdc9641659..17352b940ad4 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWorld.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class GeoWorld extends GeoBaseBBox { +class GeoWorld extends GeoBaseBBox { /** No points on the edge of the shape */ protected final static GeoPoint[] edgePoints = new GeoPoint[0]; /** Point in the middle of the world */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java index 492f7b46a50c..ec2e26ce870c 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class StandardXYZSolid extends BaseXYZSolid { +class StandardXYZSolid extends BaseXYZSolid { /** Whole world? */ protected final boolean isWholeWorld; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java index 66aac84d9161..aa4b87349771 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYdZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class XYdZSolid extends BaseXYZSolid { +class XYdZSolid extends BaseXYZSolid { /** Min-X plane */ protected final SidedPlane minXPlane; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java index d9e11b89fe6b..37f2a46631df 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class XdYZSolid extends BaseXYZSolid { +class XdYZSolid extends BaseXYZSolid { /** Min-X plane */ protected final SidedPlane minXPlane; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java index 33d0beafc656..936c184b9eb8 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XdYdZSolid.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class XdYdZSolid extends BaseXYZSolid { +class XdYdZSolid extends BaseXYZSolid { /** The points in this figure on the planet surface; also doubles for edge points */ protected final GeoPoint[] surfacePoints; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java index 48fe714c28ad..029f2154d9e3 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class dXYZSolid extends BaseXYZSolid { +class dXYZSolid extends BaseXYZSolid { /** X plane */ protected final Plane xPlane; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java index d824f2629750..a1b5e96da2a7 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXYdZSolid.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class dXYdZSolid extends BaseXYZSolid { +class dXYdZSolid extends BaseXYZSolid { /** The points in this figure on the planet surface; also doubles for edge points */ protected final GeoPoint[] surfacePoints; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java index b9942b5e6c9b..869ddeac989b 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYZSolid.java @@ -23,7 +23,7 @@ * * @lucene.internal */ -public class dXdYZSolid extends BaseXYZSolid { +class dXdYZSolid extends BaseXYZSolid { /** The points in this figure on the planet surface; also doubles for edge points */ protected final GeoPoint[] surfacePoints; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java index 66dcab8536e6..b647658072b1 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/dXdYdZSolid.java @@ -21,7 +21,7 @@ * * @lucene.internal */ -public class dXdYdZSolid extends BaseXYZSolid { +class dXdYdZSolid extends BaseXYZSolid { /** On surface? */ protected final boolean isOnSurface; From 882d1d440972d8aeb68d9bc302e232ed38fa934a Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 12:13:37 -0400 Subject: [PATCH 0271/1113] LUCENE-7157: Update API javadoc and Polygon method visibilities --- .../core/src/java/org/apache/lucene/geo/Polygon.java | 10 ++++++++++ .../java/org/apache/lucene/document/LatLonPoint.java | 5 ++--- .../lucene/document/LatLonPointInPolygonQuery.java | 3 +-- .../geopoint/search/GeoPointInPolygonQuery.java | 10 ++-------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/geo/Polygon.java b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java index b643ca2bb902..145b939e4e2b 100644 --- a/lucene/core/src/java/org/apache/lucene/geo/Polygon.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Polygon.java @@ -22,6 +22,16 @@ /** * Represents a closed polygon on the earth's surface. + *

    + * NOTES: + *

      + *
    1. Coordinates must be in clockwise order, except for holes. Holes must be in counter-clockwise order. + *
    2. The polygon must be closed: the first and last coordinates need to have the same values. + *
    3. The polygon must not be self-crossing, otherwise may result in unexpected behavior. + *
    4. All latitude/longitude values must be in decimal degrees. + *
    5. Polygons cannot cross the 180th meridian. Instead, use two polygons: one on each side. + *
    6. For more advanced GeoSpatial indexing and query operations see the {@code spatial-extras} module + *
    * @lucene.experimental */ public final class Polygon { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 26bb04ad5b64..0d8a7b2b774c 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -339,13 +339,12 @@ public static Query newDistanceQuery(String field, double latitude, double longi } /** - * Create a query for matching a polygon. - *

    - * The supplied {@code polygon} must be clockwise or counter-clockwise. + * Create a query for matching one or more polygons. * @param field field name. must not be null. * @param polygons array of polygons. must not be null or empty * @return query matching points within this polygon * @throws IllegalArgumentException if {@code field} is null, {@code polygons} is null or empty + * @see Polygon */ public static Query newPolygonQuery(String field, Polygon... polygons) { return new LatLonPointInPolygonQuery(field, polygons); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 375c83c60347..670db3de67a3 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -55,8 +55,7 @@ final class LatLonPointInPolygonQuery extends Query { final String field; final Polygon[] polygons; - /** The lats/lons must be clockwise or counter-clockwise. */ - public LatLonPointInPolygonQuery(String field, Polygon[] polygons) { + LatLonPointInPolygonQuery(String field, Polygon[] polygons) { if (field == null) { throw new IllegalArgumentException("field must not be null"); } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java index 17ce54d437da..a06bdc07149a 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/geopoint/search/GeoPointInPolygonQuery.java @@ -36,13 +36,7 @@ * term is passed to the final point in polygon check. All value comparisons are subject * to the same precision tolerance defined in {@value GeoEncodingUtils#TOLERANCE} * - *

    NOTES: - * 1. The polygon coordinates need to be in either clockwise or counter-clockwise order. - * 2. The polygon must not be self-crossing, otherwise the query may result in unexpected behavior - * 3. All latitude/longitude values must be in decimal degrees. - * 4. Complex computational geometry (e.g., dateline wrapping) is not supported - * 5. For more advanced GeoSpatial indexing and query operations see spatial module - * + * @see Polygon * @lucene.experimental */ public final class GeoPointInPolygonQuery extends GeoPointInBBoxQuery { @@ -79,7 +73,7 @@ public GeoPointInPolygonQuery(final String field, final TermEncoding termEncodin /** * Constructs a new GeoPolygonQuery that will match encoded {@link org.apache.lucene.spatial.geopoint.document.GeoPointField} terms - * that fall within or on the boundary of the polygon defined by the input parameters. + * that fall within or on the boundary of the polygons defined by the input parameters. */ public GeoPointInPolygonQuery(String field, TermEncoding termEncoding, Polygon... polygons) { this(field, termEncoding, Rectangle.fromPolygon(polygons), polygons); From e509eb6851f2fa8a9ae163073c0491e4895dfe8e Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 13:43:46 -0400 Subject: [PATCH 0272/1113] LUCENE-7157: Fix up javadoc --- .../spatial3d/geom/GeoConcavePolygon.java | 1 + .../spatial3d/geom/GeoConvexPolygon.java | 1 + .../spatial3d/geom/GeoPolygonFactory.java | 34 +++++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 7069e380fba9..375ec6088e44 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -295,6 +295,7 @@ protected class EitherBound implements Membership { protected final SidedPlane exception; /** Constructor. + * @param exception is the one plane to exclude from the check. */ public EitherBound(final SidedPlane exception) { this.exception = exception; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index 502af3f372d9..0a48eeb3a37c 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -281,6 +281,7 @@ protected class EitherBound implements Membership { protected final SidedPlane exception; /** Constructor. + * @param exception is the one plane to exclude from the check. */ public EitherBound(final SidedPlane exception) { this.exception = exception; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index d8040de64e88..d2d71b358e28 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -741,7 +741,14 @@ protected static class EdgeBuffer { protected final Set edges = new HashSet<>(); protected final Map previousEdges = new HashMap<>(); protected final Map nextEdges = new HashMap<>(); - + + /** Constructor. + * @param pointList is the list of points. + * @param startPlaneStartIndex is the index of the startPlane's starting point + * @param startPlaneEndIndex is the index of the startPlane's ending point + * @param startPlane is the starting plane + * @param startPlaneIsInternal signals whether the startPlane is an internal edge + */ public EdgeBuffer(final List pointList, final int startPlaneStartIndex, final int startPlaneEndIndex, final SidedPlane startPlane, final boolean startPlaneIsInternal) { /* System.out.println("Initial points:"); @@ -832,14 +839,26 @@ protected void verify() { } */ + /** Get the previous edge. + * @param currentEdge is the current edge. + * @return the previous edge, if found. + */ public Edge getPrevious(final Edge currentEdge) { return previousEdges.get(currentEdge); } + /** Get the next edge. + * @param currentEdge is the current edge. + * @return the next edge, if found. + */ public Edge getNext(final Edge currentEdge) { return nextEdges.get(currentEdge); } + /** Replace a list of edges with a new edge. + * @param removeList is the list of edges to remove. + * @param newEdge is the edge to add. + */ public void replace(final List removeList, final Edge newEdge) { /* System.out.println("Replacing: "); @@ -865,7 +884,9 @@ public void replace(final List removeList, final Edge newEdge) { } //verify(); } - + + /** Clear all edges. + */ public void clear() { edges.clear(); previousEdges.clear(); @@ -873,14 +894,23 @@ public void clear() { oneEdge = null; } + /** Get the size of the edge buffer. + * @return the size. + */ public int size() { return edges.size(); } + /** Get an iterator to iterate over edges. + * @return the iterator. + */ public Iterator iterator() { return new EdgeBufferIterator(this); } + /** Return a first edge. + * @return the edge. + */ public Edge pickOne() { return oneEdge; } From d119665ce43c2f0588df8a7626d94f3647cda656 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 13:59:10 -0400 Subject: [PATCH 0273/1113] LUCENE-7167: Fix spatial-extras tests that were broken by package private changes. One test is commented out while I search for a good way to do what it needs. --- .../org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java | 4 ++-- .../spatial/spatial4j/Geo3dShapeRectRelationTestCase.java | 6 ++++-- .../spatial4j/Geo3dShapeSphereModelRectRelationTest.java | 4 ++-- .../spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java index 2bef2b7cae18..aacea2cf5ba7 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java @@ -33,7 +33,7 @@ import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; -import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.PlanetModel; import org.junit.Test; import org.locationtech.spatial4j.context.SpatialContext; @@ -160,7 +160,7 @@ protected Shape randomQueryShape() { final int circleRadius = random().nextInt(179) + 1; final Point point = randomPoint(); try { - final GeoShape shape = new GeoStandardCircle(PlanetModel.SPHERE, point.getY() * DEGREES_TO_RADIANS, point.getX() * DEGREES_TO_RADIANS, + final GeoShape shape = GeoCircleFactory.makeGeoCircle(PlanetModel.SPHERE, point.getY() * DEGREES_TO_RADIANS, point.getX() * DEGREES_TO_RADIANS, circleRadius * DEGREES_TO_RADIANS); return new Geo3dShape(shape, ctx); } catch (IllegalArgumentException e) { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java index d58985f4c558..3fbfad8a208e 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java @@ -28,7 +28,7 @@ import org.apache.lucene.spatial3d.geom.LatLonBounds; import org.apache.lucene.spatial3d.geom.GeoBBox; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; -import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; @@ -110,6 +110,7 @@ protected int getBoundingMinimum(int laps) { } } + /* @Test public void testGeoCircleRect() { new Geo3dRectIntersectionTestHelper(ctx) { @@ -118,7 +119,7 @@ public void testGeoCircleRect() { protected Geo3dShape generateRandomShape(Point nearP) { final int circleRadius = 180 - random().nextInt(180);//no 0-radius final Point point = nearP; - final GeoShape shape = new GeoStandardCircle(planetModel, point.getY() * DEGREES_TO_RADIANS, point.getX() * DEGREES_TO_RADIANS, + final GeoShape shape = GeoCircleFactory.makeGeoCircle(planetModel, point.getY() * DEGREES_TO_RADIANS, point.getX() * DEGREES_TO_RADIANS, circleRadius * DEGREES_TO_RADIANS); return new Geo3dShape(planetModel, shape, ctx); } @@ -131,6 +132,7 @@ protected Point randomPointInEmptyShape(Geo3dShape shape) { }.testRelateWithRectangle(); } + */ @Test public void testGeoBBoxRect() { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java index 3bce480fe525..4aa2e1a95bb1 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeSphereModelRectRelationTest.java @@ -23,7 +23,7 @@ import org.apache.lucene.spatial3d.geom.GeoArea; import org.apache.lucene.spatial3d.geom.GeoBBox; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; -import org.apache.lucene.spatial3d.geom.GeoStandardCircle; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; @@ -60,7 +60,7 @@ public void testFailure1() { @Test public void testFailure2_LUCENE6475() { - GeoShape geo3dCircle = new GeoStandardCircle(planetModel, 1.6282053147165243E-4 * RADIANS_PER_DEGREE, + GeoShape geo3dCircle = GeoCircleFactory.makeGeoCircle(planetModel, 1.6282053147165243E-4 * RADIANS_PER_DEGREE, -70.1600629789353 * RADIANS_PER_DEGREE, 86 * RADIANS_PER_DEGREE); Geo3dShape geo3dShape = new Geo3dShape(planetModel, geo3dCircle, ctx); Rectangle rect = ctx.makeRectangle(-118, -114, -2.0, 32.0); diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java index b59d7df50a9a..00a0903ff360 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java @@ -19,8 +19,8 @@ import org.apache.lucene.spatial3d.geom.GeoArea; import org.apache.lucene.spatial3d.geom.GeoBBox; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoCircle; -import org.apache.lucene.spatial3d.geom.GeoStandardCircle; import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.PlanetModel; @@ -50,7 +50,7 @@ public void testFailure1() { public void testFailure2() { final GeoBBox rect = GeoBBoxFactory.makeGeoBBox(planetModel, -74 * RADIANS_PER_DEGREE, -90 * RADIANS_PER_DEGREE, 0 * RADIANS_PER_DEGREE, 26 * RADIANS_PER_DEGREE); - final GeoCircle circle = new GeoStandardCircle(planetModel, -87.3647352103 * RADIANS_PER_DEGREE, 52.3769709972 * RADIANS_PER_DEGREE, 1 * RADIANS_PER_DEGREE); + final GeoCircle circle = GeoCircleFactory.makeGeoCircle(planetModel, -87.3647352103 * RADIANS_PER_DEGREE, 52.3769709972 * RADIANS_PER_DEGREE, 1 * RADIANS_PER_DEGREE); assertTrue(GeoArea.DISJOINT == rect.getRelationship(circle)); // This is what the test failure claimed... //assertTrue(GeoArea.CONTAINS == rect.getRelationship(circle)); From 76f9e55431175b8ee30c6161fefd2f6cf6df84c4 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 14:41:42 -0400 Subject: [PATCH 0274/1113] LUCENE-7157: More javadoc fixes --- .../spatial3d/geom/GeoPolygonFactory.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index d2d71b358e28..847f17080710 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -675,11 +675,21 @@ protected static int getLegalIndex(int index, int size) { /** Class representing a single (unused) edge. */ protected static class Edge { + /** Plane */ public final SidedPlane plane; + /** Start point */ public final GeoPoint startPoint; + /** End point */ public final GeoPoint endPoint; + /** Internal edge flag */ public final boolean isInternal; - + + /** Constructor. + * @param startPoint the edge start point + * @param endPoint the edge end point + * @param plane the edge plane + * @param isInternal true if internal edge + */ public Edge(final GeoPoint startPoint, final GeoPoint endPoint, final SidedPlane plane, final boolean isInternal) { this.startPoint = startPoint; this.endPoint = endPoint; @@ -701,10 +711,16 @@ public boolean equals(final Object o) { /** Class representing an iterator over an EdgeBuffer. */ protected static class EdgeBufferIterator implements Iterator { + /** Edge buffer */ protected final EdgeBuffer edgeBuffer; + /** First edge */ protected final Edge firstEdge; + /** Current edge */ protected Edge currentEdge; + /** Constructor. + * @param edgeBuffer the edge buffer + */ public EdgeBufferIterator(final EdgeBuffer edgeBuffer) { this.edgeBuffer = edgeBuffer; this.currentEdge = edgeBuffer.pickOne(); @@ -737,9 +753,13 @@ public void remove() { /** Class representing a pool of unused edges, all linked together by vertices. */ protected static class EdgeBuffer { + /** Starting edge */ protected Edge oneEdge; + /** Full set of edges */ protected final Set edges = new HashSet<>(); + /** Map to previous edge */ protected final Map previousEdges = new HashMap<>(); + /** Map to next edge */ protected final Map nextEdges = new HashMap<>(); /** Constructor. From 974dc158e4511310c756df9a55c7f36ea773be19 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 14:51:28 -0400 Subject: [PATCH 0275/1113] LUCENE-7157: More javadoc fixes --- .../spatial3d/geom/GeoPolygonFactory.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 847f17080710..83c95b646ea3 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -236,6 +236,14 @@ protected static Boolean isInsidePolygon(final GeoPoint point, final List 0.0; } + /** Compute the angle for a point given rotation information. + * @param point is the point to assess + * @param sinLatitude the sine of the latitude + * @param cosLatitude the cosine of the latitude + * @param sinLongitude the sine of the longitude + * @param cosLongitude the cosine of the longitude + * @return the angle of rotation, or null if not computable + */ protected static Double computeAngle(final GeoPoint point, final double sinLatitude, final double cosLatitude, @@ -638,6 +646,13 @@ protected static Boolean findConvexPolygon(final PlanetModel planetModel, return true; } + /** Check if a point is within a set of edges. + * @param point is the point + * @param edgeSet is the set of edges + * @param extension is the new edge + * @param returnBoundary is the return edge + * @return true if within + */ protected static boolean isWithin(final GeoPoint point, final Set edgeSet, final Edge extension, final SidedPlane returnBoundary) { if (!extension.plane.isWithin(point)) { return false; @@ -648,6 +663,11 @@ protected static boolean isWithin(final GeoPoint point, final Set edgeSet, return isWithin(point, edgeSet); } + /** Check if a point is within a set of edges. + * @param point is the point + * @param edgeSet is the set of edges + * @return true if within + */ protected static boolean isWithin(final GeoPoint point, final Set edgeSet) { for (final Edge edge : edgeSet) { if (!edge.plane.isWithin(point)) { From 3330abe46f40482b7b745eb175c6b7abd26cb667 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Mon, 4 Apr 2016 16:52:51 -0400 Subject: [PATCH 0276/1113] Add Points format to o.a.l.codec package description --- lucene/core/src/java/org/apache/lucene/codecs/package-info.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/package-info.java index 28b260da498f..0c69886859ff 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/package-info.java @@ -25,6 +25,7 @@ *

  • DocValues - see {@link org.apache.lucene.codecs.DocValuesFormat}
    • *
  • Stored fields - see {@link org.apache.lucene.codecs.StoredFieldsFormat}
    • *
  • Term vectors - see {@link org.apache.lucene.codecs.TermVectorsFormat}
    • + *
  • Points - see {@link org.apache.lucene.codecs.PointsFormat}
    • *
  • FieldInfos - see {@link org.apache.lucene.codecs.FieldInfosFormat}
    • *
  • SegmentInfo - see {@link org.apache.lucene.codecs.SegmentInfoFormat}
    • *
  • Norms - see {@link org.apache.lucene.codecs.NormsFormat}
    • From b37da2b87d37e652fb572326174014bafe879082 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 4 Apr 2016 23:01:48 +0200 Subject: [PATCH 0277/1113] LUCENE-7174: Upgrade randomizedtesting to 2.3.4 --- lucene/CHANGES.txt | 4 ++++ lucene/ivy-versions.properties | 2 +- lucene/licenses/randomizedtesting-runner-2.3.2.jar.sha1 | 1 - lucene/licenses/randomizedtesting-runner-2.3.4.jar.sha1 | 1 + solr/licenses/junit4-ant-2.3.2.jar.sha1 | 1 - solr/licenses/junit4-ant-2.3.4.jar.sha1 | 1 + solr/licenses/randomizedtesting-runner-2.3.2.jar.sha1 | 1 - solr/licenses/randomizedtesting-runner-2.3.4.jar.sha1 | 1 + 8 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 lucene/licenses/randomizedtesting-runner-2.3.2.jar.sha1 create mode 100644 lucene/licenses/randomizedtesting-runner-2.3.4.jar.sha1 delete mode 100644 solr/licenses/junit4-ant-2.3.2.jar.sha1 create mode 100644 solr/licenses/junit4-ant-2.3.4.jar.sha1 delete mode 100644 solr/licenses/randomizedtesting-runner-2.3.2.jar.sha1 create mode 100644 solr/licenses/randomizedtesting-runner-2.3.4.jar.sha1 diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 2688f44afb56..da5be98b8d71 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -63,6 +63,10 @@ Bug Fixes * LUCENE-7166: Fix corner case bugs in LatLonPoint/GeoPointField bounding box queries. (Robert Muir) +Other + +* LUCENE-7174: Upgrade randomizedtesting to 2.3.4. (Uwe Schindler, Dawid Weiss) + ======================= Lucene 6.0.0 ======================= System Requirements diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index c2285771be9c..000a8ab84ac8 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -7,7 +7,7 @@ /cglib/cglib-nodep = 2.2 /com.adobe.xmp/xmpcore = 5.1.2 -com.carrotsearch.randomizedtesting.version = 2.3.2 +com.carrotsearch.randomizedtesting.version = 2.3.4 /com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version} /com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version} diff --git a/lucene/licenses/randomizedtesting-runner-2.3.2.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.3.2.jar.sha1 deleted file mode 100644 index 676dfa6e5d9a..000000000000 --- a/lucene/licenses/randomizedtesting-runner-2.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -307965917fe8a22b7ee72deba39ef4b8e6ebc069 diff --git a/lucene/licenses/randomizedtesting-runner-2.3.4.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.3.4.jar.sha1 new file mode 100644 index 000000000000..000702cddfe6 --- /dev/null +++ b/lucene/licenses/randomizedtesting-runner-2.3.4.jar.sha1 @@ -0,0 +1 @@ +9f4c0e1de0837092115c89a38c12ae57db6983e7 diff --git a/solr/licenses/junit4-ant-2.3.2.jar.sha1 b/solr/licenses/junit4-ant-2.3.2.jar.sha1 deleted file mode 100644 index f62363b7bf11..000000000000 --- a/solr/licenses/junit4-ant-2.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc8f03f6111974092491f35b8269eb0fc57f52f7 diff --git a/solr/licenses/junit4-ant-2.3.4.jar.sha1 b/solr/licenses/junit4-ant-2.3.4.jar.sha1 new file mode 100644 index 000000000000..1547f7880812 --- /dev/null +++ b/solr/licenses/junit4-ant-2.3.4.jar.sha1 @@ -0,0 +1 @@ +aafd329c4ddd57c539bdea9e4e5a4a688e142181 diff --git a/solr/licenses/randomizedtesting-runner-2.3.2.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.3.2.jar.sha1 deleted file mode 100644 index 676dfa6e5d9a..000000000000 --- a/solr/licenses/randomizedtesting-runner-2.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -307965917fe8a22b7ee72deba39ef4b8e6ebc069 diff --git a/solr/licenses/randomizedtesting-runner-2.3.4.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.3.4.jar.sha1 new file mode 100644 index 000000000000..000702cddfe6 --- /dev/null +++ b/solr/licenses/randomizedtesting-runner-2.3.4.jar.sha1 @@ -0,0 +1 @@ +9f4c0e1de0837092115c89a38c12ae57db6983e7 From 1a33e75c2934b1cb7427d5b4e214f8589ee0139b Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 4 Apr 2016 18:56:59 -0400 Subject: [PATCH 0278/1113] LUCENE-7173: Bring polygon API into compliance with 2D version. --- .../apache/lucene/spatial3d/Geo3DPoint.java | 122 +++++++++--------- 1 file changed, 62 insertions(+), 60 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index f0f202049db4..944ce39b97d9 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -22,6 +22,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.PointValues; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.spatial3d.geom.Vector; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoShape; @@ -30,6 +32,8 @@ import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoCompositePolygon; +import org.apache.lucene.spatial3d.geom.GeoPolygon; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -64,8 +68,8 @@ public final class Geo3DPoint extends Field { */ public Geo3DPoint(String name, double latitude, double longitude) { super(name, TYPE); - checkLatitude(latitude); - checkLongitude(longitude); + GeoUtils.checkLatitude(latitude); + GeoUtils.checkLongitude(longitude); // Translate latitude/longitude to x,y,z: final GeoPoint point = new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude)); fillFieldsData(point.x, point.y, point.z); @@ -94,8 +98,8 @@ protected static double fromMeters(final double meters) { * @throws IllegalArgumentException if {@code field} is null, location has invalid coordinates, or radius is invalid. */ public static Query newDistanceQuery(final String field, final double latitude, final double longitude, final double radiusMeters) { - checkLatitude(latitude); - checkLongitude(longitude); + GeoUtils.checkLatitude(latitude); + GeoUtils.checkLongitude(longitude); final GeoShape shape = GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude), fromMeters(radiusMeters)); return newShapeQuery(field, shape); } @@ -113,10 +117,10 @@ public static Query newDistanceQuery(final String field, final double latitude, * @throws IllegalArgumentException if {@code field} is null, or the box has invalid coordinates. */ public static Query newBoxQuery(final String field, final double minLatitude, final double maxLatitude, final double minLongitude, final double maxLongitude) { - checkLatitude(minLatitude); - checkLongitude(minLongitude); - checkLatitude(maxLatitude); - checkLongitude(maxLongitude); + GeoUtils.checkLatitude(minLatitude); + GeoUtils.checkLongitude(minLongitude); + GeoUtils.checkLatitude(maxLatitude); + GeoUtils.checkLongitude(maxLongitude); final GeoShape shape = GeoBBoxFactory.makeGeoBBox(PlanetModel.WGS84, fromDegrees(maxLatitude), fromDegrees(minLatitude), fromDegrees(minLongitude), fromDegrees(maxLongitude)); return newShapeQuery(field, shape); @@ -125,36 +129,28 @@ public static Query newBoxQuery(final String field, final double minLatitude, fi /** * Create a query for matching a polygon. *

    - * The supplied {@code polyLatitudes}/{@code polyLongitudes} must be clockwise or counter-clockwise. + * The supplied {@code polygons} must be clockwise on the outside level, counterclockwise on the next level in, etc. * @param field field name. must not be null. - * @param polyLatitudes latitude values for points of the polygon: must be within standard +/-90 coordinate bounds. - * @param polyLongitudes longitude values for points of the polygon: must be within standard +/-180 coordinate bounds. + * @param polygons is the list of polygons to use to construct the query; must be at least one. * @return query matching points within this polygon */ - public static Query newPolygonQuery(final String field, final double[] polyLatitudes, final double[] polyLongitudes) { - if (polyLatitudes.length != polyLongitudes.length) { - throw new IllegalArgumentException("same number of latitudes and longitudes required"); - } - if (polyLatitudes.length < 4) { - throw new IllegalArgumentException("need three or more points"); - } - if (polyLatitudes[0] != polyLatitudes[polyLatitudes.length-1] || polyLongitudes[0] != polyLongitudes[polyLongitudes.length-1]) { - throw new IllegalArgumentException("last point must equal first point"); + public static Query newPolygonQuery(final String field, final Polygon... polygons) { + if (polygons.length < 1) { + throw new IllegalArgumentException("need at least one polygon"); } - - final List polyPoints = new ArrayList<>(polyLatitudes.length-1); - for (int i = 0; i < polyLatitudes.length-1; i++) { - final double latitude = polyLatitudes[i]; - final double longitude = polyLongitudes[i]; - checkLatitude(latitude); - checkLongitude(longitude); - polyPoints.add(new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude))); + final GeoShape shape; + if (polygons.length == 1) { + shape = fromPolygon(polygons[0], false); + } else { + final GeoCompositePolygon poly = new GeoCompositePolygon(); + for (final Polygon p : polygons) { + poly.addShape(fromPolygon(p, false)); + } + shape = poly; } - // We use the polygon constructor that looks at point order. - final GeoShape shape = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, polyPoints, null); return newShapeQuery(field, shape); } - + /** * Create a query for matching a path. *

    @@ -172,14 +168,47 @@ public static Query newPathQuery(final String field, final double[] pathLatitude for (int i = 0; i < pathLatitudes.length; i++) { final double latitude = pathLatitudes[i]; final double longitude = pathLongitudes[i]; - checkLatitude(latitude); - checkLongitude(longitude); + GeoUtils.checkLatitude(latitude); + GeoUtils.checkLongitude(longitude); points[i] = new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude)); } final GeoShape shape = new GeoPath(PlanetModel.WGS84, fromMeters(pathWidthMeters), points); return newShapeQuery(field, shape); } + /** + * Convert a Polygon object into a GeoPolygon. + * This method uses + * @param polygon is the Polygon object. + * @param reverseMe is true if the order of the points should be reversed. + * @return the GeoPolygon. + */ + protected static GeoPolygon fromPolygon(final Polygon polygon, final boolean reverseMe) { + // First, assemble the "holes". The geo3d convention is to use the same polygon sense on the inner ring as the + // outer ring, so we process these recursively with reverseMe flipped. + final Polygon[] theHoles = polygon.getHoles(); + final List holeList = new ArrayList<>(theHoles.length); + for (final Polygon hole : theHoles) { + holeList.add(fromPolygon(hole, !reverseMe)); + } + // Now do the polygon itself + final double[] polyLats = polygon.getPolyLats(); + final double[] polyLons = polygon.getPolyLons(); + + // I presume the arguments have already been checked + final List points = new ArrayList<>(polyLats.length-1); + // We skip the last point anyway because the API requires it to be repeated, and geo3d doesn't repeat it. + for (int i = 0; i < polyLats.length - 1; i++) { + if (reverseMe) { + points.add(new GeoPoint(PlanetModel.WGS84, fromDegrees(polyLats[i]), fromDegrees(polyLons[i]))); + } else { + final int index = polyLats.length - 2 - i; + points.add(new GeoPoint(PlanetModel.WGS84, fromDegrees(polyLats[index]), fromDegrees(polyLons[index]))); + } + } + return GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, points, holeList); + } + /** * Creates a new Geo3DPoint field with the specified x,y,z. * @@ -235,31 +264,4 @@ public String toString() { return result.toString(); } - // TODO LUCENE-7152: share this with GeoUtils.java from spatial module - - /** Minimum longitude value. */ - private static final double MIN_LON_INCL = -180.0D; - - /** Maximum longitude value. */ - private static final double MAX_LON_INCL = 180.0D; - - /** Minimum latitude value. */ - private static final double MIN_LAT_INCL = -90.0D; - - /** Maximum latitude value. */ - private static final double MAX_LAT_INCL = 90.0D; - - /** validates latitude value is within standard +/-90 coordinate bounds */ - private static void checkLatitude(double latitude) { - if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) { - throw new IllegalArgumentException("invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL); - } - } - - /** validates longitude value is within standard +/-180 coordinate bounds */ - private static void checkLongitude(double longitude) { - if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) { - throw new IllegalArgumentException("invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL); - } - } } From 56a4b6af371239a9dd3e4bb3fea0a831e327e62e Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Tue, 5 Apr 2016 03:00:15 -0400 Subject: [PATCH 0279/1113] LUCENE-7167: Re-enable test I disabled because of the package-private changes. --- .../spatial/spatial4j/Geo3dShapeRectRelationTestCase.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java index 3fbfad8a208e..2c41f79b8808 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java @@ -28,6 +28,7 @@ import org.apache.lucene.spatial3d.geom.LatLonBounds; import org.apache.lucene.spatial3d.geom.GeoBBox; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoCircle; import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.spatial3d.geom.GeoPoint; @@ -110,7 +111,6 @@ protected int getBoundingMinimum(int laps) { } } - /* @Test public void testGeoCircleRect() { new Geo3dRectIntersectionTestHelper(ctx) { @@ -126,13 +126,12 @@ protected Geo3dShape generateRandomShape(Point nearP) { @Override protected Point randomPointInEmptyShape(Geo3dShape shape) { - GeoPoint geoPoint = ((GeoStandardCircle)shape.shape).getCenter(); + GeoPoint geoPoint = ((GeoCircle)shape.shape).getCenter(); return geoPointToSpatial4jPoint(geoPoint); } }.testRelateWithRectangle(); } - */ @Test public void testGeoBBoxRect() { From 784e3e38631dce6a6f863390ca43e17836de86eb Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Tue, 5 Apr 2016 03:41:48 -0400 Subject: [PATCH 0280/1113] LUCENE-7176: Hide GeoPath implementation in a factory/interface. --- .../spatial/spatial4j/Geo3dRptTest.java | 26 +- .../Geo3dShapeRectRelationTestCase.java | 15 +- .../Geo3dShapeWGS84ModelRectRelationTest.java | 15 +- .../apache/lucene/spatial3d/Geo3DPoint.java | 4 +- .../lucene/spatial3d/geom/GeoBaseCircle.java | 2 +- .../lucene/spatial3d/geom/GeoBasePath.java | 34 + .../apache/lucene/spatial3d/geom/GeoPath.java | 776 +---------------- .../lucene/spatial3d/geom/GeoPathFactory.java | 39 + .../spatial3d/geom/GeoStandardPath.java | 797 ++++++++++++++++++ .../lucene/spatial3d/TestGeo3DPoint.java | 13 +- .../lucene/spatial3d/geom/GeoPathTest.java | 36 +- 11 files changed, 928 insertions(+), 829 deletions(-) create mode 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePath.java mode change 100755 => 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java create mode 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java create mode 100755 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java index aacea2cf5ba7..004139e4bfbf 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java @@ -29,7 +29,7 @@ import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.serialized.SerializedDVStrategy; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; -import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; @@ -95,12 +95,12 @@ public void testFailureLucene6535() throws IOException { points.add(new GeoPoint(PlanetModel.SPHERE, -57 * DEGREES_TO_RADIANS, 146 * DEGREES_TO_RADIANS)); points.add(new GeoPoint(PlanetModel.SPHERE, 14 * DEGREES_TO_RADIANS, -180 * DEGREES_TO_RADIANS)); points.add(new GeoPoint(PlanetModel.SPHERE, -15 * DEGREES_TO_RADIANS, 153 * DEGREES_TO_RADIANS)); - final GeoPath path = new GeoPath(PlanetModel.SPHERE, 29 * DEGREES_TO_RADIANS); - path.addPoint(55.0 * DEGREES_TO_RADIANS, -26.0 * DEGREES_TO_RADIANS); - path.addPoint(-90.0 * DEGREES_TO_RADIANS, 0.0); - path.addPoint(54.0 * DEGREES_TO_RADIANS, 165.0 * DEGREES_TO_RADIANS); - path.addPoint(-90.0 * DEGREES_TO_RADIANS, 0.0); - path.done(); + final GeoPoint[] pathPoints = new GeoPoint[] { + new GeoPoint(PlanetModel.SPHERE, 55.0 * DEGREES_TO_RADIANS, -26.0 * DEGREES_TO_RADIANS), + new GeoPoint(PlanetModel.SPHERE, -90.0 * DEGREES_TO_RADIANS, 0.0), + new GeoPoint(PlanetModel.SPHERE, 54.0 * DEGREES_TO_RADIANS, 165.0 * DEGREES_TO_RADIANS), + new GeoPoint(PlanetModel.SPHERE, -90.0 * DEGREES_TO_RADIANS, 0.0)}; + final GeoShape path = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 29 * DEGREES_TO_RADIANS, pathPoints); final Shape shape = new Geo3dShape(path,ctx); final Rectangle rect = ctx.makeRectangle(131, 143, 39, 54); testOperation(rect,SpatialOperation.Intersects,shape,true); @@ -199,14 +199,14 @@ protected Shape randomQueryShape() { // Paths final int pointCount = random().nextInt(5) + 1; final double width = (random().nextInt(89)+1) * DEGREES_TO_RADIANS; + final GeoPoint[] points = new GeoPoint[pointCount]; while (true) { + for (int i = 0; i < pointCount; i++) { + final Point nextPoint = randomPoint(); + points[i] = new GeoPoint(PlanetModel.SPHERE, nextPoint.getY() * DEGREES_TO_RADIANS, nextPoint.getX() * DEGREES_TO_RADIANS); + } try { - final GeoPath path = new GeoPath(PlanetModel.SPHERE, width); - for (int i = 0; i < pointCount; i++) { - final Point nextPoint = randomPoint(); - path.addPoint(nextPoint.getY() * DEGREES_TO_RADIANS, nextPoint.getX() * DEGREES_TO_RADIANS); - } - path.done(); + final GeoShape path = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, width, points); return new Geo3dShape(path, ctx); } catch (IllegalArgumentException e) { // This is what happens when we create a shape that is invalid. Although it is conceivable that there are cases where diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java index 2c41f79b8808..f8e51d17ed9b 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java @@ -30,7 +30,7 @@ import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoCircle; import org.apache.lucene.spatial3d.geom.GeoCircleFactory; -import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; @@ -221,14 +221,15 @@ protected Geo3dShape generateRandomShape(Point nearP) { final Circle pointZone = ctx.makeCircle(centerPoint, maxDistance); final int pointCount = random().nextInt(5) + 1; final double width = (random().nextInt(89)+1) * DEGREES_TO_RADIANS; + final GeoPoint[] points = new GeoPoint[pointCount]; while (true) { + for (int i = 0; i < pointCount; i++) { + final Point nextPoint = randomPointIn(pointZone); + points[i] = new GeoPoint(planetModel, nextPoint.getY() * DEGREES_TO_RADIANS, nextPoint.getX() * DEGREES_TO_RADIANS); + } + try { - final GeoPath path = new GeoPath(planetModel, width); - for (int i = 0; i < pointCount; i++) { - final Point nextPoint = randomPointIn(pointZone); - path.addPoint(nextPoint.getY() * DEGREES_TO_RADIANS, nextPoint.getX() * DEGREES_TO_RADIANS); - } - path.done(); + final GeoShape path = GeoPathFactory.makeGeoPath(planetModel, width, points); return new Geo3dShape(planetModel, path, ctx); } catch (IllegalArgumentException e) { // This is what happens when we create a shape that is invalid. Although it is conceivable that there are cases where diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java index 00a0903ff360..43b88ca5912e 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java @@ -21,6 +21,7 @@ import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoCircle; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; import org.apache.lucene.spatial3d.geom.GeoPath; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.PlanetModel; @@ -36,9 +37,9 @@ public Geo3dShapeWGS84ModelRectRelationTest() { public void testFailure1() { final GeoBBox rect = GeoBBoxFactory.makeGeoBBox(planetModel, 90 * RADIANS_PER_DEGREE, 74 * RADIANS_PER_DEGREE, 40 * RADIANS_PER_DEGREE, 60 * RADIANS_PER_DEGREE); - final GeoPath path = new GeoPath(planetModel, 4 * RADIANS_PER_DEGREE); - path.addPoint(84.4987594274 * RADIANS_PER_DEGREE, -22.8345484402 * RADIANS_PER_DEGREE); - path.done(); + final GeoPoint[] pathPoints = new GeoPoint[] { + new GeoPoint(planetModel, 84.4987594274 * RADIANS_PER_DEGREE, -22.8345484402 * RADIANS_PER_DEGREE)}; + final GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 4 * RADIANS_PER_DEGREE, pathPoints); assertTrue(GeoArea.DISJOINT == rect.getRelationship(path)); // This is what the test failure claimed... //assertTrue(GeoArea.CONTAINS == rect.getRelationship(path)); @@ -75,10 +76,10 @@ public void testFailure3() { */ final GeoBBox rect = GeoBBoxFactory.makeGeoBBox(planetModel, 16 * RADIANS_PER_DEGREE, 16 * RADIANS_PER_DEGREE, 4 * RADIANS_PER_DEGREE, 36 * RADIANS_PER_DEGREE); final GeoPoint pt = new GeoPoint(planetModel, 16 * RADIANS_PER_DEGREE, 23.81626064835212 * RADIANS_PER_DEGREE); - final GeoPath path = new GeoPath(planetModel, 88 * RADIANS_PER_DEGREE); - path.addPoint(46.6369060853 * RADIANS_PER_DEGREE, -79.8452213228 * RADIANS_PER_DEGREE); - path.addPoint(54.9779334519 * RADIANS_PER_DEGREE, 132.029177424 * RADIANS_PER_DEGREE); - path.done(); + final GeoPoint[] pathPoints = new GeoPoint[]{ + new GeoPoint(planetModel, 46.6369060853 * RADIANS_PER_DEGREE, -79.8452213228 * RADIANS_PER_DEGREE), + new GeoPoint(planetModel, 54.9779334519 * RADIANS_PER_DEGREE, 132.029177424 * RADIANS_PER_DEGREE)}; + final GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 88 * RADIANS_PER_DEGREE, pathPoints); System.out.println("rect=" + rect); // Rectangle is within path (this is wrong; it's on the other side. Should be OVERLAPS) assertTrue(GeoArea.OVERLAPS == rect.getRelationship(path)); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index 944ce39b97d9..be8990f95081 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -31,7 +31,7 @@ import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; -import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; import org.apache.lucene.spatial3d.geom.GeoCompositePolygon; import org.apache.lucene.spatial3d.geom.GeoPolygon; import org.apache.lucene.search.Query; @@ -172,7 +172,7 @@ public static Query newPathQuery(final String field, final double[] pathLatitude GeoUtils.checkLongitude(longitude); points[i] = new GeoPoint(PlanetModel.WGS84, fromDegrees(latitude), fromDegrees(longitude)); } - final GeoShape shape = new GeoPath(PlanetModel.WGS84, fromMeters(pathWidthMeters), points); + final GeoShape shape = GeoPathFactory.makeGeoPath(PlanetModel.WGS84, fromMeters(pathWidthMeters), points); return newShapeQuery(field, shape); } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java index b34313a495be..0599c91aa5c0 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBaseCircle.java @@ -19,7 +19,7 @@ /** * GeoCircles have all the characteristics of GeoBaseDistanceShapes, plus GeoSizeable. * - * @lucene.experimental + * @lucene.internal */ abstract class GeoBaseCircle extends GeoBaseDistanceShape implements GeoCircle { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePath.java new file mode 100644 index 000000000000..c726b3af2bac --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBasePath.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial3d.geom; + +/** + * GeoPaths have all the characteristics of GeoBaseDistanceShapes. + * + * @lucene.internal + */ +abstract class GeoBasePath extends GeoBaseDistanceShape implements GeoPath { + + /** Constructor. + *@param planetModel is the planet model to use. + */ + public GeoBasePath(final PlanetModel planetModel) { + super(planetModel); + } + +} + diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java old mode 100755 new mode 100644 index a5b8b9b06608..48a613bf258a --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java @@ -16,782 +16,10 @@ */ package org.apache.lucene.spatial3d.geom; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** - * GeoShape representing a path across the surface of the globe, - * with a specified half-width. Path is described by a series of points. - * Distances are measured from the starting point along the path, and then at right - * angles to the path. + * Interface describing a path. * * @lucene.experimental */ -public class GeoPath extends GeoBaseDistanceShape { - /** The cutoff angle (width) */ - protected final double cutoffAngle; - - /** Sine of cutoff angle */ - protected final double sinAngle; - /** Cosine of cutoff angle */ - protected final double cosAngle; - - /** The original list of path points */ - protected final List points = new ArrayList(); - - /** A list of SegmentEndpoints */ - protected List endPoints; - /** A list of PathSegments */ - protected List segments; - - /** A point on the edge */ - protected GeoPoint[] edgePoints; - - /** Set to true if path has been completely constructed */ - protected boolean isDone = false; - - /** Constructor. - *@param planetModel is the planet model. - *@param maxCutoffAngle is the width of the path, measured as an angle. - *@param pathPoints are the points in the path. - */ - public GeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) { - this(planetModel, maxCutoffAngle); - Collections.addAll(points, pathPoints); - done(); - } - - /** Piece-wise constructor. Use in conjunction with addPoint() and done(). - *@param planetModel is the planet model. - *@param maxCutoffAngle is the width of the path, measured as an angle. - */ - public GeoPath(final PlanetModel planetModel, final double maxCutoffAngle) { - super(planetModel); - if (maxCutoffAngle <= 0.0 || maxCutoffAngle > Math.PI * 0.5) - throw new IllegalArgumentException("Cutoff angle out of bounds"); - this.cutoffAngle = maxCutoffAngle; - this.cosAngle = Math.cos(maxCutoffAngle); - this.sinAngle = Math.sin(maxCutoffAngle); - } - - /** Add a point to the path. - *@param lat is the latitude of the point. - *@param lon is the longitude of the point. - */ - public void addPoint(final double lat, final double lon) { - if (isDone) - throw new IllegalStateException("Can't call addPoint() if done() already called"); - points.add(new GeoPoint(planetModel, lat, lon)); - } - - /** Complete the path. - */ - public void done() { - if (isDone) - throw new IllegalStateException("Can't call done() twice"); - if (points.size() == 0) - throw new IllegalArgumentException("Path must have at least one point"); - isDone = true; - - endPoints = new ArrayList<>(points.size()); - segments = new ArrayList<>(points.size()); - // Compute an offset to use for all segments. This will be based on the minimum magnitude of - // the entire ellipsoid. - final double cutoffOffset = this.sinAngle * planetModel.getMinimumMagnitude(); - - // First, build all segments. We'll then go back and build corresponding segment endpoints. - GeoPoint lastPoint = null; - for (final GeoPoint end : points) { - if (lastPoint != null) { - final Plane normalizedConnectingPlane = new Plane(lastPoint, end); - if (normalizedConnectingPlane == null) { - continue; - } - segments.add(new PathSegment(planetModel, lastPoint, end, normalizedConnectingPlane, cutoffOffset)); - } - lastPoint = end; - } - - if (segments.size() == 0) { - // Simple circle - double lat = points.get(0).getLatitude(); - double lon = points.get(0).getLongitude(); - // Compute two points on the circle, with the right angle from the center. We'll use these - // to obtain the perpendicular plane to the circle. - double upperLat = lat + cutoffAngle; - double upperLon = lon; - if (upperLat > Math.PI * 0.5) { - upperLon += Math.PI; - if (upperLon > Math.PI) - upperLon -= 2.0 * Math.PI; - upperLat = Math.PI - upperLat; - } - double lowerLat = lat - cutoffAngle; - double lowerLon = lon; - if (lowerLat < -Math.PI * 0.5) { - lowerLon += Math.PI; - if (lowerLon > Math.PI) - lowerLon -= 2.0 * Math.PI; - lowerLat = -Math.PI - lowerLat; - } - final GeoPoint upperPoint = new GeoPoint(planetModel, upperLat, upperLon); - final GeoPoint lowerPoint = new GeoPoint(planetModel, lowerLat, lowerLon); - final GeoPoint point = points.get(0); - - // Construct normal plane - final Plane normalPlane = Plane.constructNormalizedZPlane(upperPoint, lowerPoint, point); - - final SegmentEndpoint onlyEndpoint = new SegmentEndpoint(point, normalPlane, upperPoint, lowerPoint); - endPoints.add(onlyEndpoint); - this.edgePoints = new GeoPoint[]{onlyEndpoint.circlePlane.getSampleIntersectionPoint(planetModel, normalPlane)}; - return; - } - - // Create segment endpoints. Use an appropriate constructor for the start and end of the path. - for (int i = 0; i < segments.size(); i++) { - final PathSegment currentSegment = segments.get(i); - - if (i == 0) { - // Starting endpoint - final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start, - currentSegment.startCutoffPlane, currentSegment.ULHC, currentSegment.LLHC); - endPoints.add(startEndpoint); - this.edgePoints = new GeoPoint[]{currentSegment.ULHC}; - continue; - } - - // General intersection case - final PathSegment prevSegment = segments.get(i-1); - // We construct four separate planes, and evaluate which one includes all interior points with least overlap - final SidedPlane candidate1 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, prevSegment.URHC, currentSegment.ULHC, currentSegment.LLHC); - final SidedPlane candidate2 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, currentSegment.ULHC, currentSegment.LLHC, prevSegment.LRHC); - final SidedPlane candidate3 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, currentSegment.LLHC, prevSegment.LRHC, prevSegment.URHC); - final SidedPlane candidate4 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, prevSegment.LRHC, prevSegment.URHC, currentSegment.ULHC); - - if (candidate1 == null && candidate2 == null && candidate3 == null && candidate4 == null) { - // The planes are identical. We wouldn't need a circle at all except for the possibility of - // backing up, which is hard to detect here. - final SegmentEndpoint midEndpoint = new SegmentEndpoint(currentSegment.start, - prevSegment.endCutoffPlane, currentSegment.startCutoffPlane, currentSegment.ULHC, currentSegment.LLHC); - //don't need a circle at all. Special constructor... - endPoints.add(midEndpoint); - } else { - endPoints.add(new SegmentEndpoint(currentSegment.start, - prevSegment.endCutoffPlane, currentSegment.startCutoffPlane, - prevSegment.URHC, prevSegment.LRHC, - currentSegment.ULHC, currentSegment.LLHC, - candidate1, candidate2, candidate3, candidate4)); - } - } - // Do final endpoint - final PathSegment lastSegment = segments.get(segments.size()-1); - endPoints.add(new SegmentEndpoint(lastSegment.end, - lastSegment.endCutoffPlane, lastSegment.URHC, lastSegment.LRHC)); - - } - - @Override - protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { - // Algorithm: - // (1) If the point is within any of the segments along the path, return that value. - // (2) If the point is within any of the segment end circles along the path, return that value. - double currentDistance = 0.0; - for (PathSegment segment : segments) { - double distance = segment.pathDistance(planetModel, distanceStyle, x,y,z); - if (distance != Double.MAX_VALUE) - return currentDistance + distance; - currentDistance += segment.fullPathDistance(distanceStyle); - } - - int segmentIndex = 0; - currentDistance = 0.0; - for (SegmentEndpoint endpoint : endPoints) { - double distance = endpoint.pathDistance(distanceStyle, x, y, z); - if (distance != Double.MAX_VALUE) - return currentDistance + distance; - if (segmentIndex < segments.size()) - currentDistance += segments.get(segmentIndex++).fullPathDistance(distanceStyle); - } - - return Double.MAX_VALUE; - } - - @Override - protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { - double minDistance = Double.MAX_VALUE; - for (final SegmentEndpoint endpoint : endPoints) { - final double newDistance = endpoint.outsideDistance(distanceStyle, x,y,z); - if (newDistance < minDistance) - minDistance = newDistance; - } - for (final PathSegment segment : segments) { - final double newDistance = segment.outsideDistance(planetModel, distanceStyle, x, y, z); - if (newDistance < minDistance) - minDistance = newDistance; - } - return minDistance; - } - - @Override - public boolean isWithin(final double x, final double y, final double z) { - for (SegmentEndpoint pathPoint : endPoints) { - if (pathPoint.isWithin(x, y, z)) - return true; - } - for (PathSegment pathSegment : segments) { - if (pathSegment.isWithin(x, y, z)) - return true; - } - return false; - } - - @Override - public GeoPoint[] getEdgePoints() { - return edgePoints; - } - - @Override - public boolean intersects(final Plane plane, final GeoPoint[] notablePoints, final Membership... bounds) { - // We look for an intersection with any of the exterior edges of the path. - // We also have to look for intersections with the cones described by the endpoints. - // Return "true" if any such intersections are found. - - // For plane intersections, the basic idea is to come up with an equation of the line that is - // the intersection (if any). Then, find the intersections with the unit sphere (if any). If - // any of the intersection points are within the bounds, then we've detected an intersection. - // Well, sort of. We can detect intersections also due to overlap of segments with each other. - // But that's an edge case and we won't be optimizing for it. - //System.err.println(" Looking for intersection of plane "+plane+" with path "+this); - for (final SegmentEndpoint pathPoint : endPoints) { - if (pathPoint.intersects(planetModel, plane, notablePoints, bounds)) { - return true; - } - } - - for (final PathSegment pathSegment : segments) { - if (pathSegment.intersects(planetModel, plane, notablePoints, bounds)) { - return true; - } - } - - return false; - } - - @Override - public void getBounds(Bounds bounds) { - super.getBounds(bounds); - // For building bounds, order matters. We want to traverse - // never more than 180 degrees longitude at a pop or we risk having the - // bounds object get itself inverted. So do the edges first. - for (PathSegment pathSegment : segments) { - pathSegment.getBounds(planetModel, bounds); - } - for (SegmentEndpoint pathPoint : endPoints) { - pathPoint.getBounds(planetModel, bounds); - } - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof GeoPath)) - return false; - GeoPath p = (GeoPath) o; - if (!super.equals(p)) - return false; - if (cutoffAngle != p.cutoffAngle) - return false; - return points.equals(p.points); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - long temp = Double.doubleToLongBits(cutoffAngle); - result = 31 * result + (int) (temp ^ (temp >>> 32)); - result = 31 * result + points.hashCode(); - return result; - } - - @Override - public String toString() { - return "GeoPath: {planetmodel=" + planetModel+", width=" + cutoffAngle + "(" + cutoffAngle * 180.0 / Math.PI + "), points={" + points + "}}"; - } - - /** - * This is precalculated data for segment endpoint. - * Note well: This is not necessarily a circle. There are four cases: - * (1) The path consists of a single endpoint. In this case, we build a simple circle with the proper cutoff offset. - * (2) This is the end of a path. The circle plane must be constructed to go through two supplied points and be perpendicular to a connecting plane. - * (2.5) Intersection, but the path on both sides is linear. We generate a circle, but we use the cutoff planes to limit its influence in the straight line case. - * (3) This is an intersection in a path. We are supplied FOUR planes. If there are intersections within bounds for both upper and lower, then - * we generate no circle at all. If there is one intersection only, then we generate a plane that includes that intersection, as well as the remaining - * cutoff plane/edge plane points. - */ - public static class SegmentEndpoint { - /** The center point of the endpoint */ - public final GeoPoint point; - /** A plane describing the circle */ - public final SidedPlane circlePlane; - /** Pertinent cutoff planes from adjoining segments */ - public final Membership[] cutoffPlanes; - /** Notable points for this segment endpoint */ - public final GeoPoint[] notablePoints; - /** No notable points from the circle itself */ - public final static GeoPoint[] circlePoints = new GeoPoint[0]; - /** Null membership */ - public final static Membership[] NO_MEMBERSHIP = new Membership[0]; - - /** Base case. Does nothing at all. - */ - public SegmentEndpoint(final GeoPoint point) { - this.point = point; - this.circlePlane = null; - this.cutoffPlanes = null; - this.notablePoints = null; - } - - /** Constructor for case (1). - * Generate a simple circle cutoff plane. - *@param point is the center point. - *@param upperPoint is a point that must be on the circle plane. - *@param lowerPoint is another point that must be on the circle plane. - */ - public SegmentEndpoint(final GeoPoint point, final Plane normalPlane, final GeoPoint upperPoint, final GeoPoint lowerPoint) { - this.point = point; - // Construct a sided plane that goes through the two points and whose normal is in the normalPlane. - this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, normalPlane, upperPoint, lowerPoint); - this.cutoffPlanes = NO_MEMBERSHIP; - this.notablePoints = circlePoints; - } - - /** Constructor for case (2). - * Generate an endpoint, given a single cutoff plane plus upper and lower edge points. - *@param point is the center point. - *@param cutoffPlane is the plane from the adjoining path segment marking the boundary between this endpoint and that segment. - *@param topEdgePoint is a point on the cutoffPlane that should be also on the circle plane. - *@param bottomEdgePoint is another point on the cutoffPlane that should be also on the circle plane. - */ - public SegmentEndpoint(final GeoPoint point, - final SidedPlane cutoffPlane, final GeoPoint topEdgePoint, final GeoPoint bottomEdgePoint) { - this.point = point; - this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane)}; - this.notablePoints = new GeoPoint[]{topEdgePoint, bottomEdgePoint}; - // To construct the plane, we now just need D, which is simply the negative of the evaluation of the circle normal vector at one of the points. - this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, cutoffPlane, topEdgePoint, bottomEdgePoint); - } - - /** Constructor for case (2.5). - * Generate an endpoint, given two cutoff planes plus upper and lower edge points. - *@param point is the center. - *@param cutoffPlane1 is one adjoining path segment cutoff plane. - *@param cutoffPlane2 is another adjoining path segment cutoff plane. - *@param topEdgePoint is a point on the cutoffPlane that should be also on the circle plane. - *@param bottomEdgePoint is another point on the cutoffPlane that should be also on the circle plane. - */ - public SegmentEndpoint(final GeoPoint point, - final SidedPlane cutoffPlane1, final SidedPlane cutoffPlane2, final GeoPoint topEdgePoint, final GeoPoint bottomEdgePoint) { - this.point = point; - this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane1), new SidedPlane(cutoffPlane2)}; - this.notablePoints = new GeoPoint[]{topEdgePoint, bottomEdgePoint}; - // To construct the plane, we now just need D, which is simply the negative of the evaluation of the circle normal vector at one of the points. - this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, cutoffPlane1, topEdgePoint, bottomEdgePoint); - } - - /** Constructor for case (3). - * Generate an endpoint for an intersection, given four points. - *@param point is the center. - *@param prevCutoffPlane is the previous adjoining segment cutoff plane. - *@param nextCutoffPlane is the next path segment cutoff plane. - *@param notCand2Point is a point NOT on candidate2. - *@param notCand1Point is a point NOT on candidate1. - *@param notCand3Point is a point NOT on candidate3. - *@param notCand4Point is a point NOT on candidate4. - *@param candidate1 one of four candidate circle planes. - *@param candidate2 one of four candidate circle planes. - *@param candidate3 one of four candidate circle planes. - *@param candidate4 one of four candidate circle planes. - */ - public SegmentEndpoint(final GeoPoint point, - final SidedPlane prevCutoffPlane, final SidedPlane nextCutoffPlane, - final GeoPoint notCand2Point, final GeoPoint notCand1Point, - final GeoPoint notCand3Point, final GeoPoint notCand4Point, - final SidedPlane candidate1, final SidedPlane candidate2, final SidedPlane candidate3, final SidedPlane candidate4) { - // Note: What we really need is a single plane that goes through all four points. - // Since that's not possible in the ellipsoid case (because three points determine a plane, not four), we - // need an approximation that at least creates a boundary that has no interruptions. - // There are three obvious choices for the third point: either (a) one of the two remaining points, or (b) the top or bottom edge - // intersection point. (a) has no guarantee of continuity, while (b) is capable of producing something very far from a circle if - // the angle between segments is acute. - // The solution is to look for the side (top or bottom) that has an intersection within the shape. We use the two points from - // the opposite side to determine the plane, AND we pick the third to be either of the two points on the intersecting side - // PROVIDED that the other point is within the final circle we come up with. - this.point = point; - - // We construct four separate planes, and evaluate which one includes all interior points with least overlap - // (Constructed beforehand because we need them for degeneracy check) - - final boolean cand1IsOtherWithin = candidate1!=null?candidate1.isWithin(notCand1Point):false; - final boolean cand2IsOtherWithin = candidate2!=null?candidate2.isWithin(notCand2Point):false; - final boolean cand3IsOtherWithin = candidate3!=null?candidate3.isWithin(notCand3Point):false; - final boolean cand4IsOtherWithin = candidate4!=null?candidate4.isWithin(notCand4Point):false; - - if (cand1IsOtherWithin && cand2IsOtherWithin && cand3IsOtherWithin && cand4IsOtherWithin) { - // The only way we should see both within is if all four points are coplanar. In that case, we default to the simplest treatment. - this.circlePlane = candidate1; // doesn't matter which - this.notablePoints = new GeoPoint[]{notCand2Point, notCand3Point, notCand1Point, notCand4Point}; - this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane), new SidedPlane(nextCutoffPlane)}; - } else if (cand1IsOtherWithin) { - // Use candidate1, and DON'T include prevCutoffPlane in the cutoff planes list - this.circlePlane = candidate1; - this.notablePoints = new GeoPoint[]{notCand2Point, notCand3Point, notCand4Point}; - this.cutoffPlanes = new Membership[]{new SidedPlane(nextCutoffPlane)}; - } else if (cand2IsOtherWithin) { - // Use candidate2 - this.circlePlane = candidate2; - this.notablePoints = new GeoPoint[]{notCand3Point, notCand4Point, notCand1Point}; - this.cutoffPlanes = new Membership[]{new SidedPlane(nextCutoffPlane)}; - } else if (cand3IsOtherWithin) { - this.circlePlane = candidate3; - this.notablePoints = new GeoPoint[]{notCand4Point, notCand1Point, notCand2Point}; - this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane)}; - } else if (cand4IsOtherWithin) { - this.circlePlane = candidate4; - this.notablePoints = new GeoPoint[]{notCand1Point, notCand2Point, notCand3Point}; - this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane)}; - } else { - // dunno what happened - throw new RuntimeException("Couldn't come up with a plane through three points that included the fourth"); - } - } - - /** Check if point is within this endpoint. - *@param point is the point. - *@return true of within. - */ - public boolean isWithin(final Vector point) { - if (circlePlane == null) - return false; - if (!circlePlane.isWithin(point)) - return false; - for (final Membership m : cutoffPlanes) { - if (!m.isWithin(point)) { - return false; - } - } - return true; - } - - /** Check if point is within this endpoint. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return true of within. - */ - public boolean isWithin(final double x, final double y, final double z) { - if (circlePlane == null) - return false; - if (!circlePlane.isWithin(x, y, z)) - return false; - for (final Membership m : cutoffPlanes) { - if (!m.isWithin(x,y,z)) { - return false; - } - } - return true; - } - - /** Compute interior path distance. - *@param distanceStyle is the distance style. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return the distance metric. - */ - public double pathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { - if (!isWithin(x,y,z)) - return Double.MAX_VALUE; - return distanceStyle.computeDistance(this.point, x, y, z); - } - - /** Compute external distance. - *@param distanceStyle is the distance style. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return the distance metric. - */ - public double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { - return distanceStyle.computeDistance(this.point, x, y, z); - } - - /** Determine if this endpoint intersects a specified plane. - *@param planetModel is the planet model. - *@param p is the plane. - *@param notablePoints are the points associated with the plane. - *@param bounds are any bounds which the intersection must lie within. - *@return true if there is a matching intersection. - */ - public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) { - //System.err.println(" looking for intersection between plane "+p+" and circle "+circlePlane+" on proper side of "+cutoffPlanes+" within "+bounds); - if (circlePlane == null) - return false; - return circlePlane.intersects(planetModel, p, notablePoints, this.notablePoints, bounds, this.cutoffPlanes); - } - - /** Get the bounds for a segment endpoint. - *@param planetModel is the planet model. - *@param bounds are the bounds to be modified. - */ - public void getBounds(final PlanetModel planetModel, Bounds bounds) { - bounds.addPoint(point); - if (circlePlane == null) - return; - bounds.addPlane(planetModel, circlePlane); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof SegmentEndpoint)) - return false; - SegmentEndpoint other = (SegmentEndpoint) o; - return point.equals(other.point); - } - - @Override - public int hashCode() { - return point.hashCode(); - } - - @Override - public String toString() { - return point.toString(); - } - } - - /** - * This is the pre-calculated data for a path segment. - */ - public static class PathSegment { - /** Starting point of the segment */ - public final GeoPoint start; - /** End point of the segment */ - public final GeoPoint end; - /** Place to keep any complete segment distances we've calculated so far */ - public final Map fullDistanceCache = new HashMap(); - /** Normalized plane connecting the two points and going through world center */ - public final Plane normalizedConnectingPlane; - /** Cutoff plane parallel to connecting plane representing one side of the path segment */ - public final SidedPlane upperConnectingPlane; - /** Cutoff plane parallel to connecting plane representing the other side of the path segment */ - public final SidedPlane lowerConnectingPlane; - /** Plane going through the center and start point, marking the start edge of the segment */ - public final SidedPlane startCutoffPlane; - /** Plane going through the center and end point, marking the end edge of the segment */ - public final SidedPlane endCutoffPlane; - /** Upper right hand corner of segment */ - public final GeoPoint URHC; - /** Lower right hand corner of segment */ - public final GeoPoint LRHC; - /** Upper left hand corner of segment */ - public final GeoPoint ULHC; - /** Lower left hand corner of segment */ - public final GeoPoint LLHC; - /** Notable points for the upper connecting plane */ - public final GeoPoint[] upperConnectingPlanePoints; - /** Notable points for the lower connecting plane */ - public final GeoPoint[] lowerConnectingPlanePoints; - /** Notable points for the start cutoff plane */ - public final GeoPoint[] startCutoffPlanePoints; - /** Notable points for the end cutoff plane */ - public final GeoPoint[] endCutoffPlanePoints; - - /** Construct a path segment. - *@param planetModel is the planet model. - *@param start is the starting point. - *@param end is the ending point. - *@param normalizedConnectingPlane is the connecting plane. - *@param planeBoundingOffset is the linear offset from the connecting plane to either side. - */ - public PathSegment(final PlanetModel planetModel, final GeoPoint start, final GeoPoint end, - final Plane normalizedConnectingPlane, final double planeBoundingOffset) { - this.start = start; - this.end = end; - this.normalizedConnectingPlane = normalizedConnectingPlane; - - // Either start or end should be on the correct side - upperConnectingPlane = new SidedPlane(start, normalizedConnectingPlane, -planeBoundingOffset); - lowerConnectingPlane = new SidedPlane(start, normalizedConnectingPlane, planeBoundingOffset); - // Cutoff planes use opposite endpoints as correct side examples - startCutoffPlane = new SidedPlane(end, normalizedConnectingPlane, start); - endCutoffPlane = new SidedPlane(start, normalizedConnectingPlane, end); - final Membership[] upperSide = new Membership[]{upperConnectingPlane}; - final Membership[] lowerSide = new Membership[]{lowerConnectingPlane}; - final Membership[] startSide = new Membership[]{startCutoffPlane}; - final Membership[] endSide = new Membership[]{endCutoffPlane}; - GeoPoint[] points; - points = upperConnectingPlane.findIntersections(planetModel, startCutoffPlane, lowerSide, endSide); - if (points.length == 0) { - throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); - } - this.ULHC = points[0]; - points = upperConnectingPlane.findIntersections(planetModel, endCutoffPlane, lowerSide, startSide); - if (points.length == 0) { - throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); - } - this.URHC = points[0]; - points = lowerConnectingPlane.findIntersections(planetModel, startCutoffPlane, upperSide, endSide); - if (points.length == 0) { - throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); - } - this.LLHC = points[0]; - points = lowerConnectingPlane.findIntersections(planetModel, endCutoffPlane, upperSide, startSide); - if (points.length == 0) { - throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); - } - this.LRHC = points[0]; - upperConnectingPlanePoints = new GeoPoint[]{ULHC, URHC}; - lowerConnectingPlanePoints = new GeoPoint[]{LLHC, LRHC}; - startCutoffPlanePoints = new GeoPoint[]{ULHC, LLHC}; - endCutoffPlanePoints = new GeoPoint[]{URHC, LRHC}; - } - - /** Compute the full distance along this path segment. - *@param distanceStyle is the distance style. - *@return the distance metric. - */ - public double fullPathDistance(final DistanceStyle distanceStyle) { - synchronized (fullDistanceCache) { - Double dist = fullDistanceCache.get(distanceStyle); - if (dist == null) { - dist = new Double(distanceStyle.computeDistance(start, end.x, end.y, end.z)); - fullDistanceCache.put(distanceStyle, dist); - } - return dist.doubleValue(); - } - } - - /** Check if point is within this segment. - *@param point is the point. - *@return true of within. - */ - public boolean isWithin(final Vector point) { - return startCutoffPlane.isWithin(point) && - endCutoffPlane.isWithin(point) && - upperConnectingPlane.isWithin(point) && - lowerConnectingPlane.isWithin(point); - } - - /** Check if point is within this segment. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return true of within. - */ - public boolean isWithin(final double x, final double y, final double z) { - return startCutoffPlane.isWithin(x, y, z) && - endCutoffPlane.isWithin(x, y, z) && - upperConnectingPlane.isWithin(x, y, z) && - lowerConnectingPlane.isWithin(x, y, z); - } - - /** Compute interior path distance. - *@param planetModel is the planet model. - *@param distanceStyle is the distance style. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return the distance metric. - */ - public double pathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) { - if (!isWithin(x,y,z)) - return Double.MAX_VALUE; - - // (1) Compute normalizedPerpPlane. If degenerate, then return point distance from start to point. - // Want no allocations or expensive operations! so we do this the hard way - final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y; - final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z; - final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x; - final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ); - if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION) - return distanceStyle.computeDistance(start, x,y,z); - final double normFactor = 1.0/magnitude; - final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0); - - // Old computation: too expensive, because it calculates the intersection point twice. - //return distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x, y, z, startCutoffPlane, endCutoffPlane) + - // distanceStyle.computeDistance(planetModel, normalizedPerpPlane, start.x, start.y, start.z, upperConnectingPlane, lowerConnectingPlane); - - final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane); - GeoPoint thePoint; - if (intersectionPoints.length == 0) - throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z); - else if (intersectionPoints.length == 1) - thePoint = intersectionPoints[0]; - else { - if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0])) - thePoint = intersectionPoints[0]; - else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1])) - thePoint = intersectionPoints[1]; - else - throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z); - } - return distanceStyle.computeDistance(thePoint, x, y, z) + distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z); - } - - /** Compute external distance. - *@param planetModel is the planet model. - *@param distanceStyle is the distance style. - *@param x is the point x. - *@param y is the point y. - *@param z is the point z. - *@return the distance metric. - */ - public double outsideDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) { - final double upperDistance = distanceStyle.computeDistance(planetModel, upperConnectingPlane, x,y,z, lowerConnectingPlane, startCutoffPlane, endCutoffPlane); - final double lowerDistance = distanceStyle.computeDistance(planetModel, lowerConnectingPlane, x,y,z, upperConnectingPlane, startCutoffPlane, endCutoffPlane); - final double startDistance = distanceStyle.computeDistance(planetModel, startCutoffPlane, x,y,z, endCutoffPlane, lowerConnectingPlane, upperConnectingPlane); - final double endDistance = distanceStyle.computeDistance(planetModel, endCutoffPlane, x,y,z, startCutoffPlane, lowerConnectingPlane, upperConnectingPlane); - final double ULHCDistance = distanceStyle.computeDistance(ULHC, x,y,z); - final double URHCDistance = distanceStyle.computeDistance(URHC, x,y,z); - final double LLHCDistance = distanceStyle.computeDistance(LLHC, x,y,z); - final double LRHCDistance = distanceStyle.computeDistance(LRHC, x,y,z); - return Math.min( - Math.min( - Math.min(upperDistance,lowerDistance), - Math.min(startDistance,endDistance)), - Math.min( - Math.min(ULHCDistance, URHCDistance), - Math.min(LLHCDistance, LRHCDistance))); - } - - /** Determine if this endpoint intersects a specified plane. - *@param planetModel is the planet model. - *@param p is the plane. - *@param notablePoints are the points associated with the plane. - *@param bounds are any bounds which the intersection must lie within. - *@return true if there is a matching intersection. - */ - public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) { - return upperConnectingPlane.intersects(planetModel, p, notablePoints, upperConnectingPlanePoints, bounds, lowerConnectingPlane, startCutoffPlane, endCutoffPlane) || - lowerConnectingPlane.intersects(planetModel, p, notablePoints, lowerConnectingPlanePoints, bounds, upperConnectingPlane, startCutoffPlane, endCutoffPlane); - } - - /** Get the bounds for a segment endpoint. - *@param planetModel is the planet model. - *@param bounds are the bounds to be modified. - */ - public void getBounds(final PlanetModel planetModel, Bounds bounds) { - // We need to do all bounding planes as well as corner points - bounds.addPoint(start).addPoint(end).addPoint(ULHC).addPoint(URHC).addPoint(LRHC).addPoint(LLHC); - bounds.addPlane(planetModel, upperConnectingPlane, lowerConnectingPlane, startCutoffPlane, endCutoffPlane); - bounds.addPlane(planetModel, lowerConnectingPlane, upperConnectingPlane, startCutoffPlane, endCutoffPlane); - bounds.addPlane(planetModel, startCutoffPlane, endCutoffPlane, upperConnectingPlane, lowerConnectingPlane); - bounds.addPlane(planetModel, endCutoffPlane, startCutoffPlane, upperConnectingPlane, lowerConnectingPlane); - } - - } - +public interface GeoPath extends GeoDistanceShape { } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java new file mode 100644 index 000000000000..3fd7cf7d6696 --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial3d.geom; + +/** + * Class which constructs a GeoPath representing an arbitrary path. + * + * @lucene.experimental + */ +public class GeoPathFactory { + private GeoPathFactory() { + } + + /** + * Create a GeoPath of the right kind given the specified information. + * @param planetModel is the planet model. + * @param maxCutoffAngle is the width of the path, measured as an angle. + * @param pathPoints are the points in the path. + * @return a GeoPath corresponding to what was specified. + */ + public static GeoPath makeGeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) { + return new GeoStandardPath(planetModel, maxCutoffAngle, pathPoints); + } + +} diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java new file mode 100755 index 000000000000..1546439a01d2 --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java @@ -0,0 +1,797 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial3d.geom; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * GeoShape representing a path across the surface of the globe, + * with a specified half-width. Path is described by a series of points. + * Distances are measured from the starting point along the path, and then at right + * angles to the path. + * + * @lucene.internal + */ +class GeoStandardPath extends GeoBasePath { + /** The cutoff angle (width) */ + protected final double cutoffAngle; + + /** Sine of cutoff angle */ + protected final double sinAngle; + /** Cosine of cutoff angle */ + protected final double cosAngle; + + /** The original list of path points */ + protected final List points = new ArrayList(); + + /** A list of SegmentEndpoints */ + protected List endPoints; + /** A list of PathSegments */ + protected List segments; + + /** A point on the edge */ + protected GeoPoint[] edgePoints; + + /** Set to true if path has been completely constructed */ + protected boolean isDone = false; + + /** Constructor. + *@param planetModel is the planet model. + *@param maxCutoffAngle is the width of the path, measured as an angle. + *@param pathPoints are the points in the path. + */ + public GeoStandardPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) { + this(planetModel, maxCutoffAngle); + Collections.addAll(points, pathPoints); + done(); + } + + /** Piece-wise constructor. Use in conjunction with addPoint() and done(). + *@param planetModel is the planet model. + *@param maxCutoffAngle is the width of the path, measured as an angle. + */ + public GeoStandardPath(final PlanetModel planetModel, final double maxCutoffAngle) { + super(planetModel); + if (maxCutoffAngle <= 0.0 || maxCutoffAngle > Math.PI * 0.5) + throw new IllegalArgumentException("Cutoff angle out of bounds"); + this.cutoffAngle = maxCutoffAngle; + this.cosAngle = Math.cos(maxCutoffAngle); + this.sinAngle = Math.sin(maxCutoffAngle); + } + + /** Add a point to the path. + *@param lat is the latitude of the point. + *@param lon is the longitude of the point. + */ + public void addPoint(final double lat, final double lon) { + if (isDone) + throw new IllegalStateException("Can't call addPoint() if done() already called"); + points.add(new GeoPoint(planetModel, lat, lon)); + } + + /** Complete the path. + */ + public void done() { + if (isDone) + throw new IllegalStateException("Can't call done() twice"); + if (points.size() == 0) + throw new IllegalArgumentException("Path must have at least one point"); + isDone = true; + + endPoints = new ArrayList<>(points.size()); + segments = new ArrayList<>(points.size()); + // Compute an offset to use for all segments. This will be based on the minimum magnitude of + // the entire ellipsoid. + final double cutoffOffset = this.sinAngle * planetModel.getMinimumMagnitude(); + + // First, build all segments. We'll then go back and build corresponding segment endpoints. + GeoPoint lastPoint = null; + for (final GeoPoint end : points) { + if (lastPoint != null) { + final Plane normalizedConnectingPlane = new Plane(lastPoint, end); + if (normalizedConnectingPlane == null) { + continue; + } + segments.add(new PathSegment(planetModel, lastPoint, end, normalizedConnectingPlane, cutoffOffset)); + } + lastPoint = end; + } + + if (segments.size() == 0) { + // Simple circle + double lat = points.get(0).getLatitude(); + double lon = points.get(0).getLongitude(); + // Compute two points on the circle, with the right angle from the center. We'll use these + // to obtain the perpendicular plane to the circle. + double upperLat = lat + cutoffAngle; + double upperLon = lon; + if (upperLat > Math.PI * 0.5) { + upperLon += Math.PI; + if (upperLon > Math.PI) + upperLon -= 2.0 * Math.PI; + upperLat = Math.PI - upperLat; + } + double lowerLat = lat - cutoffAngle; + double lowerLon = lon; + if (lowerLat < -Math.PI * 0.5) { + lowerLon += Math.PI; + if (lowerLon > Math.PI) + lowerLon -= 2.0 * Math.PI; + lowerLat = -Math.PI - lowerLat; + } + final GeoPoint upperPoint = new GeoPoint(planetModel, upperLat, upperLon); + final GeoPoint lowerPoint = new GeoPoint(planetModel, lowerLat, lowerLon); + final GeoPoint point = points.get(0); + + // Construct normal plane + final Plane normalPlane = Plane.constructNormalizedZPlane(upperPoint, lowerPoint, point); + + final SegmentEndpoint onlyEndpoint = new SegmentEndpoint(point, normalPlane, upperPoint, lowerPoint); + endPoints.add(onlyEndpoint); + this.edgePoints = new GeoPoint[]{onlyEndpoint.circlePlane.getSampleIntersectionPoint(planetModel, normalPlane)}; + return; + } + + // Create segment endpoints. Use an appropriate constructor for the start and end of the path. + for (int i = 0; i < segments.size(); i++) { + final PathSegment currentSegment = segments.get(i); + + if (i == 0) { + // Starting endpoint + final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start, + currentSegment.startCutoffPlane, currentSegment.ULHC, currentSegment.LLHC); + endPoints.add(startEndpoint); + this.edgePoints = new GeoPoint[]{currentSegment.ULHC}; + continue; + } + + // General intersection case + final PathSegment prevSegment = segments.get(i-1); + // We construct four separate planes, and evaluate which one includes all interior points with least overlap + final SidedPlane candidate1 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, prevSegment.URHC, currentSegment.ULHC, currentSegment.LLHC); + final SidedPlane candidate2 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, currentSegment.ULHC, currentSegment.LLHC, prevSegment.LRHC); + final SidedPlane candidate3 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, currentSegment.LLHC, prevSegment.LRHC, prevSegment.URHC); + final SidedPlane candidate4 = SidedPlane.constructNormalizedThreePointSidedPlane(currentSegment.start, prevSegment.LRHC, prevSegment.URHC, currentSegment.ULHC); + + if (candidate1 == null && candidate2 == null && candidate3 == null && candidate4 == null) { + // The planes are identical. We wouldn't need a circle at all except for the possibility of + // backing up, which is hard to detect here. + final SegmentEndpoint midEndpoint = new SegmentEndpoint(currentSegment.start, + prevSegment.endCutoffPlane, currentSegment.startCutoffPlane, currentSegment.ULHC, currentSegment.LLHC); + //don't need a circle at all. Special constructor... + endPoints.add(midEndpoint); + } else { + endPoints.add(new SegmentEndpoint(currentSegment.start, + prevSegment.endCutoffPlane, currentSegment.startCutoffPlane, + prevSegment.URHC, prevSegment.LRHC, + currentSegment.ULHC, currentSegment.LLHC, + candidate1, candidate2, candidate3, candidate4)); + } + } + // Do final endpoint + final PathSegment lastSegment = segments.get(segments.size()-1); + endPoints.add(new SegmentEndpoint(lastSegment.end, + lastSegment.endCutoffPlane, lastSegment.URHC, lastSegment.LRHC)); + + } + + @Override + protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { + // Algorithm: + // (1) If the point is within any of the segments along the path, return that value. + // (2) If the point is within any of the segment end circles along the path, return that value. + double currentDistance = 0.0; + for (PathSegment segment : segments) { + double distance = segment.pathDistance(planetModel, distanceStyle, x,y,z); + if (distance != Double.MAX_VALUE) + return currentDistance + distance; + currentDistance += segment.fullPathDistance(distanceStyle); + } + + int segmentIndex = 0; + currentDistance = 0.0; + for (SegmentEndpoint endpoint : endPoints) { + double distance = endpoint.pathDistance(distanceStyle, x, y, z); + if (distance != Double.MAX_VALUE) + return currentDistance + distance; + if (segmentIndex < segments.size()) + currentDistance += segments.get(segmentIndex++).fullPathDistance(distanceStyle); + } + + return Double.MAX_VALUE; + } + + @Override + protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { + double minDistance = Double.MAX_VALUE; + for (final SegmentEndpoint endpoint : endPoints) { + final double newDistance = endpoint.outsideDistance(distanceStyle, x,y,z); + if (newDistance < minDistance) + minDistance = newDistance; + } + for (final PathSegment segment : segments) { + final double newDistance = segment.outsideDistance(planetModel, distanceStyle, x, y, z); + if (newDistance < minDistance) + minDistance = newDistance; + } + return minDistance; + } + + @Override + public boolean isWithin(final double x, final double y, final double z) { + for (SegmentEndpoint pathPoint : endPoints) { + if (pathPoint.isWithin(x, y, z)) + return true; + } + for (PathSegment pathSegment : segments) { + if (pathSegment.isWithin(x, y, z)) + return true; + } + return false; + } + + @Override + public GeoPoint[] getEdgePoints() { + return edgePoints; + } + + @Override + public boolean intersects(final Plane plane, final GeoPoint[] notablePoints, final Membership... bounds) { + // We look for an intersection with any of the exterior edges of the path. + // We also have to look for intersections with the cones described by the endpoints. + // Return "true" if any such intersections are found. + + // For plane intersections, the basic idea is to come up with an equation of the line that is + // the intersection (if any). Then, find the intersections with the unit sphere (if any). If + // any of the intersection points are within the bounds, then we've detected an intersection. + // Well, sort of. We can detect intersections also due to overlap of segments with each other. + // But that's an edge case and we won't be optimizing for it. + //System.err.println(" Looking for intersection of plane "+plane+" with path "+this); + for (final SegmentEndpoint pathPoint : endPoints) { + if (pathPoint.intersects(planetModel, plane, notablePoints, bounds)) { + return true; + } + } + + for (final PathSegment pathSegment : segments) { + if (pathSegment.intersects(planetModel, plane, notablePoints, bounds)) { + return true; + } + } + + return false; + } + + @Override + public void getBounds(Bounds bounds) { + super.getBounds(bounds); + // For building bounds, order matters. We want to traverse + // never more than 180 degrees longitude at a pop or we risk having the + // bounds object get itself inverted. So do the edges first. + for (PathSegment pathSegment : segments) { + pathSegment.getBounds(planetModel, bounds); + } + for (SegmentEndpoint pathPoint : endPoints) { + pathPoint.getBounds(planetModel, bounds); + } + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof GeoStandardPath)) + return false; + GeoStandardPath p = (GeoStandardPath) o; + if (!super.equals(p)) + return false; + if (cutoffAngle != p.cutoffAngle) + return false; + return points.equals(p.points); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + long temp = Double.doubleToLongBits(cutoffAngle); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + points.hashCode(); + return result; + } + + @Override + public String toString() { + return "GeoStandardPath: {planetmodel=" + planetModel+", width=" + cutoffAngle + "(" + cutoffAngle * 180.0 / Math.PI + "), points={" + points + "}}"; + } + + /** + * This is precalculated data for segment endpoint. + * Note well: This is not necessarily a circle. There are four cases: + * (1) The path consists of a single endpoint. In this case, we build a simple circle with the proper cutoff offset. + * (2) This is the end of a path. The circle plane must be constructed to go through two supplied points and be perpendicular to a connecting plane. + * (2.5) Intersection, but the path on both sides is linear. We generate a circle, but we use the cutoff planes to limit its influence in the straight line case. + * (3) This is an intersection in a path. We are supplied FOUR planes. If there are intersections within bounds for both upper and lower, then + * we generate no circle at all. If there is one intersection only, then we generate a plane that includes that intersection, as well as the remaining + * cutoff plane/edge plane points. + */ + public static class SegmentEndpoint { + /** The center point of the endpoint */ + public final GeoPoint point; + /** A plane describing the circle */ + public final SidedPlane circlePlane; + /** Pertinent cutoff planes from adjoining segments */ + public final Membership[] cutoffPlanes; + /** Notable points for this segment endpoint */ + public final GeoPoint[] notablePoints; + /** No notable points from the circle itself */ + public final static GeoPoint[] circlePoints = new GeoPoint[0]; + /** Null membership */ + public final static Membership[] NO_MEMBERSHIP = new Membership[0]; + + /** Base case. Does nothing at all. + */ + public SegmentEndpoint(final GeoPoint point) { + this.point = point; + this.circlePlane = null; + this.cutoffPlanes = null; + this.notablePoints = null; + } + + /** Constructor for case (1). + * Generate a simple circle cutoff plane. + *@param point is the center point. + *@param upperPoint is a point that must be on the circle plane. + *@param lowerPoint is another point that must be on the circle plane. + */ + public SegmentEndpoint(final GeoPoint point, final Plane normalPlane, final GeoPoint upperPoint, final GeoPoint lowerPoint) { + this.point = point; + // Construct a sided plane that goes through the two points and whose normal is in the normalPlane. + this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, normalPlane, upperPoint, lowerPoint); + this.cutoffPlanes = NO_MEMBERSHIP; + this.notablePoints = circlePoints; + } + + /** Constructor for case (2). + * Generate an endpoint, given a single cutoff plane plus upper and lower edge points. + *@param point is the center point. + *@param cutoffPlane is the plane from the adjoining path segment marking the boundary between this endpoint and that segment. + *@param topEdgePoint is a point on the cutoffPlane that should be also on the circle plane. + *@param bottomEdgePoint is another point on the cutoffPlane that should be also on the circle plane. + */ + public SegmentEndpoint(final GeoPoint point, + final SidedPlane cutoffPlane, final GeoPoint topEdgePoint, final GeoPoint bottomEdgePoint) { + this.point = point; + this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane)}; + this.notablePoints = new GeoPoint[]{topEdgePoint, bottomEdgePoint}; + // To construct the plane, we now just need D, which is simply the negative of the evaluation of the circle normal vector at one of the points. + this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, cutoffPlane, topEdgePoint, bottomEdgePoint); + } + + /** Constructor for case (2.5). + * Generate an endpoint, given two cutoff planes plus upper and lower edge points. + *@param point is the center. + *@param cutoffPlane1 is one adjoining path segment cutoff plane. + *@param cutoffPlane2 is another adjoining path segment cutoff plane. + *@param topEdgePoint is a point on the cutoffPlane that should be also on the circle plane. + *@param bottomEdgePoint is another point on the cutoffPlane that should be also on the circle plane. + */ + public SegmentEndpoint(final GeoPoint point, + final SidedPlane cutoffPlane1, final SidedPlane cutoffPlane2, final GeoPoint topEdgePoint, final GeoPoint bottomEdgePoint) { + this.point = point; + this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane1), new SidedPlane(cutoffPlane2)}; + this.notablePoints = new GeoPoint[]{topEdgePoint, bottomEdgePoint}; + // To construct the plane, we now just need D, which is simply the negative of the evaluation of the circle normal vector at one of the points. + this.circlePlane = SidedPlane.constructNormalizedPerpendicularSidedPlane(point, cutoffPlane1, topEdgePoint, bottomEdgePoint); + } + + /** Constructor for case (3). + * Generate an endpoint for an intersection, given four points. + *@param point is the center. + *@param prevCutoffPlane is the previous adjoining segment cutoff plane. + *@param nextCutoffPlane is the next path segment cutoff plane. + *@param notCand2Point is a point NOT on candidate2. + *@param notCand1Point is a point NOT on candidate1. + *@param notCand3Point is a point NOT on candidate3. + *@param notCand4Point is a point NOT on candidate4. + *@param candidate1 one of four candidate circle planes. + *@param candidate2 one of four candidate circle planes. + *@param candidate3 one of four candidate circle planes. + *@param candidate4 one of four candidate circle planes. + */ + public SegmentEndpoint(final GeoPoint point, + final SidedPlane prevCutoffPlane, final SidedPlane nextCutoffPlane, + final GeoPoint notCand2Point, final GeoPoint notCand1Point, + final GeoPoint notCand3Point, final GeoPoint notCand4Point, + final SidedPlane candidate1, final SidedPlane candidate2, final SidedPlane candidate3, final SidedPlane candidate4) { + // Note: What we really need is a single plane that goes through all four points. + // Since that's not possible in the ellipsoid case (because three points determine a plane, not four), we + // need an approximation that at least creates a boundary that has no interruptions. + // There are three obvious choices for the third point: either (a) one of the two remaining points, or (b) the top or bottom edge + // intersection point. (a) has no guarantee of continuity, while (b) is capable of producing something very far from a circle if + // the angle between segments is acute. + // The solution is to look for the side (top or bottom) that has an intersection within the shape. We use the two points from + // the opposite side to determine the plane, AND we pick the third to be either of the two points on the intersecting side + // PROVIDED that the other point is within the final circle we come up with. + this.point = point; + + // We construct four separate planes, and evaluate which one includes all interior points with least overlap + // (Constructed beforehand because we need them for degeneracy check) + + final boolean cand1IsOtherWithin = candidate1!=null?candidate1.isWithin(notCand1Point):false; + final boolean cand2IsOtherWithin = candidate2!=null?candidate2.isWithin(notCand2Point):false; + final boolean cand3IsOtherWithin = candidate3!=null?candidate3.isWithin(notCand3Point):false; + final boolean cand4IsOtherWithin = candidate4!=null?candidate4.isWithin(notCand4Point):false; + + if (cand1IsOtherWithin && cand2IsOtherWithin && cand3IsOtherWithin && cand4IsOtherWithin) { + // The only way we should see both within is if all four points are coplanar. In that case, we default to the simplest treatment. + this.circlePlane = candidate1; // doesn't matter which + this.notablePoints = new GeoPoint[]{notCand2Point, notCand3Point, notCand1Point, notCand4Point}; + this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane), new SidedPlane(nextCutoffPlane)}; + } else if (cand1IsOtherWithin) { + // Use candidate1, and DON'T include prevCutoffPlane in the cutoff planes list + this.circlePlane = candidate1; + this.notablePoints = new GeoPoint[]{notCand2Point, notCand3Point, notCand4Point}; + this.cutoffPlanes = new Membership[]{new SidedPlane(nextCutoffPlane)}; + } else if (cand2IsOtherWithin) { + // Use candidate2 + this.circlePlane = candidate2; + this.notablePoints = new GeoPoint[]{notCand3Point, notCand4Point, notCand1Point}; + this.cutoffPlanes = new Membership[]{new SidedPlane(nextCutoffPlane)}; + } else if (cand3IsOtherWithin) { + this.circlePlane = candidate3; + this.notablePoints = new GeoPoint[]{notCand4Point, notCand1Point, notCand2Point}; + this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane)}; + } else if (cand4IsOtherWithin) { + this.circlePlane = candidate4; + this.notablePoints = new GeoPoint[]{notCand1Point, notCand2Point, notCand3Point}; + this.cutoffPlanes = new Membership[]{new SidedPlane(prevCutoffPlane)}; + } else { + // dunno what happened + throw new RuntimeException("Couldn't come up with a plane through three points that included the fourth"); + } + } + + /** Check if point is within this endpoint. + *@param point is the point. + *@return true of within. + */ + public boolean isWithin(final Vector point) { + if (circlePlane == null) + return false; + if (!circlePlane.isWithin(point)) + return false; + for (final Membership m : cutoffPlanes) { + if (!m.isWithin(point)) { + return false; + } + } + return true; + } + + /** Check if point is within this endpoint. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return true of within. + */ + public boolean isWithin(final double x, final double y, final double z) { + if (circlePlane == null) + return false; + if (!circlePlane.isWithin(x, y, z)) + return false; + for (final Membership m : cutoffPlanes) { + if (!m.isWithin(x,y,z)) { + return false; + } + } + return true; + } + + /** Compute interior path distance. + *@param distanceStyle is the distance style. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return the distance metric. + */ + public double pathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { + if (!isWithin(x,y,z)) + return Double.MAX_VALUE; + return distanceStyle.computeDistance(this.point, x, y, z); + } + + /** Compute external distance. + *@param distanceStyle is the distance style. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return the distance metric. + */ + public double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) { + return distanceStyle.computeDistance(this.point, x, y, z); + } + + /** Determine if this endpoint intersects a specified plane. + *@param planetModel is the planet model. + *@param p is the plane. + *@param notablePoints are the points associated with the plane. + *@param bounds are any bounds which the intersection must lie within. + *@return true if there is a matching intersection. + */ + public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) { + //System.err.println(" looking for intersection between plane "+p+" and circle "+circlePlane+" on proper side of "+cutoffPlanes+" within "+bounds); + if (circlePlane == null) + return false; + return circlePlane.intersects(planetModel, p, notablePoints, this.notablePoints, bounds, this.cutoffPlanes); + } + + /** Get the bounds for a segment endpoint. + *@param planetModel is the planet model. + *@param bounds are the bounds to be modified. + */ + public void getBounds(final PlanetModel planetModel, Bounds bounds) { + bounds.addPoint(point); + if (circlePlane == null) + return; + bounds.addPlane(planetModel, circlePlane); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SegmentEndpoint)) + return false; + SegmentEndpoint other = (SegmentEndpoint) o; + return point.equals(other.point); + } + + @Override + public int hashCode() { + return point.hashCode(); + } + + @Override + public String toString() { + return point.toString(); + } + } + + /** + * This is the pre-calculated data for a path segment. + */ + public static class PathSegment { + /** Starting point of the segment */ + public final GeoPoint start; + /** End point of the segment */ + public final GeoPoint end; + /** Place to keep any complete segment distances we've calculated so far */ + public final Map fullDistanceCache = new HashMap(); + /** Normalized plane connecting the two points and going through world center */ + public final Plane normalizedConnectingPlane; + /** Cutoff plane parallel to connecting plane representing one side of the path segment */ + public final SidedPlane upperConnectingPlane; + /** Cutoff plane parallel to connecting plane representing the other side of the path segment */ + public final SidedPlane lowerConnectingPlane; + /** Plane going through the center and start point, marking the start edge of the segment */ + public final SidedPlane startCutoffPlane; + /** Plane going through the center and end point, marking the end edge of the segment */ + public final SidedPlane endCutoffPlane; + /** Upper right hand corner of segment */ + public final GeoPoint URHC; + /** Lower right hand corner of segment */ + public final GeoPoint LRHC; + /** Upper left hand corner of segment */ + public final GeoPoint ULHC; + /** Lower left hand corner of segment */ + public final GeoPoint LLHC; + /** Notable points for the upper connecting plane */ + public final GeoPoint[] upperConnectingPlanePoints; + /** Notable points for the lower connecting plane */ + public final GeoPoint[] lowerConnectingPlanePoints; + /** Notable points for the start cutoff plane */ + public final GeoPoint[] startCutoffPlanePoints; + /** Notable points for the end cutoff plane */ + public final GeoPoint[] endCutoffPlanePoints; + + /** Construct a path segment. + *@param planetModel is the planet model. + *@param start is the starting point. + *@param end is the ending point. + *@param normalizedConnectingPlane is the connecting plane. + *@param planeBoundingOffset is the linear offset from the connecting plane to either side. + */ + public PathSegment(final PlanetModel planetModel, final GeoPoint start, final GeoPoint end, + final Plane normalizedConnectingPlane, final double planeBoundingOffset) { + this.start = start; + this.end = end; + this.normalizedConnectingPlane = normalizedConnectingPlane; + + // Either start or end should be on the correct side + upperConnectingPlane = new SidedPlane(start, normalizedConnectingPlane, -planeBoundingOffset); + lowerConnectingPlane = new SidedPlane(start, normalizedConnectingPlane, planeBoundingOffset); + // Cutoff planes use opposite endpoints as correct side examples + startCutoffPlane = new SidedPlane(end, normalizedConnectingPlane, start); + endCutoffPlane = new SidedPlane(start, normalizedConnectingPlane, end); + final Membership[] upperSide = new Membership[]{upperConnectingPlane}; + final Membership[] lowerSide = new Membership[]{lowerConnectingPlane}; + final Membership[] startSide = new Membership[]{startCutoffPlane}; + final Membership[] endSide = new Membership[]{endCutoffPlane}; + GeoPoint[] points; + points = upperConnectingPlane.findIntersections(planetModel, startCutoffPlane, lowerSide, endSide); + if (points.length == 0) { + throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); + } + this.ULHC = points[0]; + points = upperConnectingPlane.findIntersections(planetModel, endCutoffPlane, lowerSide, startSide); + if (points.length == 0) { + throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); + } + this.URHC = points[0]; + points = lowerConnectingPlane.findIntersections(planetModel, startCutoffPlane, upperSide, endSide); + if (points.length == 0) { + throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); + } + this.LLHC = points[0]; + points = lowerConnectingPlane.findIntersections(planetModel, endCutoffPlane, upperSide, startSide); + if (points.length == 0) { + throw new IllegalArgumentException("Some segment boundary points are off the ellipsoid; path too wide"); + } + this.LRHC = points[0]; + upperConnectingPlanePoints = new GeoPoint[]{ULHC, URHC}; + lowerConnectingPlanePoints = new GeoPoint[]{LLHC, LRHC}; + startCutoffPlanePoints = new GeoPoint[]{ULHC, LLHC}; + endCutoffPlanePoints = new GeoPoint[]{URHC, LRHC}; + } + + /** Compute the full distance along this path segment. + *@param distanceStyle is the distance style. + *@return the distance metric. + */ + public double fullPathDistance(final DistanceStyle distanceStyle) { + synchronized (fullDistanceCache) { + Double dist = fullDistanceCache.get(distanceStyle); + if (dist == null) { + dist = new Double(distanceStyle.computeDistance(start, end.x, end.y, end.z)); + fullDistanceCache.put(distanceStyle, dist); + } + return dist.doubleValue(); + } + } + + /** Check if point is within this segment. + *@param point is the point. + *@return true of within. + */ + public boolean isWithin(final Vector point) { + return startCutoffPlane.isWithin(point) && + endCutoffPlane.isWithin(point) && + upperConnectingPlane.isWithin(point) && + lowerConnectingPlane.isWithin(point); + } + + /** Check if point is within this segment. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return true of within. + */ + public boolean isWithin(final double x, final double y, final double z) { + return startCutoffPlane.isWithin(x, y, z) && + endCutoffPlane.isWithin(x, y, z) && + upperConnectingPlane.isWithin(x, y, z) && + lowerConnectingPlane.isWithin(x, y, z); + } + + /** Compute interior path distance. + *@param planetModel is the planet model. + *@param distanceStyle is the distance style. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return the distance metric. + */ + public double pathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) { + if (!isWithin(x,y,z)) + return Double.MAX_VALUE; + + // (1) Compute normalizedPerpPlane. If degenerate, then return point distance from start to point. + // Want no allocations or expensive operations! so we do this the hard way + final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y; + final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z; + final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x; + final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ); + if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION) + return distanceStyle.computeDistance(start, x,y,z); + final double normFactor = 1.0/magnitude; + final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0); + + // Old computation: too expensive, because it calculates the intersection point twice. + //return distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x, y, z, startCutoffPlane, endCutoffPlane) + + // distanceStyle.computeDistance(planetModel, normalizedPerpPlane, start.x, start.y, start.z, upperConnectingPlane, lowerConnectingPlane); + + final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane); + GeoPoint thePoint; + if (intersectionPoints.length == 0) + throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z); + else if (intersectionPoints.length == 1) + thePoint = intersectionPoints[0]; + else { + if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0])) + thePoint = intersectionPoints[0]; + else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1])) + thePoint = intersectionPoints[1]; + else + throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z); + } + return distanceStyle.computeDistance(thePoint, x, y, z) + distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z); + } + + /** Compute external distance. + *@param planetModel is the planet model. + *@param distanceStyle is the distance style. + *@param x is the point x. + *@param y is the point y. + *@param z is the point z. + *@return the distance metric. + */ + public double outsideDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) { + final double upperDistance = distanceStyle.computeDistance(planetModel, upperConnectingPlane, x,y,z, lowerConnectingPlane, startCutoffPlane, endCutoffPlane); + final double lowerDistance = distanceStyle.computeDistance(planetModel, lowerConnectingPlane, x,y,z, upperConnectingPlane, startCutoffPlane, endCutoffPlane); + final double startDistance = distanceStyle.computeDistance(planetModel, startCutoffPlane, x,y,z, endCutoffPlane, lowerConnectingPlane, upperConnectingPlane); + final double endDistance = distanceStyle.computeDistance(planetModel, endCutoffPlane, x,y,z, startCutoffPlane, lowerConnectingPlane, upperConnectingPlane); + final double ULHCDistance = distanceStyle.computeDistance(ULHC, x,y,z); + final double URHCDistance = distanceStyle.computeDistance(URHC, x,y,z); + final double LLHCDistance = distanceStyle.computeDistance(LLHC, x,y,z); + final double LRHCDistance = distanceStyle.computeDistance(LRHC, x,y,z); + return Math.min( + Math.min( + Math.min(upperDistance,lowerDistance), + Math.min(startDistance,endDistance)), + Math.min( + Math.min(ULHCDistance, URHCDistance), + Math.min(LLHCDistance, LRHCDistance))); + } + + /** Determine if this endpoint intersects a specified plane. + *@param planetModel is the planet model. + *@param p is the plane. + *@param notablePoints are the points associated with the plane. + *@param bounds are any bounds which the intersection must lie within. + *@return true if there is a matching intersection. + */ + public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) { + return upperConnectingPlane.intersects(planetModel, p, notablePoints, upperConnectingPlanePoints, bounds, lowerConnectingPlane, startCutoffPlane, endCutoffPlane) || + lowerConnectingPlane.intersects(planetModel, p, notablePoints, lowerConnectingPlanePoints, bounds, upperConnectingPlane, startCutoffPlane, endCutoffPlane); + } + + /** Get the bounds for a segment endpoint. + *@param planetModel is the planet model. + *@param bounds are the bounds to be modified. + */ + public void getBounds(final PlanetModel planetModel, Bounds bounds) { + // We need to do all bounding planes as well as corner points + bounds.addPoint(start).addPoint(end).addPoint(ULHC).addPoint(URHC).addPoint(LRHC).addPoint(LLHC); + bounds.addPlane(planetModel, upperConnectingPlane, lowerConnectingPlane, startCutoffPlane, endCutoffPlane); + bounds.addPlane(planetModel, lowerConnectingPlane, upperConnectingPlane, startCutoffPlane, endCutoffPlane); + bounds.addPlane(planetModel, startCutoffPlane, endCutoffPlane, upperConnectingPlane, lowerConnectingPlane); + bounds.addPlane(planetModel, endCutoffPlane, startCutoffPlane, upperConnectingPlane, lowerConnectingPlane); + } + + } + +} diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 398458c907f4..3caf039a2d01 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -40,7 +40,7 @@ import org.apache.lucene.spatial3d.geom.GeoAreaFactory; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoCircleFactory; -import org.apache.lucene.spatial3d.geom.GeoPath; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; import org.apache.lucene.spatial3d.geom.GeoShape; @@ -625,13 +625,12 @@ private static GeoShape randomShape(PlanetModel planetModel) { // Paths final int pointCount = random().nextInt(5) + 1; final double width = toRadians(random().nextInt(89)+1); + final GeoPoint[] points = new GeoPoint[pointCount]; + for (int i = 0; i < pointCount; i++) { + points[i] = new GeoPoint(planetModel, toRadians(randomLat()), toRadians(randomLon())); + } try { - final GeoPath path = new GeoPath(planetModel, width); - for (int i = 0; i < pointCount; i++) { - path.addPoint(toRadians(randomLat()), toRadians(randomLon())); - } - path.done(); - return path; + return GeoPathFactory.makeGeoPath(planetModel, width, points); } catch (IllegalArgumentException e) { // This is what happens when we create a shape that is invalid. Although it is conceivable that there are cases where // the exception is thrown incorrectly, we aren't going to be able to do that in this random test. diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java index 37460699359e..e68e3f48779a 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java @@ -28,9 +28,9 @@ public class GeoPathTest { @Test public void testPathDistance() { // Start with a really simple case - GeoPath p; + GeoStandardPath p; GeoPoint gp; - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); p.addPoint(0.0, 0.0); p.addPoint(0.0, 0.1); p.addPoint(0.0, 0.2); @@ -49,7 +49,7 @@ public void testPathDistance() { assertEquals(0.0 + 0.05, p.computeDistance(DistanceStyle.ARC,gp), 0.000001); // Compute path distances now - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); p.addPoint(0.0, 0.0); p.addPoint(0.0, 0.1); p.addPoint(0.0, 0.2); @@ -60,7 +60,7 @@ public void testPathDistance() { assertEquals(0.12, p.computeDistance(DistanceStyle.ARC,gp), 0.000001); // Now try a vertical path, and make sure distances are as expected - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); p.addPoint(-Math.PI * 0.25, -0.5); p.addPoint(Math.PI * 0.25, -0.5); p.done(); @@ -77,9 +77,9 @@ public void testPathDistance() { @Test public void testPathPointWithin() { // Tests whether we can properly detect whether a point is within a path or not - GeoPath p; + GeoStandardPath p; GeoPoint gp; - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); // Build a diagonal path crossing the equator p.addPoint(-0.2, -0.2); p.addPoint(0.2, 0.2); @@ -101,7 +101,7 @@ public void testPathPointWithin() { gp = new GeoPoint(PlanetModel.SPHERE, 0.0, Math.PI); assertFalse(p.isWithin(gp)); // Repeat the test, but across the terminator - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); // Build a diagonal path crossing the equator p.addPoint(-0.2, Math.PI - 0.2); p.addPoint(0.2, -Math.PI + 0.2); @@ -128,8 +128,8 @@ public void testPathPointWithin() { @Test public void testGetRelationship() { GeoArea rect; - GeoPath p; - GeoPath c; + GeoStandardPath p; + GeoStandardPath c; GeoPoint point; GeoPoint pointApprox; int relationship; @@ -137,7 +137,7 @@ public void testGetRelationship() { PlanetModel planetModel; planetModel = new PlanetModel(1.151145876105594, 0.8488541238944061); - c = new GeoPath(planetModel, 0.008726646259971648); + c = new GeoStandardPath(planetModel, 0.008726646259971648); c.addPoint(-0.6925658899376476, 0.6316613927914589); c.addPoint(0.27828548161836364, 0.6785795524104564); c.done(); @@ -148,7 +148,7 @@ public void testGetRelationship() { // Start by testing the basic kinds of relationship, increasing in order of difficulty. - p = new GeoPath(PlanetModel.SPHERE, 0.1); + p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); p.addPoint(-0.3, -0.3); p.addPoint(0.3, 0.3); p.done(); @@ -179,7 +179,7 @@ public void testGetRelationship() { @Test public void testPathBounds() { - GeoPath c; + GeoStandardPath c; LatLonBounds b; XYZBounds xyzb; GeoPoint point; @@ -188,7 +188,7 @@ public void testPathBounds() { PlanetModel planetModel; planetModel = new PlanetModel(0.751521665790406,1.248478334209594); - c = new GeoPath(planetModel, 0.7504915783575618); + c = new GeoStandardPath(planetModel, 0.7504915783575618); c.addPoint(0.10869761172400265, 0.08895880215465272); c.addPoint(0.22467878641991612, 0.10972973084229565); c.addPoint(-0.7398772468744732, -0.4465812941383364); @@ -202,10 +202,10 @@ public void testPathBounds() { relationship = area.getRelationship(c); assertTrue(relationship == GeoArea.WITHIN || relationship == GeoArea.OVERLAPS); assertTrue(area.isWithin(point)); - // No longer true due to fixed GeoPath waypoints. + // No longer true due to fixed GeoStandardPath waypoints. //assertTrue(c.isWithin(point)); - c = new GeoPath(PlanetModel.WGS84, 0.6894050545377601); + c = new GeoStandardPath(PlanetModel.WGS84, 0.6894050545377601); c.addPoint(-0.0788176065762948, 0.9431251741731624); c.addPoint(0.510387871458147, 0.5327078872484678); c.addPoint(-0.5624521609859962, 1.5398841746888388); @@ -224,7 +224,7 @@ public void testPathBounds() { assertTrue(relationship == GeoArea.WITHIN || relationship == GeoArea.OVERLAPS); assertTrue(area.isWithin(point)); - c = new GeoPath(PlanetModel.WGS84, 0.7766715171374766); + c = new GeoStandardPath(PlanetModel.WGS84, 0.7766715171374766); c.addPoint(-0.2751718361148076, -0.7786721269011477); c.addPoint(0.5728375851539309, -1.2700115736820465); c.done(); @@ -240,7 +240,7 @@ public void testPathBounds() { assertTrue(relationship == GeoArea.WITHIN || relationship == GeoArea.OVERLAPS); assertTrue(area.isWithin(point)); - c = new GeoPath(PlanetModel.SPHERE, 0.1); + c = new GeoStandardPath(PlanetModel.SPHERE, 0.1); c.addPoint(-0.3, -0.3); c.addPoint(0.3, 0.3); c.done(); @@ -260,7 +260,7 @@ public void testPathBounds() { @Test public void testCoLinear() { // p1: (12,-90), p2: (11, -55), (129, -90) - GeoPath p = new GeoPath(PlanetModel.SPHERE, 0.1); + GeoStandardPath p = new GeoStandardPath(PlanetModel.SPHERE, 0.1); p.addPoint(toRadians(-90), toRadians(12));//south pole p.addPoint(toRadians(-55), toRadians(11)); p.addPoint(toRadians(-90), toRadians(129));//south pole again From d67ec549320ae1f12981797fa5c64eb62d401e86 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 5 Apr 2016 22:45:36 +0530 Subject: [PATCH 0281/1113] SOLR-8750: replace anonymous inner class for callable, Runnable etc --- .../solr/analytics/expression/Expression.java | 14 +-- .../java/org/apache/solr/hadoop/GoLive.java | 27 +++--- .../solr/hadoop/MapReduceIndexerTool.java | 7 +- .../solr/hadoop/ZooKeeperInspector.java | 9 +- .../solr/SolrMorphlineZkAvroTest.java | 14 +-- .../OverseerAutoReplicaFailoverThread.java | 8 +- .../apache/solr/core/ConfigSetService.java | 9 +- .../org/apache/solr/core/CoreContainer.java | 35 +++---- .../solr/core/HdfsDirectoryFactory.java | 21 +--- .../java/org/apache/solr/core/SolrCore.java | 83 ++++++---------- .../java/org/apache/solr/core/SolrCores.java | 25 +++-- .../solr/handler/CdcrReplicatorScheduler.java | 19 ++-- .../solr/handler/MoreLikeThisHandler.java | 12 +-- .../handler/component/HttpShardHandler.java | 97 +++++++++---------- .../PerSegmentSingleValuedFaceting.java | 9 +- .../org/apache/solr/request/SimpleFacets.java | 35 +++---- .../solr/schema/AbstractSpatialFieldType.java | 7 +- .../solr/update/SolrCmdDistributor.java | 11 +-- .../org/apache/solr/search/TestRecovery.java | 7 +- .../apache/solr/search/TestRecoveryHdfs.java | 13 +-- .../solr/update/AddBlockUpdateTest.java | 20 ++-- .../org/apache/solr/update/TestUpdate.java | 16 +-- .../client/solrj/impl/CloudSolrClient.java | 7 +- .../client/solrj/impl/HttpSolrClient.java | 7 +- .../client/solrj/io/comp/FieldComparator.java | 42 ++++---- .../solr/cloud/MiniSolrCloudCluster.java | 14 +-- 26 files changed, 204 insertions(+), 364 deletions(-) diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java index 4c9bd585d95e..ba26d9adfafc 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/expression/Expression.java @@ -28,15 +28,11 @@ public abstract class Expression { public abstract Comparable getValue(); public Comparator comparator(final FacetSortDirection direction) { - return new Comparator(){ - @SuppressWarnings("unchecked") - @Override - public int compare(Expression a, Expression b) { - if( direction == FacetSortDirection.ASCENDING ){ - return a.getValue().compareTo(b.getValue()); - } else { - return b.getValue().compareTo(a.getValue()); - } + return (a, b) -> { + if( direction == FacetSortDirection.ASCENDING ){ + return a.getValue().compareTo(b.getValue()); + } else { + return b.getValue().compareTo(a.getValue()); } }; } diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java index 103e3ee239e1..bd48c643623d 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java @@ -87,22 +87,19 @@ public boolean goLive(Options options, FileStatus[] outDirs) { baseUrl = baseUrl.substring(0, lastPathIndex); final String mergeUrl = baseUrl; - Callable task = new Callable() { - @Override - public Request call() { - Request req = new Request(); - LOG.info("Live merge " + dir.getPath() + " into " + mergeUrl); - try (final HttpSolrClient client = new HttpSolrClient(mergeUrl)) { - CoreAdminRequest.MergeIndexes mergeRequest = new CoreAdminRequest.MergeIndexes(); - mergeRequest.setCoreName(name); - mergeRequest.setIndexDirs(Arrays.asList(dir.getPath().toString() + "/data/index")); - mergeRequest.process(client); - req.success = true; - } catch (SolrServerException | IOException e) { - req.e = e; - } - return req; + Callable task = () -> { + Request req = new Request(); + LOG.info("Live merge " + dir.getPath() + " into " + mergeUrl); + try (final HttpSolrClient client = new HttpSolrClient(mergeUrl)) { + CoreAdminRequest.MergeIndexes mergeRequest = new CoreAdminRequest.MergeIndexes(); + mergeRequest.setCoreName(name); + mergeRequest.setIndexDirs(Arrays.asList(dir.getPath().toString() + "/data/index")); + mergeRequest.process(client); + req.success = true; + } catch (SolrServerException | IOException e) { + req.e = e; } + return req; }; pending.add(completionService.submit(task)); } diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java index a2430e7b1c16..23d04c6200a7 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java @@ -1188,12 +1188,7 @@ public boolean accept(Path path) { } // use alphanumeric sort (rather than lexicographical sort) to properly handle more than 99999 shards - Arrays.sort(dirs, new Comparator() { - @Override - public int compare(FileStatus f1, FileStatus f2) { - return new AlphaNumericComparator().compare(f1.getPath().getName(), f2.getPath().getName()); - } - }); + Arrays.sort(dirs, (f1, f2) -> new AlphaNumericComparator().compare(f1.getPath().getName(), f2.getPath().getName())); return dirs; } diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java index 3672e0b7c53c..9b86dcda97ae 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java @@ -114,12 +114,9 @@ public SolrZkClient getZkClient(String zkHost) { public List getSortedSlices(Collection slices) { List sorted = new ArrayList(slices); - Collections.sort(sorted, new Comparator() { - @Override - public int compare(Slice slice1, Slice slice2) { - Comparator c = new AlphaNumericComparator(); - return c.compare(slice1.getName(), slice2.getName()); - } + Collections.sort(sorted, (slice1, slice2) -> { + Comparator c = new AlphaNumericComparator(); + return c.compare(slice1.getName(), slice2.getName()); }); LOG.trace("Sorted slices: {}", sorted); return sorted; diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java index 0eb9e400469b..1c30a84e1079 100644 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java +++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/SolrMorphlineZkAvroTest.java @@ -79,12 +79,7 @@ public void test() throws Exception { assertEquals(2104, collector.getRecords().size()); assertEquals(collector.getRecords().size(), rsp.getResults().size()); - Collections.sort(collector.getRecords(), new Comparator() { - @Override - public int compare(Record r1, Record r2) { - return r1.get("id").toString().compareTo(r2.get("id").toString()); - } - }); + Collections.sort(collector.getRecords(), (r1, r2) -> r1.get("id").toString().compareTo(r2.get("id").toString())); // fetch test input data and sort like solr result set List records = new ArrayList(); @@ -94,12 +89,7 @@ public int compare(Record r1, Record r2) { records.add(expected); } assertEquals(collector.getRecords().size(), records.size()); - Collections.sort(records, new Comparator() { - @Override - public int compare(GenericData.Record r1, GenericData.Record r2) { - return r1.get("id").toString().compareTo(r2.get("id").toString()); - } - }); + Collections.sort(records, (r1, r2) -> r1.get("id").toString().compareTo(r2.get("id").toString())); Object lastId = null; for (int i = 0; i < records.size(); i++) { diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java b/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java index d6aa39262297..a9f8bdd0997c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java @@ -248,13 +248,7 @@ private boolean addReplica(final String collection, DownReplica badReplica) { log.debug("submit call to {}", createUrl); MDC.put("OverseerAutoReplicaFailoverThread.createUrl", createUrl); try { - updateExecutor.submit(new Callable() { - - @Override - public Boolean call() { - return createSolrCore(collection, createUrl, dataDir, ulogDir, coreNodeName, coreName); - } - }); + updateExecutor.submit(() -> createSolrCore(collection, createUrl, dataDir, ulogDir, coreNodeName, coreName)); } finally { MDC.remove("OverseerAutoReplicaFailoverThread.createUrl"); } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index ac0cabd8827f..3e5c8f9e7b7a 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -204,12 +204,9 @@ public IndexSchema createIndexSchema(final CoreDescriptor cd, final SolrConfig s if (Files.exists(schemaFile)) { try { String cachedName = cacheName(schemaFile); - return schemaCache.get(cachedName, new Callable() { - @Override - public IndexSchema call() throws Exception { - logger.info("Creating new index schema for core {}", cd.getName()); - return IndexSchemaFactory.buildIndexSchema(cd.getSchemaName(), solrConfig); - } + return schemaCache.get(cachedName, () -> { + logger.info("Creating new index schema for core {}", cd.getName()); + return IndexSchemaFactory.buildIndexSchema(cd.getSchemaName(), solrConfig); }); } catch (ExecutionException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index c140fb4dba8e..1074d690cbec 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -456,28 +456,25 @@ public void load() { solrCores.markCoreAsLoading(cd); } if (cd.isLoadOnStartup()) { - futures.add(coreLoadExecutor.submit(new Callable() { - @Override - public SolrCore call() throws Exception { - SolrCore core; - try { - if (zkSys.getZkController() != null) { - zkSys.getZkController().throwErrorIfReplicaReplaced(cd); - } - - core = create(cd, false); - } finally { - if (asyncSolrCoreLoad) { - solrCores.markCoreAsNotLoading(cd); - } + futures.add(coreLoadExecutor.submit(() -> { + SolrCore core; + try { + if (zkSys.getZkController() != null) { + zkSys.getZkController().throwErrorIfReplicaReplaced(cd); } - try { - zkSys.registerInZk(core, true); - } catch (RuntimeException e) { - SolrException.log(log, "Error registering SolrCore", e); + + core = create(cd, false); + } finally { + if (asyncSolrCoreLoad) { + solrCores.markCoreAsNotLoading(cd); } - return core; } + try { + zkSys.registerInZk(core, true); + } catch (RuntimeException e) { + SolrException.log(log, "Error registering SolrCore", e); + } + return core; })); } } diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java index 9d6797a4fb81..77d7bd9b5f15 100644 --- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java @@ -318,12 +318,7 @@ public boolean exists(String path) { FileSystem fileSystem = null; try { // no need to close the fs, the cache will do it - fileSystem = tmpFsCache.get(path, new Callable() { - @Override - public FileSystem call() throws IOException { - return FileSystem.get(hdfsDirPath.toUri(), conf); - } - }); + fileSystem = tmpFsCache.get(path, () -> FileSystem.get(hdfsDirPath.toUri(), conf)); } catch (ExecutionException e) { throw new RuntimeException(e); } @@ -351,12 +346,7 @@ protected synchronized void removeDirectory(final CacheValue cacheValue) try { // no need to close the fs, the cache will do it - fileSystem = tmpFsCache.get(cacheValue.path, new Callable() { - @Override - public FileSystem call() throws IOException { - return FileSystem.get(new Path(cacheValue.path).toUri(), conf); - } - }); + fileSystem = tmpFsCache.get(cacheValue.path, () -> FileSystem.get(new Path(cacheValue.path).toUri(), conf)); } catch (ExecutionException e) { throw new RuntimeException(e); } @@ -487,12 +477,7 @@ public void cleanupOldIndexDirectories(final String dataDir, final String curren final Configuration conf = getConf(); FileSystem fileSystem = null; try { - fileSystem = tmpFsCache.get(dataDir, new Callable() { - @Override - public FileSystem call() throws IOException { - return FileSystem.get(dataDirPath.toUri(), conf); - } - }); + fileSystem = tmpFsCache.get(dataDir, () -> FileSystem.get(dataDirPath.toUri(), conf)); } catch (ExecutionException e) { throw new RuntimeException(e); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index e511de054c8d..3e10efeb6bd2 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -735,12 +735,9 @@ public SolrCore(String name, String dataDir, SolrConfig config, // cause the executor to stall so firstSearcher events won't fire // until after inform() has been called for all components. // searchExecutor must be single-threaded for this to work - searcherExecutor.submit(new Callable() { - @Override - public Void call() throws Exception { - latch.await(); - return null; - } + searcherExecutor.submit(() -> { + latch.await(); + return null; }); this.updateHandler = initUpdateHandler(updateHandler); @@ -854,14 +851,7 @@ private void initSearcher(SolrCore prev) throws IOException { if (iwRef != null) { final IndexWriter iw = iwRef.get(); final SolrCore core = this; - newReaderCreator = new Callable() { - // this is used during a core reload - - @Override - public DirectoryReader call() throws Exception { - return indexReaderFactory.newReader(iw, core); - } - }; + newReaderCreator = () -> indexReaderFactory.newReader(iw, core); } } @@ -1779,57 +1769,48 @@ public RefCounted getSearcher(boolean forceNew, boolean retur // warm the new searcher based on the current searcher. // should this go before the other event handlers or after? if (currSearcher != null) { - future = searcherExecutor.submit(new Callable() { - @Override - public Object call() throws Exception { - try { - newSearcher.warm(currSearcher); - } catch (Throwable e) { - SolrException.log(log, e); - if (e instanceof Error) { - throw (Error) e; - } + future = searcherExecutor.submit(() -> { + try { + newSearcher.warm(currSearcher); + } catch (Throwable e) { + SolrException.log(log, e); + if (e instanceof Error) { + throw (Error) e; } - return null; } + return null; }); } if (currSearcher == null) { - future = searcherExecutor.submit(new Callable() { - @Override - public Object call() throws Exception { - try { - for (SolrEventListener listener : firstSearcherListeners) { - listener.newSearcher(newSearcher, null); - } - } catch (Throwable e) { - SolrException.log(log, null, e); - if (e instanceof Error) { - throw (Error) e; - } + future = searcherExecutor.submit(() -> { + try { + for (SolrEventListener listener : firstSearcherListeners) { + listener.newSearcher(newSearcher, null); + } + } catch (Throwable e) { + SolrException.log(log, null, e); + if (e instanceof Error) { + throw (Error) e; } - return null; } + return null; }); } if (currSearcher != null) { - future = searcherExecutor.submit(new Callable() { - @Override - public Object call() throws Exception { - try { - for (SolrEventListener listener : newSearcherListeners) { - listener.newSearcher(newSearcher, currSearcher); - } - } catch (Throwable e) { - SolrException.log(log, null, e); - if (e instanceof Error) { - throw (Error) e; - } + future = searcherExecutor.submit(() -> { + try { + for (SolrEventListener listener : newSearcherListeners) { + listener.newSearcher(newSearcher, currSearcher); + } + } catch (Throwable e) { + SolrException.log(log, null, e); + if (e instanceof Error) { + throw (Error) e; } - return null; } + return null; }); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCores.java b/solr/core/src/java/org/apache/solr/core/SolrCores.java index 86fab0d8223f..af287b7b0fb3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCores.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCores.java @@ -127,22 +127,19 @@ protected void close() { new DefaultSolrThreadFactory("coreCloseExecutor")); try { for (SolrCore core : coreList) { - coreCloseExecutor.submit(new Callable() { - @Override - public SolrCore call() throws Exception { - MDCLoggingContext.setCore(core); - try { - core.close(); - } catch (Throwable e) { - SolrException.log(log, "Error shutting down core", e); - if (e instanceof Error) { - throw (Error) e; - } - } finally { - MDCLoggingContext.clear(); + coreCloseExecutor.submit(() -> { + MDCLoggingContext.setCore(core); + try { + core.close(); + } catch (Throwable e) { + SolrException.log(log, "Error shutting down core", e); + if (e instanceof Error) { + throw (Error) e; } - return core; + } finally { + MDCLoggingContext.clear(); } + return core; }); } } finally { diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java index 7217abb91882..bb817e5df2a4 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java @@ -73,19 +73,14 @@ void start() { for (int i = 0; i < nCandidates; i++) { // a thread that poll one state from the queue, execute the replication task, and push back // the state in the queue when the task is completed - replicatorsPool.execute(new Runnable() { - - @Override - public void run() { - CdcrReplicatorState state = statesQueue.poll(); - assert state != null; // Should never happen - try { - new CdcrReplicator(state, batchSize).run(); - } finally { - statesQueue.offer(state); - } + replicatorsPool.execute(() -> { + CdcrReplicatorState state = statesQueue.poll(); + assert state != null; // Should never happen + try { + new CdcrReplicator(state, batchSize).run(); + } finally { + statesQueue.offer(state); } - }); } diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 89b0180127b1..a4d9ea081c99 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -284,17 +284,7 @@ public static class InterestingTerm { public Term term; public float boost; - - public static Comparator BOOST_ORDER = new Comparator() { - @Override - public int compare(InterestingTerm t1, InterestingTerm t2) { - float d = t1.boost - t2.boost; - if( d == 0 ) { - return 0; - } - return (d>0)?1:-1; - } - }; + } /** diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 09dc1b862e12..4ae013c56811 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -160,64 +160,61 @@ public void submit(final ShardRequest sreq, final String shard, final Modifiable // do this outside of the callable for thread safety reasons final List urls = getURLs(shard, preferredHostAddress); - Callable task = new Callable() { - @Override - public ShardResponse call() throws Exception { + Callable task = () -> { - ShardResponse srsp = new ShardResponse(); - if (sreq.nodeName != null) { - srsp.setNodeName(sreq.nodeName); + ShardResponse srsp = new ShardResponse(); + if (sreq.nodeName != null) { + srsp.setNodeName(sreq.nodeName); + } + srsp.setShardRequest(sreq); + srsp.setShard(shard); + SimpleSolrResponse ssr = new SimpleSolrResponse(); + srsp.setSolrResponse(ssr); + long startTime = System.nanoTime(); + + try { + params.remove(CommonParams.WT); // use default (currently javabin) + params.remove(CommonParams.VERSION); + + QueryRequest req = makeQueryRequest(sreq, params, shard); + req.setMethod(SolrRequest.METHOD.POST); + + // no need to set the response parser as binary is the default + // req.setResponseParser(new BinaryResponseParser()); + + // if there are no shards available for a slice, urls.size()==0 + if (urls.size()==0) { + // TODO: what's the right error code here? We should use the same thing when + // all of the servers for a shard are down. + throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "no servers hosting shard: " + shard); } - srsp.setShardRequest(sreq); - srsp.setShard(shard); - SimpleSolrResponse ssr = new SimpleSolrResponse(); - srsp.setSolrResponse(ssr); - long startTime = System.nanoTime(); - - try { - params.remove(CommonParams.WT); // use default (currently javabin) - params.remove(CommonParams.VERSION); - - QueryRequest req = makeQueryRequest(sreq, params, shard); - req.setMethod(SolrRequest.METHOD.POST); - - // no need to set the response parser as binary is the default - // req.setResponseParser(new BinaryResponseParser()); - - // if there are no shards available for a slice, urls.size()==0 - if (urls.size()==0) { - // TODO: what's the right error code here? We should use the same thing when - // all of the servers for a shard are down. - throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "no servers hosting shard: " + shard); - } - if (urls.size() <= 1) { - String url = urls.get(0); - srsp.setShardAddress(url); - try (SolrClient client = new HttpSolrClient(url, httpClient)) { - ssr.nl = client.request(req); - } - } else { - LBHttpSolrClient.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls); - ssr.nl = rsp.getResponse(); - srsp.setShardAddress(rsp.getServer()); + if (urls.size() <= 1) { + String url = urls.get(0); + srsp.setShardAddress(url); + try (SolrClient client = new HttpSolrClient(url, httpClient)) { + ssr.nl = client.request(req); } + } else { + LBHttpSolrClient.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls); + ssr.nl = rsp.getResponse(); + srsp.setShardAddress(rsp.getServer()); } - catch( ConnectException cex ) { - srsp.setException(cex); //???? - } catch (Exception th) { - srsp.setException(th); - if (th instanceof SolrException) { - srsp.setResponseCode(((SolrException)th).code()); - } else { - srsp.setResponseCode(-1); - } + } + catch( ConnectException cex ) { + srsp.setException(cex); //???? + } catch (Exception th) { + srsp.setException(th); + if (th instanceof SolrException) { + srsp.setResponseCode(((SolrException)th).code()); + } else { + srsp.setResponseCode(-1); } + } - ssr.elapsedTime = TimeUnit.MILLISECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + ssr.elapsedTime = TimeUnit.MILLISECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); - return transfomResponse(sreq, srsp, shard); - } + return transfomResponse(sreq, srsp, shard); }; try { diff --git a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java index 62a41468a7a9..2be8d1ec389f 100644 --- a/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java +++ b/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java @@ -103,12 +103,9 @@ NamedList getFacetCounts(Executor executor) throws IOException { for (final LeafReaderContext leave : leaves) { final SegFacet segFacet = new SegFacet(leave); - Callable task = new Callable() { - @Override - public SegFacet call() throws Exception { - segFacet.countTerms(); - return segFacet; - } + Callable task = () -> { + segFacet.countTerms(); + return segFacet; }; // TODO: if limiting threads, submit by largest segment first? diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index a9ce11dcfde6..3dd340328ff1 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -662,26 +662,23 @@ public NamedList getFacetFieldCounts() final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS); final String key = parsed.key; final String facetValue = parsed.facetValue; - Callable callable = new Callable() { - @Override - public NamedList call() throws Exception { - try { - NamedList result = new SimpleOrderedMap<>(); - if(termList != null) { - List terms = StrUtils.splitSmart(termList, ",", true); - result.add(key, getListedTermCounts(facetValue, parsed, terms)); - } else { - result.add(key, getTermCounts(facetValue, parsed)); - } - return result; - } catch (SolrException se) { - throw se; - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Exception during facet.field: " + facetValue, e); - } finally { - semaphore.release(); + Callable callable = () -> { + try { + NamedList result = new SimpleOrderedMap<>(); + if(termList != null) { + List terms = StrUtils.splitSmart(termList, ",", true); + result.add(key, getListedTermCounts(facetValue, parsed, terms)); + } else { + result.add(key, getTermCounts(facetValue, parsed)); } + return result; + } catch (SolrException se) { + throw se; + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Exception during facet.field: " + facetValue, e); + } finally { + semaphore.release(); } }; diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index 7addb20254d7..39d45c22a087 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -407,12 +407,7 @@ protected ValueSource getValueSourceFromSpatialArgs(QParser parser, SchemaField */ public T getStrategy(final String fieldName) { try { - return fieldStrategyCache.get(fieldName, new Callable() { - @Override - public T call() throws Exception { - return newSpatialStrategy(fieldName); - } - }); + return fieldStrategyCache.get(fieldName, () -> newSpatialStrategy(fieldName)); } catch (ExecutionException e) { throw Throwables.propagate(e.getCause()); } diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index a99952dbd573..9a196b7ca407 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -279,14 +279,9 @@ private void submit(final Req req, boolean isCommit) { // a commit using ConncurrentUpdateSolrServer is not async, // so we make it async to prevent commits from happening // serially across multiple nodes - pending.add(completionService.submit(new Callable() { - - @Override - public Object call() throws Exception { - doRequest(req); - return null; - } - + pending.add(completionService.submit(() -> { + doRequest(req); + return null; })); } else { doRequest(req); diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java index 307492f53b60..6439498dfeb1 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java @@ -197,12 +197,7 @@ public void testBuffering() throws Exception { } }; - UpdateLog.testing_logReplayFinishHook = new Runnable() { - @Override - public void run() { - logReplayFinish.release(); - } - }; + UpdateLog.testing_logReplayFinishHook = logReplayFinish::release; SolrQueryRequest req = req(); diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java index d0d61be5bcff..7de7a2bcff44 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java @@ -161,7 +161,7 @@ public void testLogReplay() throws Exception { } }; - UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release(); + UpdateLog.testing_logReplayFinishHook = logReplayFinish::release; clearIndex(); @@ -257,7 +257,7 @@ public void testBuffering() throws Exception { } }; - UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release(); + UpdateLog.testing_logReplayFinishHook = logReplayFinish::release; SolrQueryRequest req = req(); @@ -412,7 +412,7 @@ public void testDropBuffered() throws Exception { } }; - UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release(); + UpdateLog.testing_logReplayFinishHook = logReplayFinish::release; SolrQueryRequest req = req(); @@ -722,12 +722,7 @@ public void testRemoveOldLogs() throws Exception { } }; - UpdateLog.testing_logReplayFinishHook = new Runnable() { - @Override - public void run() { - logReplayFinish.release(); - } - }; + UpdateLog.testing_logReplayFinishHook = logReplayFinish::release; clearIndex(); diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java index 3be8c2eba062..557e6b230327 100644 --- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -575,22 +575,14 @@ private Collection> callables(List blocks) { for (Document block : blocks) { final String msg = block.asXML(); if (msg.length() > 0) { - rez.add(new Callable() { - @Override - public Void call() { - assertBlockU(msg); - return null; - } - + rez.add(() -> { + assertBlockU(msg); + return null; }); if (rarely()) { - rez.add(new Callable() { - @Override - public Void call() { - assertBlockU(commit()); - return null; - } - + rez.add(() -> { + assertBlockU(commit()); + return null; }); } } diff --git a/solr/core/src/test/org/apache/solr/update/TestUpdate.java b/solr/core/src/test/org/apache/solr/update/TestUpdate.java index 13a24797e835..6efa5bbaed36 100644 --- a/solr/core/src/test/org/apache/solr/update/TestUpdate.java +++ b/solr/core/src/test/org/apache/solr/update/TestUpdate.java @@ -37,20 +37,12 @@ public void testUpdateableDocs() throws Exception { // Test both by running the same test with and without commits // do without commits - doUpdateTest(new Callable() { - @Override - public Object call() throws Exception { - return null; - } - }); + doUpdateTest(() -> null); // do with commits - doUpdateTest(new Callable() { - @Override - public Object call() throws Exception { - assertU(commit("softCommit","false")); - return null; - } + doUpdateTest(() -> { + assertU(commit("softCommit","false")); + return null; }); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index edfe1c3e1178..4ad37260a948 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -603,12 +603,7 @@ private NamedList directUpdate(AbstractUpdateRequest request, String col final LBHttpSolrClient.Req lbRequest = entry.getValue(); try { MDC.put("CloudSolrClient.url", url); - responseFutures.put(url, threadPool.submit(new Callable>() { - @Override - public NamedList call() throws Exception { - return lbClient.request(lbRequest).getResponse(); - } - })); + responseFutures.put(url, threadPool.submit(() -> lbClient.request(lbRequest).getResponse())); } finally { MDC.remove("CloudSolrClient.url"); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index fe4459705617..b987afc032f9 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -278,12 +278,7 @@ public HttpUriRequestResponse httpUriRequest(final SolrRequest request, final Re ExecutorService pool = ExecutorUtil.newMDCAwareFixedThreadPool(1, new SolrjNamedThreadFactory("httpUriRequest")); try { MDC.put("HttpSolrClient.url", baseUrl); - mrr.future = pool.submit(new Callable>(){ - - @Override - public NamedList call() throws Exception { - return executeMethod(method, processor); - }}); + mrr.future = pool.submit(() -> executeMethod(method, processor)); } finally { pool.shutdown(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java index c3a22891b195..10c7c294e6e7 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/comp/FieldComparator.java @@ -97,34 +97,28 @@ public StreamExpressionParameter toExpression(StreamFactory factory){ */ private void assignComparator(){ if(ComparatorOrder.DESCENDING == order){ - comparator = new ComparatorLambda() { - @Override - public int compare(Tuple leftTuple, Tuple rightTuple) { - Comparable leftComp = (Comparable)leftTuple.get(leftFieldName); - Comparable rightComp = (Comparable)rightTuple.get(rightFieldName); - - if(leftComp == rightComp){ return 0; } // if both null then they are equal. if both are same ref then are equal - if(null == leftComp){ return 1; } - if(null == rightComp){ return -1; } - - return rightComp.compareTo(leftComp); - } + comparator = (leftTuple, rightTuple) -> { + Comparable leftComp = (Comparable)leftTuple.get(leftFieldName); + Comparable rightComp = (Comparable)rightTuple.get(rightFieldName); + + if(leftComp == rightComp){ return 0; } // if both null then they are equal. if both are same ref then are equal + if(null == leftComp){ return 1; } + if(null == rightComp){ return -1; } + + return rightComp.compareTo(leftComp); }; } else{ // See above for black magic reasoning. - comparator = new ComparatorLambda() { - @Override - public int compare(Tuple leftTuple, Tuple rightTuple) { - Comparable leftComp = (Comparable)leftTuple.get(leftFieldName); - Comparable rightComp = (Comparable)rightTuple.get(rightFieldName); - - if(leftComp == rightComp){ return 0; } // if both null then they are equal. if both are same ref then are equal - if(null == leftComp){ return -1; } - if(null == rightComp){ return 1; } - - return leftComp.compareTo(rightComp); - } + comparator = (leftTuple, rightTuple) -> { + Comparable leftComp = (Comparable)leftTuple.get(leftFieldName); + Comparable rightComp = (Comparable)rightTuple.get(rightFieldName); + + if(leftComp == rightComp){ return 0; } // if both null then they are equal. if both are same ref then are equal + if(null == leftComp){ return -1; } + if(null == rightComp){ return 1; } + + return leftComp.compareTo(rightComp); }; } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 002304173cc3..118f7bef621e 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -206,12 +206,7 @@ public MiniSolrCloudCluster(int numServers, Path baseDir, String solrXml, JettyC List> startups = new ArrayList<>(numServers); for (int i = 0; i < numServers; ++i) { - startups.add(new Callable() { - @Override - public JettySolrRunner call() throws Exception { - return startJettySolrRunner(newNodeName(), jettyConfig.context, jettyConfig); - } - }); + startups.add(() -> startJettySolrRunner(newNodeName(), jettyConfig.context, jettyConfig)); } Collection> futures = executor.invokeAll(startups); @@ -424,12 +419,7 @@ public void shutdown() throws Exception { solrClient.close(); List> shutdowns = new ArrayList<>(jettys.size()); for (final JettySolrRunner jetty : jettys) { - shutdowns.add(new Callable() { - @Override - public JettySolrRunner call() throws Exception { - return stopJettySolrRunner(jetty); - } - }); + shutdowns.add(() -> stopJettySolrRunner(jetty)); } jettys.clear(); Collection> futures = executor.invokeAll(shutdowns); From 88d2b3bfbed3d823a13b486ba44cb2d219ae0941 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 5 Apr 2016 23:21:07 +0530 Subject: [PATCH 0282/1113] SOLR-8750: replace anonymous inner class for callable, Runnable etc --- .../solr/cloud/BasicDistributedZkTest.java | 80 +++++++++---------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 8222e91677f4..4a6b2d7af6b6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -959,35 +959,32 @@ private void createCollection(String collection, private void createSolrCore(final String collection, List collectionClients, final String baseUrl, final int num, final String shardId) { - Callable call = new Callable() { - @Override - public Object call() { - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { - client.setConnectionTimeout(15000); - Create createCmd = new Create(); - createCmd.setRoles("none"); - createCmd.setCoreName(collection + num); - createCmd.setCollection(collection); - - if (random().nextBoolean()) { - // sometimes we use an explicit core node name - createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); - } - - if (shardId == null) { - createCmd.setNumShards(2); - } - createCmd.setDataDir(getDataDir(createTempDir(collection).toFile().getAbsolutePath())); - if (shardId != null) { - createCmd.setShardId(shardId); - } - client.request(createCmd); - } catch (Exception e) { - e.printStackTrace(); - //fail + Callable call = () -> { + try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + client.setConnectionTimeout(15000); + Create createCmd = new Create(); + createCmd.setRoles("none"); + createCmd.setCoreName(collection + num); + createCmd.setCollection(collection); + + if (random().nextBoolean()) { + // sometimes we use an explicit core node name + createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); } - return null; + + if (shardId == null) { + createCmd.setNumShards(2); + } + createCmd.setDataDir(getDataDir(createTempDir(collection).toFile().getAbsolutePath())); + if (shardId != null) { + createCmd.setShardId(shardId); + } + client.request(createCmd); + } catch (Exception e) { + e.printStackTrace(); + //fail } + return null; }; pending.add(completionService.submit(call)); @@ -1090,23 +1087,20 @@ private void createNewCollection(final String collection) throws InterruptedExce ((HttpSolrClient) client).getBaseURL().length() - DEFAULT_COLLECTION.length() -1); final int frozeUnique = unique; - Callable call = new Callable() { - @Override - public Object call() { - - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { - client.setConnectionTimeout(15000); - client.setSoTimeout(60000); - Create createCmd = new Create(); - createCmd.setCoreName(collection); - createCmd.setDataDir(getDataDir(createTempDir(collection).toFile().getAbsolutePath())); - client.request(createCmd); - } catch (Exception e) { - e.printStackTrace(); - //fails - } - return null; + Callable call = () -> { + + try (HttpSolrClient client1 = new HttpSolrClient(baseUrl)) { + client1.setConnectionTimeout(15000); + client1.setSoTimeout(60000); + Create createCmd = new Create(); + createCmd.setCoreName(collection); + createCmd.setDataDir(getDataDir(createTempDir(collection).toFile().getAbsolutePath())); + client1.request(createCmd); + } catch (Exception e) { + e.printStackTrace(); + //fails } + return null; }; collectionClients.add(createNewSolrClient(collection, baseUrl)); From 4cfd33f7434509d535fa8d5faaf3d20ced1935ce Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 5 Apr 2016 14:15:31 -0400 Subject: [PATCH 0283/1113] SOLR-8875: Fix null clusterState from ZkStateWriter. Revert my sanity check asserts in Overseer. (cherry picked from commit 3bbf8aa) --- solr/CHANGES.txt | 5 +++-- solr/core/src/java/org/apache/solr/cloud/Overseer.java | 1 + .../java/org/apache/solr/cloud/overseer/ZkStateWriter.java | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 17e949a93570..cc2a43ee599c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -70,7 +70,8 @@ Bug Fixes * SOLR-8902: Make sure ReturnFields only returns the requested fields from (fl=) evn when DocumentTransformers ask for getExtraRequestFields() (ryan) -* SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown. (Mark Miller) +* SOLR-8875: SolrCloud Overseer clusterState could unexpectedly be null resulting in NPE. + (Scott Blum via David Smiley) Optimizations ---------------------- @@ -499,7 +500,7 @@ Other Changes * SOLR-8758: Add a new SolrCloudTestCase class, using MiniSolrCloudCluster (Alan Woodward) - + * SOLR-8764: Remove all deprecated methods and classes from master prior to the 6.0 release. (Steve Rowe) * SOLR-8780: Remove unused OverseerCollectionMessageHandler#getClusterStatus method. (Varun Thacker) diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 0e5bded5f3f5..f25cab7b670c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -151,6 +151,7 @@ else if (LeaderStatus.YES != isLeader) { continue; // not a no, not a yes, try ask again } + //TODO consider removing 'refreshClusterState' and simply check if clusterState is null if (refreshClusterState) { try { reader.updateClusterState(); diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java index 9fb3ada40c63..ec67ed7dbc4f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud.overseer; +import java.lang.invoke.MethodHandles; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -35,8 +36,6 @@ import static java.util.Collections.singletonMap; -import java.lang.invoke.MethodHandles; - /** * ZkStateWriter is responsible for writing updates to the cluster state stored in ZooKeeper for * both stateFormat=1 collection (stored in shared /clusterstate.json in ZK) and stateFormat=2 collections @@ -84,6 +83,7 @@ public ZkStateWriter(ZkStateReader zkStateReader, Overseer.Stats stats) { this.reader = zkStateReader; this.stats = stats; + this.clusterState = zkStateReader.getClusterState(); } /** From 2b0963ee38ad751036066260944ce34bdeb1a55b Mon Sep 17 00:00:00 2001 From: Gregory Chanan Date: Thu, 31 Mar 2016 17:16:17 -0700 Subject: [PATCH 0284/1113] SOLR-8892: Allow SolrInfoMBeans to return different statistics for /jmx vs web ui calls --- solr/CHANGES.txt | 3 + .../org/apache/solr/core/JmxMonitoredMap.java | 23 ++++++- .../solr/core/SolrInfoMBeanWrapper.java | 62 +++++++++++++++++++ .../solr/search/SolrFieldCacheMBean.java | 15 ++++- .../apache/solr/core/TestJmxMonitoredMap.java | 37 +++++++++++ .../solr/search/TestSolrFieldCacheMBean.java | 36 ++++++++--- 6 files changed, 165 insertions(+), 11 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/SolrInfoMBeanWrapper.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index cc2a43ee599c..35aacbeb6167 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -99,6 +99,9 @@ Other Changes * SOLR-8869: Optionally disable printing field cache entries in SolrFieldCacheMBean (Gregory Chanan) +* SOLR-8892: Allow SolrInfoMBeans to return different statistics for /jmx vs web ui calls. + (Gregory Chanan, Mark Miller) + ================== 6.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java index 0260f516d2a7..57bde93decd6 100644 --- a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java +++ b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java @@ -258,7 +258,15 @@ public SolrDynamicMBean(String coreHashCode, SolrInfoMBean managedResource) { public SolrDynamicMBean(String coreHashCode, SolrInfoMBean managedResource, boolean useCachedStatsBetweenGetMBeanInfoCalls) { this.useCachedStatsBetweenGetMBeanInfoCalls = useCachedStatsBetweenGetMBeanInfoCalls; - this.infoBean = managedResource; + if (managedResource instanceof JmxAugmentedSolrInfoMBean) { + final JmxAugmentedSolrInfoMBean jmxSpecific = (JmxAugmentedSolrInfoMBean)managedResource; + this.infoBean = new SolrInfoMBeanWrapper(jmxSpecific) { + @Override + public NamedList getStatistics() { return jmxSpecific.getStatisticsForJmx(); } + }; + } else { + this.infoBean = managedResource; + } staticStats = new HashSet<>(); // For which getters are already available in SolrInfoMBean @@ -412,4 +420,17 @@ public Object invoke(String actionName, Object[] params, String[] signature) throw new UnsupportedOperationException("Operation not Supported"); } } + + /** + * SolrInfoMBean that provides JMX-specific statistics. Used, for example, + * if generating full statistics is expensive; the expensive statistics can + * be generated normally for use with the web ui, while an abbreviated version + * are generated for period jmx use. + */ + public interface JmxAugmentedSolrInfoMBean extends SolrInfoMBean { + /** + * JMX-specific statistics + */ + public NamedList getStatisticsForJmx(); + } } diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBeanWrapper.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBeanWrapper.java new file mode 100644 index 000000000000..534b88449318 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBeanWrapper.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import java.net.URL; + +import org.apache.solr.common.util.NamedList; + +/** + * Wraps a {@link SolrInfoMBean}. + */ +public class SolrInfoMBeanWrapper implements SolrInfoMBean { + private final SolrInfoMBean mbean; + + public SolrInfoMBeanWrapper(SolrInfoMBean mbean) { + this.mbean = mbean; + } + + /** {@inheritDoc} */ + @Override + public String getName() { return mbean.getName(); } + + /** {@inheritDoc} */ + @Override + public String getVersion() { return mbean.getVersion(); } + + /** {@inheritDoc} */ + @Override + public String getDescription() { return mbean.getDescription(); } + + /** {@inheritDoc} */ + @Override + public Category getCategory() { return mbean.getCategory(); } + + /** {@inheritDoc} */ + @Override + public String getSource() { return mbean.getSource(); } + + /** {@inheritDoc} */ + @Override + public URL[] getDocs() { return mbean.getDocs(); } + + /** {@inheritDoc} */ + @Override + public NamedList getStatistics() { return mbean.getStatistics(); } + +} diff --git a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java index cfc68c4d6879..da997081c2ea 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java +++ b/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java @@ -24,14 +24,16 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoMBean; +import org.apache.solr.core.JmxMonitoredMap.JmxAugmentedSolrInfoMBean; /** * A SolrInfoMBean that provides introspection of the Solr FieldCache * */ -public class SolrFieldCacheMBean implements SolrInfoMBean { +public class SolrFieldCacheMBean implements JmxAugmentedSolrInfoMBean { private boolean disableEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryList"); + private boolean disableJmxEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryListJmx"); @Override public String getName() { return this.getClass().getName(); } @@ -51,10 +53,19 @@ public URL[] getDocs() { } @Override public NamedList getStatistics() { + return getStats(!disableEntryList); + } + + @Override + public NamedList getStatisticsForJmx() { + return getStats(!disableEntryList && !disableJmxEntryList); + } + + private NamedList getStats(boolean listEntries) { NamedList stats = new SimpleOrderedMap(); String[] entries = UninvertingReader.getUninvertedStats(); stats.add("entries_count", entries.length); - if (!disableEntryList) { + if (listEntries) { for (int i = 0; i < entries.length; i++) { stats.add("entry#" + i, entries[i]); } diff --git a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java index e98b79029002..9f1e2ab515b7 100644 --- a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java +++ b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java @@ -180,4 +180,41 @@ public void testPutRemoveClear() throws Exception { } + @Test + public void testJmxAugmentedSolrInfoMBean() throws Exception { + final MockInfoMBean mock = new MockInfoMBean(); + final String jmxKey = "jmx"; + final String jmxValue = "jmxValue"; + + MockJmxAugmentedSolrInfoMBean mbean = new MockJmxAugmentedSolrInfoMBean(mock) { + @Override + public NamedList getStatisticsForJmx() { + NamedList stats = getStatistics(); + stats.add(jmxKey, jmxValue); + return stats; + } + }; + monitoredMap.put("mock", mbean); + + // assert getStatistics called when used as a map. Note can't use equals here to compare + // because getStatistics returns a new Object each time. + assertNull(monitoredMap.get("mock").getStatistics().get(jmxKey)); + + // assert getStatisticsForJmx called when used as jmx server + Set objects = mbeanServer.queryMBeans(null, Query.match( + Query.attr("name"), Query.value("mock"))); + ObjectName name = objects.iterator().next().getObjectName(); + assertMBeanTypeAndValue(name, jmxKey, jmxValue.getClass(), jmxValue); + } + + private static abstract class MockJmxAugmentedSolrInfoMBean + extends SolrInfoMBeanWrapper implements JmxMonitoredMap.JmxAugmentedSolrInfoMBean { + + public MockJmxAugmentedSolrInfoMBean(SolrInfoMBean mbean) { + super(mbean); + } + + @Override + public abstract NamedList getStatisticsForJmx(); + } } diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java index b90430e9ef6c..5343f7345fb0 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrFieldCacheMBean.java @@ -42,20 +42,40 @@ public void testEntryList() throws Exception { assertU(commit()); assertQ(req("q", "*:*", "sort", "id asc"), "//*[@numFound='1']"); - SolrFieldCacheMBean mbean = new SolrFieldCacheMBean(); - NamedList stats = mbean.getStatistics(); - assert(new Integer(stats.get("entries_count").toString()) > 0); - assertNotNull(stats.get("entry#0")); + // Test with entry list enabled + assertEntryListIncluded(false); // Test again with entry list disabled System.setProperty("disableSolrFieldCacheMBeanEntryList", "true"); try { - mbean = new SolrFieldCacheMBean(); - stats = mbean.getStatistics(); - assert(new Integer(stats.get("entries_count").toString()) > 0); - assertNull(stats.get("entry#0")); + assertEntryListNotIncluded(false); } finally { System.clearProperty("disableSolrFieldCacheMBeanEntryList"); } + + // Test with entry list enabled for jmx + assertEntryListIncluded(true); + + // Test with entry list disabled for jmx + System.setProperty("disableSolrFieldCacheMBeanEntryListJmx", "true"); + try { + assertEntryListNotIncluded(true); + } finally { + System.clearProperty("disableSolrFieldCacheMBeanEntryListJmx"); + } + } + + private void assertEntryListIncluded(boolean checkJmx) { + SolrFieldCacheMBean mbean = new SolrFieldCacheMBean(); + NamedList stats = checkJmx ? mbean.getStatisticsForJmx() : mbean.getStatistics(); + assert(new Integer(stats.get("entries_count").toString()) > 0); + assertNotNull(stats.get("entry#0")); + } + + private void assertEntryListNotIncluded(boolean checkJmx) { + SolrFieldCacheMBean mbean = new SolrFieldCacheMBean(); + NamedList stats = checkJmx ? mbean.getStatisticsForJmx() : mbean.getStatistics(); + assert(new Integer(stats.get("entries_count").toString()) > 0); + assertNull(stats.get("entry#0")); } } From 6c7487351f3df0b35356763d5587b1e987be5bbf Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Wed, 6 Apr 2016 11:56:27 +0530 Subject: [PATCH 0285/1113] SOLR-8948: OverseerTaskQueue.containsTaskWithRequestId encounters json parse error if a SolrResponse node is in the overseer queue (cherry picked from commit 4205b1c) --- solr/CHANGES.txt | 3 + .../apache/solr/cloud/OverseerTaskQueue.java | 21 ++++-- .../solr/cloud/OverseerTaskQueueTest.java | 67 +++++++++++++++++++ 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 35aacbeb6167..d0c1c361ac70 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -73,6 +73,9 @@ Bug Fixes * SOLR-8875: SolrCloud Overseer clusterState could unexpectedly be null resulting in NPE. (Scott Blum via David Smiley) +* SOLR-8948: OverseerTaskQueue.containsTaskWithRequestId encounters json parse error if a + SolrResponse node is in the overseer queue. (Jessica Cheng Mallet via shalin) + Optimizations ---------------------- * SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation. diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java index cf9d5834e8d9..4cee814e38c5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java @@ -61,7 +61,7 @@ public boolean containsTaskWithRequestId(String requestIdKey, String requestId) List childNames = zookeeper.getChildren(dir, null, true); stats.setQueueLength(childNames.size()); for (String childName : childNames) { - if (childName != null) { + if (childName != null && childName.startsWith(PREFIX)) { try { byte[] data = zookeeper.getData(dir + "/" + childName, null, null, true); if (data != null) { @@ -185,17 +185,14 @@ public QueueEvent offer(byte[] data, long timeout) throws KeeperException, try { // Create and watch the response node before creating the request node; // otherwise we may miss the response. - String watchID = createData( - dir + "/" + response_prefix, - null, CreateMode.EPHEMERAL_SEQUENTIAL); + String watchID = createResponseNode(); Object lock = new Object(); LatchWatcher watcher = new LatchWatcher(lock); Stat stat = zookeeper.exists(watchID, watcher, true); // create the request node - createData(dir + "/" + PREFIX + watchID.substring(watchID.lastIndexOf("-") + 1), - data, CreateMode.PERSISTENT); + createRequestNode(data, watchID); synchronized (lock) { if (stat != null && watcher.getWatchedEvent() == null) { @@ -213,6 +210,18 @@ public QueueEvent offer(byte[] data, long timeout) throws KeeperException, } } + void createRequestNode(byte[] data, String watchID) throws KeeperException, InterruptedException { + createData(dir + "/" + PREFIX + watchID.substring(watchID.lastIndexOf("-") + 1), + data, CreateMode.PERSISTENT); + } + + String createResponseNode() throws KeeperException, InterruptedException { + return createData( + dir + "/" + response_prefix, + null, CreateMode.EPHEMERAL_SEQUENTIAL); + } + + public List peekTopN(int n, Set excludeSet, long waitMillis) throws KeeperException, InterruptedException { ArrayList topN = new ArrayList<>(); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java index 028f85fb2725..95cdd4077820 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java @@ -16,6 +16,19 @@ */ package org.apache.solr.cloud; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.response.SolrResponseBase; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CommonAdminParams; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.Utils; +import org.junit.Test; + public class OverseerTaskQueueTest extends DistributedQueueTest { @@ -25,4 +38,58 @@ public class OverseerTaskQueueTest extends DistributedQueueTest { protected OverseerTaskQueue makeDistributedQueue(String dqZNode) throws Exception { return new OverseerTaskQueue(zkClient, setupNewDistributedQueueZNode(dqZNode)); } + + @Test + public void testContainsTaskWithRequestId() throws Exception { + String tqZNode = "/taskqueue/test"; + String requestId = "foo"; + String nonExistentRequestId = "bar"; + + OverseerTaskQueue tq = makeDistributedQueue(tqZNode); + + // Basic ops + // Put an expected Overseer task onto the queue + final Map props = new HashMap<>(); + props.put(CommonParams.NAME, "coll1"); + props.put(OverseerCollectionMessageHandler.COLL_CONF, "myconf"); + props.put(OverseerCollectionMessageHandler.NUM_SLICES, 1); + props.put(ZkStateReader.REPLICATION_FACTOR, 3); + props.put(CommonAdminParams.ASYNC, requestId); + tq.offer(Utils.toJSON(props)); + + assertTrue("Task queue should contain task with requestid " + requestId, + tq.containsTaskWithRequestId(CommonAdminParams.ASYNC, requestId)); + + assertFalse("Task queue should not contain task with requestid " + nonExistentRequestId, + tq.containsTaskWithRequestId(CommonAdminParams.ASYNC, nonExistentRequestId)); + + // Create a response node as if someone is waiting for a response from the Overseer; then, + // create the request node. + // Here we're reaching a bit into the internals of OverseerTaskQueue in order to create the same + // response node structure but without setting a watch on it and removing it immediately when + // a response is set, in order to artificially create the race condition that + // containsTaskWithRequestId runs while the response is still in the queue. + String watchID = tq.createResponseNode(); + String requestId2 = "baz"; + props.put(CommonAdminParams.ASYNC, requestId2); + tq.createRequestNode(Utils.toJSON(props), watchID); + + // Set a SolrResponse as the response node by removing the QueueEvent, as done in OverseerTaskProcessor + List queueEvents = tq.peekTopN(2, Collections.emptySet(), 1000); + OverseerTaskQueue.QueueEvent requestId2Event = null; + for (OverseerTaskQueue.QueueEvent queueEvent : queueEvents) { + Map eventProps = (Map) Utils.fromJSON(queueEvent.getBytes()); + if (requestId2.equals(eventProps.get(CommonAdminParams.ASYNC))) { + requestId2Event = queueEvent; + break; + } + } + assertNotNull("Didn't find event with requestid " + requestId2, requestId2Event); + requestId2Event.setBytes(SolrResponse.serializable(new SolrResponseBase())); + tq.remove(requestId2Event); + + // Make sure this call to check if requestId exists doesn't barf with Json parse exception + assertTrue("Task queue should contain task with requestid " + requestId, + tq.containsTaskWithRequestId(CommonAdminParams.ASYNC, requestId)); + } } From 3d6f2aa31cee0251521f976276cc349ebe6f45c0 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 6 Apr 2016 12:48:52 +0100 Subject: [PATCH 0286/1113] LUCENE-7178: TestCoreParser tweaks Summary: * rename testSimpleXML to testTermQueryXML * rename testSimpleTermsQueryXML to testTermsQueryXML * try-with-resources and file-not-found-assert for TestCoreParser.parse * TestCoreParser.dumpResults verbose logging now includes qType and numDocs (Ramkumar Aiyengar, Nathan Visagan, Christine Poerschke) --- .../lucene/queryparser/xml/TestCoreParser.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java index c8b357ed30e4..04faa7d5f1ca 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestCoreParser.java @@ -95,12 +95,12 @@ public static void afterClass() throws Exception { analyzer = null; } - public void testSimpleXML() throws ParserException, IOException { + public void testTermQueryXML() throws ParserException, IOException { Query q = parse("TermQuery.xml"); dumpResults("TermQuery", q, 5); } - public void testSimpleTermsQueryXML() throws ParserException, IOException { + public void testTermsQueryXML() throws ParserException, IOException { Query q = parse("TermsQuery.xml"); dumpResults("TermsQuery", q, 5); } @@ -187,10 +187,11 @@ protected CoreParser coreParser() { } protected Query parse(String xmlFileName) throws ParserException, IOException { - InputStream xmlStream = TestCoreParser.class.getResourceAsStream(xmlFileName); - Query result = coreParser().parse(xmlStream); - xmlStream.close(); - return result; + try (InputStream xmlStream = TestCoreParser.class.getResourceAsStream(xmlFileName)) { + assertNotNull("Test XML file " + xmlFileName + " cannot be found", xmlStream); + Query result = coreParser().parse(xmlStream); + return result; + } } protected Query rewrite(Query q) throws IOException { @@ -199,7 +200,7 @@ protected Query rewrite(Query q) throws IOException { protected void dumpResults(String qType, Query q, int numDocs) throws IOException { if (VERBOSE) { - System.out.println("TEST: query=" + q); + System.out.println("TEST: qType=" + qType + " query=" + q + " numDocs=" + numDocs); } TopDocs hits = searcher.search(q, numDocs); assertTrue(qType + " should produce results ", hits.totalHits > 0); From a903eeaae8733b2f1ddc0fc6370a9ba6a7934aeb Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Wed, 6 Apr 2016 18:18:57 +0100 Subject: [PATCH 0287/1113] Tweak TestRateLimiter.testThreads logic. (Make visible the fact that the machine/actualMBPerSec was way-too-slow i.e. the not-too-fast portion of the test wasn't really checked.) --- .../core/src/test/org/apache/lucene/store/TestRateLimiter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/store/TestRateLimiter.java b/lucene/core/src/test/org/apache/lucene/store/TestRateLimiter.java index ca44bf04f77f..50db1cde2d82 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestRateLimiter.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestRateLimiter.java @@ -94,7 +94,7 @@ public void run() { double ratio = actualMBPerSec/targetMBPerSec; // Only enforce that it wasn't too fast; if machine is bogged down (can't schedule threads / sleep properly) then it may falsely be too slow: - //assertTrue("targetMBPerSec=" + targetMBPerSec + " actualMBPerSec=" + actualMBPerSec, ratio >= 0.9 && ratio <= 1.1); + assumeTrue("actualMBPerSec=" + actualMBPerSec + " targetMBPerSec=" + targetMBPerSec, 0.9 <= ratio); assertTrue("targetMBPerSec=" + targetMBPerSec + " actualMBPerSec=" + actualMBPerSec, ratio <= 1.1); } } From 99edbaaa12c3a7b9b67121b0bdc4be1f9f2a7b68 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 6 Apr 2016 14:17:34 -0400 Subject: [PATCH 0288/1113] don't test LatLonPoint's quantization with epsilons: demand exact answers --- .../apache/lucene/document/LatLonPoint.java | 9 +- .../lucene/document/TestLatLonPoint.java | 263 +++++++----------- 2 files changed, 105 insertions(+), 167 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 0d8a7b2b774c..107d83cb8772 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -46,8 +46,9 @@ *
  • {@link #newPolygonQuery newPolygonQuery()} for matching points within an arbitrary polygon. * *

    - * WARNING: Values are indexed with some loss of precision, incurring up to 1E-7 error from the - * original {@code double} values. + * WARNING: Values are indexed with some loss of precision from the + * original {@code double} values (4.190951585769653E-8 for the latitude component + * and 8.381903171539307E-8 for longitude). * @see PointValues */ // TODO ^^^ that is very sandy and hurts the API, usage, and tests tremendously, because what the user passes @@ -98,9 +99,9 @@ public LatLonPoint(String name, double latitude, double longitude) { private static final int BITS = 32; private static final double LONGITUDE_MUL = (0x1L<", LatLonPoint.newDistanceSort("field", 18.0, 19.0).toString()); } - - public void testEncodeDecode() throws Exception { - // just for testing quantization error - final double ENCODING_TOLERANCE = 1e-7; - - int iters = atLeast(10000); - for(int iter=0;iter Math.PI) { + maximumAngle = Math.PI; + } + // The minimum angle is MINIMUM_EDGE_ANGLE, or enough to be sure nobody afterwards needs more than + // 180 degrees. And since we have three points to start with, we already know that. + final double minimumAngle = MINIMUM_EDGE_ANGLE; + // Pick the angle + final double angle = random().nextDouble() * (maximumAngle - minimumAngle) + minimumAngle; + angles[i] = angle; + accumulatedAngle += angle; + } + // Pick the arc distance randomly + arcDistance[i] = random().nextDouble() * (Math.PI * 0.5 - MINIMUM_ARC_ANGLE) + MINIMUM_ARC_ANGLE; + } + if (clockwiseDesired) { + // Reverse the signs + for (int i = 0; i < pointCount; i++) { + angles[i] = -angles[i]; + } + } + + // Now, use the pole's information plus angles and arcs to create GeoPoints in the right order. + final List polyPoints = convertToPoints(pm, pole, angles, arcDistance); + + // Create the geo3d polygon, so we can test out our poles. + GeoPolygon poly = GeoPolygonFactory.makeGeoPolygon(pm, polyPoints, null); + + // Next, do some holes. No more than 2 of these. The poles for holes must always be within the polygon, so we're + // going to use Geo3D to help us select those given the points we just made. + + final int holeCount = TestUtil.nextInt(random(), 0, 2); + + final Polygon[] holes = new Polygon[holeCount]; + + for (int i = 0; i < holeCount; i++) { + // Choose a pole. The poly has to be within the polygon, but it also cannot be on the polygon edge. + // We try indefinitely to find a good pole... + while (true) { + final GeoPoint poleChoice = new GeoPoint(pm, toRadians(randomLat()), toRadians(randomLon())); + if (!poly.isWithin(poleChoice)) { + continue; + } + // We have a pole within the polygon. Now try 100 times to build a polygon that does not intersect the outside ring. + // After that we give up and pick a new pole. + boolean foundOne = false; + for (int j = 0; j < 100; j++) { + final Polygon insidePoly = makePoly(pm, poleChoice, !clockwiseDesired); + // Verify that the inside polygon is OK. If not, discard and repeat. + if (!verifyPolygon(pm, insidePoly, poly)) { + continue; + } + holes[i] = insidePoly; + foundOne = true; + } + if (foundOne) { + break; + } + } + } + + // Finally, build the polygon and return it + final double[] lats = new double[polyPoints.size() + 1]; + final double[] lons = new double[polyPoints.size() + 1]; + + for (int i = 0; i < polyPoints.size(); i++) { + lats[i] = polyPoints.get(i).getLatitude() * 180.0 / Math.PI; + lons[i] = polyPoints.get(i).getLongitude() * 180.0 / Math.PI; + } + lats[polyPoints.size()] = lats[0]; + lons[polyPoints.size()] = lons[0]; + return new Polygon(lats, lons, holes); + } + + protected static List convertToPoints(final PlanetModel pm, final GeoPoint pole, final double[] angles, final double[] arcDistances) { + // To do the point rotations, we need the sine and cosine of the pole latitude and longitude. Get it here for performance. + final double sinLatitude = Math.sin(pole.getLatitude()); + final double cosLatitude = Math.cos(pole.getLatitude()); + final double sinLongitude = Math.sin(pole.getLongitude()); + final double cosLongitude = Math.cos(pole.getLongitude()); + final List rval = new ArrayList<>(); + for (int i = 0; i < angles.length; i++) { + rval.add(createPoint(pm, angles[i], arcDistances[i], sinLatitude, cosLatitude, sinLongitude, cosLongitude)); + } + return rval; + } + + protected static GeoPoint createPoint(final PlanetModel pm, + final double angle, + final double arcDistance, + final double sinLatitude, + final double cosLatitude, + final double sinLongitude, + final double cosLongitude) { + // From the angle and arc distance, convert to (x,y,z) in unit space. + // We want the perspective to be looking down the x axis. The "angle" measurement is thus in the Y-Z plane. + // The arcdistance is in X. + final double x = Math.cos(arcDistance); + final double yzScale = Math.sin(arcDistance); + final double y = Math.cos(angle) * yzScale; + final double z = Math.sin(angle) * yzScale; + // Now, rotate coordinates so that we shift everything from pole = x-axis to actual coordinates. + // This transformation should take the point (1,0,0) and transform it to the pole's actual (x,y,z) coordinates. + // Coordinate rotation formula: + // x1 = x0 cos T - y0 sin T + // y1 = x0 sin T + y0 cos T + // We're in essence undoing the following transformation (from GeoPolygonFactory): + // x1 = x0 cos az - y0 sin az + // y1 = x0 sin az + y0 cos az + // z1 = z0 + // x2 = x1 cos al - z1 sin al + // y2 = y1 + // z2 = x1 sin al + z1 cos al + // So, we reverse the order of the transformations, AND we transform backwards. + // Transforming backwards means using these identities: sin(-angle) = -sin(angle), cos(-angle) = cos(angle) + // So: + // x1 = x0 cos al + z0 sin al + // y1 = y0 + // z1 = - x0 sin al + z0 cos al + // x2 = x1 cos az + y1 sin az + // y2 = - x1 sin az + y1 cos az + // z2 = z1 + final double x1 = x * cosLatitude + z * sinLatitude; + final double y1 = y; + final double z1 = - x * sinLatitude + z * cosLatitude; + final double x2 = x1 * cosLongitude + y1 * sinLongitude; + final double y2 = - x1 * sinLongitude + y1 * cosLongitude; + final double z2 = z1; + + // Scale final (x,y,z) to land on planet surface + // Equation of ellipsoid: x^2 / a^2 + y^2 / b^2 + z^2 / c^2 - 1 = 0 + // Use a parameterization, e.g. x = t * x2, y = t * y2, z = t * z2, and find t. + // t^2 ( x2^2 / a^2 + y2^2 / b^2 + z2^2 / c^2 ) = 1 + // t = +/- sqrt( 1 / ( x2^2 / a^2 + y2^2 / b^2 + z2^2 / c^2 ) ) + // We want the + variant because we're scaling in the same direction as the original vector. + final double t = Math.sqrt( 1.0 / (x2 * x2 * pm.inverseAbSquared + y2 * y2 * pm.inverseAbSquared + z2 * z2 * pm.inverseCSquared)); + return new GeoPoint(x2 * t, y2 * t, z2 * t); + } + + protected static boolean verifyPolygon(final PlanetModel pm, final Polygon polygon, final GeoPolygon outsidePolygon) { + // Each point in the new poly should be inside the outside poly, and each edge should not intersect the outside poly edge + final double[] lats = polygon.getPolyLats(); + final double[] lons = polygon.getPolyLons(); + final List polyPoints = new ArrayList<>(lats.length-1); + for (int i = 0; i < lats.length - 1; i++) { + final GeoPoint newPoint = new GeoPoint(pm, Math.toRadians(lats[i]), Math.toRadians(lons[i])); + if (!outsidePolygon.isWithin(newPoint)) { + return false; + } + polyPoints.add(newPoint); + } + // We don't need to construct the world to find intersections -- just the bordering planes. + for (int planeIndex = 0; planeIndex < polyPoints.size(); planeIndex++) { + final GeoPoint startPoint = polyPoints.get(planeIndex); + final GeoPoint endPoint = polyPoints.get(legalIndex(planeIndex + 1, polyPoints.size())); + final GeoPoint beforeStartPoint = polyPoints.get(legalIndex(planeIndex - 1, polyPoints.size())); + final GeoPoint afterEndPoint = polyPoints.get(legalIndex(planeIndex + 2, polyPoints.size())); + final SidedPlane beforePlane = new SidedPlane(endPoint, beforeStartPoint, startPoint); + final SidedPlane afterPlane = new SidedPlane(startPoint, endPoint, afterEndPoint); + final Plane plane = new Plane(startPoint, endPoint); + + // Check for intersections!! + if (outsidePolygon.intersects(plane, null, beforePlane, afterPlane)) { + return false; + } + } + return true; + } + + protected static int legalIndex(int index, int size) { + if (index > size) { + index -= size; + } + if (index < 0) { + index += size; + } + return index; + } } From 69514992bcafc0b3901ade1417ff6e6d0f7f992f Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Wed, 6 Apr 2016 17:17:50 -0400 Subject: [PATCH 0290/1113] LUCENE-7173: Begin adding tests for random polygons with nesting. Also found and fixed a tiling problem. --- .../spatial3d/geom/GeoPolygonFactory.java | 169 +++++++++++++++--- .../lucene/spatial3d/TestGeo3DPoint.java | 15 +- .../lucene/spatial3d/geom/GeoPolygonTest.java | 42 +++++ 3 files changed, 201 insertions(+), 25 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 83c95b646ea3..fdd51c5744ed 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -64,10 +64,11 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final List holes) { // The basic operation uses a set of points, two points determining one particular edge, and a sided plane // describing membership. - return buildPolygonShape(planetModel, pointList, convexPointIndex, getLegalIndex(convexPointIndex + 1, pointList.size()), + return buildPolygonShape(new GeoCompositePolygon(), + planetModel, pointList, new BitSet(), + convexPointIndex, getLegalIndex(convexPointIndex + 1, pointList.size()), new SidedPlane(pointList.get(getLegalIndex(convexPointIndex - 1, pointList.size())), pointList.get(convexPointIndex), pointList.get(getLegalIndex(convexPointIndex + 1, pointList.size()))), - false, holes, null); } @@ -137,15 +138,15 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final SidedPlane initialPlane = new SidedPlane(testPoint, pointList.get(0), pointList.get(1)); // We don't know if this is the correct siding choice. We will only know as we build the complex polygon. // So we need to be prepared to try both possibilities. - final GeoPolygon trial = buildPolygonShape(planetModel, pointList, 0, 1, initialPlane, false, holes, testPoint); + final GeoPolygon trial = buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, testPoint); if (trial == null) { // The testPoint was within the shape. Was that intended? if (testPointInside) { // Yes: build it for real - return buildPolygonShape(planetModel, pointList, 0, 1, initialPlane, false, holes, null); + return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, null); } // No: do the complement and return that. - return buildPolygonShape(planetModel, pointList, 0, 1, new SidedPlane(initialPlane), false, holes, null); + return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); } else { // The testPoint was outside the shape. Was that intended? if (!testPointInside) { @@ -153,7 +154,7 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, return trial; } // No: return the complement - return buildPolygonShape(planetModel, pointList, 0, 1, new SidedPlane(initialPlane), false, holes, null); + return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); } } @@ -280,10 +281,12 @@ protected static Double computeAngle(final GeoPoint point, } /** Build a GeoPolygon out of one concave part and multiple convex parts given points, starting edge, and whether starting edge is internal or not. - * @param pointsList is a list of the GeoPoints to build an arbitrary polygon out of. + * @param rval is the composite polygon to add to. + * @param planetModel is the planet model. + * @param pointsList is a list of the GeoPoints to build an arbitrary polygon out of. + * @param internalEdges specifies which edges are internal. * @param startPointIndex is the first of the points, constituting the starting edge. * @param startingEdge is the plane describing the starting edge. - * @param isInternalEdge is true if the specified edge is an internal one. * @param holes is the list of holes in the polygon, or null if none. * @param testPoint is an (optional) test point, which will be used to determine if we are generating * a shape with the proper sidedness. It is passed in only when the test point is supposed to be outside @@ -293,18 +296,19 @@ protected static Double computeAngle(final GeoPoint point, * which result to use. If the test point is supposed to be within the shape, then it must be outside of the * complement shape. If the test point is supposed to be outside the shape, then it must be outside of the * original shape. Either way, we can figure out the right thing to use. - * @return a GeoMembershipShape corresponding to what was specified, or null if what was specified + * @return the GeoPolygon passed in in the rval parameter, or null if what was specified * was inconsistent with what we generated. Specifically, if we specify an exterior point that is * found in the interior of the shape we create here we return null, which is a signal that we chose * our initial plane sidedness backwards. */ public static GeoPolygon buildPolygonShape( + final GeoCompositePolygon rval, final PlanetModel planetModel, final List pointsList, + final BitSet internalEdges, final int startPointIndex, final int endPointIndex, final SidedPlane startingEdge, - final boolean isInternalEdge, final List holes, final GeoPoint testPoint) { @@ -320,11 +324,7 @@ public static GeoPolygon buildPolygonShape( // convex polygon. That internal edge is used to extend the list of edges in the concave polygon edge list. // The edge buffer. - final EdgeBuffer edgeBuffer = new EdgeBuffer(pointsList, startPointIndex, endPointIndex, startingEdge, isInternalEdge); - - // Current composite. This is what we'll actually be returning. This will have a number of convex polygons, and - // maybe a single concave one too. - final GeoCompositePolygon rval = new GeoCompositePolygon(); + final EdgeBuffer edgeBuffer = new EdgeBuffer(pointsList, internalEdges, startPointIndex, endPointIndex, startingEdge); // Starting state: // The stopping point @@ -363,6 +363,130 @@ public static GeoPolygon buildPolygonShape( } } + // Look for any reason that the concave polygon cannot be created. + // This test is really the converse of the one for a convex polygon. + // Points on the edge of a convex polygon MUST be inside all the other + // edges. For a concave polygon, this check is still the same, except we have + // to look at the reverse sided planes, not the forward ones. + + // If we find a point that is outside of the complementary edges, it means that + // the point is in fact able to form a convex polygon with the edge it is + // offending. + + // If what is left has any plane/point pair that is on the wrong side, we have to split using one of the plane endpoints and the + // point in question. This is best structured as a recursion, if detected. + final Iterator checkIterator = edgeBuffer.iterator(); + while (checkIterator.hasNext()) { + final Edge checkEdge = checkIterator.next(); + final SidedPlane flippedPlane = new SidedPlane(checkEdge.plane); + // Now walk around again looking for points that fail + final Iterator confirmIterator = edgeBuffer.iterator(); + while (confirmIterator.hasNext()) { + final Edge confirmEdge = confirmIterator.next(); + if (confirmEdge == checkEdge) { + continue; + } + final GeoPoint thePoint; + if (checkEdge.startPoint != confirmEdge.startPoint && checkEdge.endPoint != confirmEdge.startPoint && !flippedPlane.isWithin(confirmEdge.startPoint)) { + thePoint = confirmEdge.startPoint; + } else if (checkEdge.startPoint != confirmEdge.endPoint && checkEdge.endPoint != confirmEdge.endPoint && !flippedPlane.isWithin(confirmEdge.endPoint)) { + thePoint = confirmEdge.endPoint; + } else { + thePoint = null; + } + if (thePoint != null) { + // Found a split!! + + // This should be the only problematic part of the polygon. + // We know that thePoint is on the "wrong" side of the edge -- that is, it's on the side that the + // edge is pointing at. + final List thirdPartPoints = new ArrayList<>(); + final BitSet thirdPartInternal = new BitSet(); + thirdPartPoints.add(checkEdge.startPoint); + thirdPartInternal.set(0, checkEdge.isInternal); + thirdPartPoints.add(checkEdge.endPoint); + thirdPartInternal.set(1, true); + thirdPartPoints.add(thePoint); + thirdPartInternal.set(2, true); + //System.out.println("Doing convex part..."); + final GeoPolygon thirdPoly = buildPolygonShape(rval, + planetModel, + thirdPartPoints, + thirdPartInternal, + 0, + 1, + checkEdge.plane, + holes, + testPoint); + //System.out.println("...done convex part."); + if (thirdPoly == null) { + return null; + } + + // The part preceding the bad edge, back to thePoint, needs to be recursively + // processed. So, assemble what we need, which is basically a list of edges. + Edge loopEdge = edgeBuffer.getPrevious(checkEdge); + final List firstPartPoints = new ArrayList<>(); + final BitSet firstPartInternal = new BitSet(); + int i = 0; + while (true) { + firstPartPoints.add(loopEdge.endPoint); + if (loopEdge.endPoint == thePoint) { + break; + } + firstPartInternal.set(i++, loopEdge.isInternal); + loopEdge = edgeBuffer.getPrevious(loopEdge); + } + firstPartInternal.set(i, true); + //System.out.println("Doing first part..."); + final GeoPolygon firstPoly = buildPolygonShape(rval, + planetModel, + firstPartPoints, + firstPartInternal, + firstPartPoints.size()-1, + 0, + new SidedPlane(checkEdge.endPoint, false, checkEdge.startPoint, thePoint), + holes, + testPoint); + //System.out.println("...done first part."); + if (firstPoly == null) { + return null; + } + final List secondPartPoints = new ArrayList<>(); + final BitSet secondPartInternal = new BitSet(); + loopEdge = edgeBuffer.getNext(checkEdge); + i = 0; + while (true) { + secondPartPoints.add(loopEdge.startPoint); + if (loopEdge.startPoint == thePoint) { + break; + } + secondPartInternal.set(i++, loopEdge.isInternal); + loopEdge = edgeBuffer.getNext(loopEdge); + } + secondPartInternal.set(i, true); + //System.out.println("Doing second part..."); + final GeoPolygon secondPoly = buildPolygonShape(rval, + planetModel, + secondPartPoints, + secondPartInternal, + secondPartPoints.size()-1, + 0, + new SidedPlane(checkEdge.endPoint, true, checkEdge.startPoint, thePoint), + holes, + testPoint); + //System.out.println("... done second part"); + if (secondPoly == null) { + return null; + } + + return rval; + } + } + } + + // No violations found: we know it's a legal concave polygon. + // If there's anything left in the edge buffer, convert to concave polygon. if (makeConcavePolygon(planetModel, rval, edgeBuffer, holes, testPoint) == false) { return null; @@ -399,9 +523,11 @@ protected static boolean makeConcavePolygon(final PlanetModel planetModel, final List points = new ArrayList(edgeBuffer.size()); final BitSet internalEdges = new BitSet(edgeBuffer.size()-1); + //System.out.println("Concave polygon points:"); Edge edge = edgeBuffer.pickOne(); boolean isInternal = false; for (int i = 0; i < edgeBuffer.size(); i++) { + //System.out.println(" "+edge.plane+": "+edge.startPoint+"->"+edge.endPoint+"; previous? "+(edge.plane.isWithin(edgeBuffer.getPrevious(edge).startPoint)?"in":"out")+" next? "+(edge.plane.isWithin(edgeBuffer.getNext(edge).endPoint)?"in":"out")); points.add(edge.startPoint); if (i < edgeBuffer.size() - 1) { internalEdges.set(i, edge.isInternal); @@ -784,29 +910,24 @@ protected static class EdgeBuffer { /** Constructor. * @param pointList is the list of points. + * @param internalEdges is the list of edges that are internal (includes return edge) * @param startPlaneStartIndex is the index of the startPlane's starting point * @param startPlaneEndIndex is the index of the startPlane's ending point * @param startPlane is the starting plane - * @param startPlaneIsInternal signals whether the startPlane is an internal edge */ - public EdgeBuffer(final List pointList, final int startPlaneStartIndex, final int startPlaneEndIndex, final SidedPlane startPlane, final boolean startPlaneIsInternal) { + public EdgeBuffer(final List pointList, final BitSet internalEdges, final int startPlaneStartIndex, final int startPlaneEndIndex, final SidedPlane startPlane) { /* System.out.println("Initial points:"); for (final GeoPoint p : pointList) { System.out.println(" "+p); } - System.out.println("For start plane, the following points are in/out:"); - for (final GeoPoint p: pointList) { - System.out.println(" "+p+" is: "+(startPlane.isWithin(p)?"in":"out")); - } */ - final Edge startEdge = new Edge(pointList.get(startPlaneStartIndex), pointList.get(startPlaneEndIndex), startPlane, startPlaneIsInternal); + final Edge startEdge = new Edge(pointList.get(startPlaneStartIndex), pointList.get(startPlaneEndIndex), startPlane, internalEdges.get(startPlaneStartIndex)); // Fill in the EdgeBuffer by walking around creating more stuff Edge currentEdge = startEdge; int startIndex = startPlaneStartIndex; int endIndex = startPlaneEndIndex; - boolean isInternal = startPlaneIsInternal; while (true) { // Compute the next edge startIndex = endIndex; @@ -825,7 +946,7 @@ public EdgeBuffer(final List pointList, final int startPlaneStartIndex System.out.println(" "+p+" is: "+(newPlane.isWithin(p)?"in":"out")); } */ - final Edge newEdge = new Edge(pointList.get(startIndex), pointList.get(endIndex), newPlane, false); + final Edge newEdge = new Edge(pointList.get(startIndex), pointList.get(endIndex), newPlane, internalEdges.get(startIndex)); // Link it in previousEdges.put(newEdge, currentEdge); diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 01f5e8306450..5453c33327ed 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -69,6 +69,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.junit.BeforeClass; +import org.junit.Ignore; import com.carrotsearch.randomizedtesting.generators.RandomInts; @@ -826,6 +827,18 @@ public void testEquals() { assertFalse(q.equals(Geo3DPoint.newShapeQuery("point", shape2))); } + @Ignore + public void testComplexPolygons() { + final PlanetModel pm = PlanetModel.WGS84; + // Pick a random pole + final GeoPoint randomPole = new GeoPoint(pm, Math.toRadians(randomLat()), Math.toRadians(randomLon())); + // Create a polygon that's less than 180 degrees + final Polygon clockWise = makePoly(pm, randomPole, true); + // Create a polygon that's greater than 180 degrees + final Polygon counterClockWise = makePoly(pm, randomPole, false); + + } + protected static double MINIMUM_EDGE_ANGLE = Math.toRadians(5.0); protected static double MINIMUM_ARC_ANGLE = Math.toRadians(1.0); @@ -865,7 +878,7 @@ protected Polygon makePoly(final PlanetModel pm, final GeoPoint pole, final bool accumulatedAngle += angle; } // Pick the arc distance randomly - arcDistance[i] = random().nextDouble() * (Math.PI * 0.5 - MINIMUM_ARC_ANGLE) + MINIMUM_ARC_ANGLE; + arcDistance[i] = random().nextDouble() * (Math.PI - MINIMUM_ARC_ANGLE) + MINIMUM_ARC_ANGLE; } if (clockwiseDesired) { // Reverse the signs diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java index 9f2bea41b484..33840dab1d92 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java @@ -336,4 +336,46 @@ public void testGeoConcaveRelationshipCase1() { assertTrue(xyzSolid.getRelationship(c) == GeoArea.OVERLAPS); } + @Test + public void testPolygonFactoryCase1() { + /* + [junit4] 1> Initial points: + [junit4] 1> [X=-0.17279348371564082, Y=0.24422965662722748, Z=0.9521675605930696] + [junit4] 1> [X=-0.6385022730019092, Y=-0.6294493901210775, Z=0.4438687423720006] + [junit4] 1> [X=-0.9519561011293354, Y=-0.05324061687857965, Z=-0.30423702782227385] + [junit4] 1> [X=-0.30329807815178533, Y=-0.9447434167936289, Z=0.13262941042055737] + [junit4] 1> [X=-0.5367607140926697, Y=0.8179452639396644, Z=0.21163783898691005] + [junit4] 1> [X=0.39285411191111597, Y=0.6369575362013932, Z=0.6627439307500357] + [junit4] 1> [X=-0.44715655239362595, Y=0.8332957749253644, Z=0.3273923501593971] + [junit4] 1> [X=0.33024322515264537, Y=0.6945246730529289, Z=0.6387986432043298] + [junit4] 1> [X=-0.1699323603224724, Y=0.8516746480592872, Z=0.4963385521664198] + [junit4] 1> [X=0.2654788898359613, Y=0.7380222309164597, Z=0.6200740473100581] + [junit4] 1> For start plane, the following points are in/out: + [junit4] 1> [X=-0.17279348371564082, Y=0.24422965662722748, Z=0.9521675605930696] is: in + [junit4] 1> [X=-0.6385022730019092, Y=-0.6294493901210775, Z=0.4438687423720006] is: in + [junit4] 1> [X=-0.9519561011293354, Y=-0.05324061687857965, Z=-0.30423702782227385] is: out + [junit4] 1> [X=-0.30329807815178533, Y=-0.9447434167936289, Z=0.13262941042055737] is: in + [junit4] 1> [X=-0.5367607140926697, Y=0.8179452639396644, Z=0.21163783898691005] is: out + [junit4] 1> [X=0.39285411191111597, Y=0.6369575362013932, Z=0.6627439307500357] is: in + [junit4] 1> [X=-0.44715655239362595, Y=0.8332957749253644, Z=0.3273923501593971] is: out + [junit4] 1> [X=0.33024322515264537, Y=0.6945246730529289, Z=0.6387986432043298] is: in + [junit4] 1> [X=-0.1699323603224724, Y=0.8516746480592872, Z=0.4963385521664198] is: out + [junit4] 1> [X=0.2654788898359613, Y=0.7380222309164597, Z=0.6200740473100581] is: out + */ + + final List points = new ArrayList<>(); + points.add(new GeoPoint(0.17279348371564082, 0.24422965662722748, 0.9521675605930696)); + points.add(new GeoPoint(-0.6385022730019092, -0.6294493901210775, 0.4438687423720006)); + points.add(new GeoPoint(-0.9519561011293354, -0.05324061687857965, -0.30423702782227385)); + points.add(new GeoPoint(-0.30329807815178533, -0.9447434167936289, 0.13262941042055737)); + points.add(new GeoPoint(-0.5367607140926697, 0.8179452639396644, 0.21163783898691005)); + points.add(new GeoPoint(0.39285411191111597, 0.6369575362013932, 0.6627439307500357)); + points.add(new GeoPoint(-0.44715655239362595, 0.8332957749253644, 0.3273923501593971)); + points.add(new GeoPoint(0.33024322515264537, 0.6945246730529289, 0.6387986432043298)); + points.add(new GeoPoint(-0.1699323603224724, 0.8516746480592872, 0.4963385521664198)); + points.add(new GeoPoint(0.2654788898359613, 0.7380222309164597, 0.6200740473100581)); + + final GeoPolygon p = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, points, null); + } + } From a3ea71e04eed0bca0a33fda46206b77dc861aacd Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Wed, 6 Apr 2016 18:57:00 -0400 Subject: [PATCH 0291/1113] LUCENE-7173: Get the random nested polygon code working. --- .../lucene/spatial3d/TestGeo3DPoint.java | 162 ++++++++++-------- 1 file changed, 86 insertions(+), 76 deletions(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 5453c33327ed..d0d3388be485 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -827,15 +827,14 @@ public void testEquals() { assertFalse(q.equals(Geo3DPoint.newShapeQuery("point", shape2))); } - @Ignore public void testComplexPolygons() { final PlanetModel pm = PlanetModel.WGS84; // Pick a random pole final GeoPoint randomPole = new GeoPoint(pm, Math.toRadians(randomLat()), Math.toRadians(randomLon())); // Create a polygon that's less than 180 degrees - final Polygon clockWise = makePoly(pm, randomPole, true); + final Polygon clockWise = makePoly(pm, randomPole, true, true); // Create a polygon that's greater than 180 degrees - final Polygon counterClockWise = makePoly(pm, randomPole, false); + final Polygon counterClockWise = makePoly(pm, randomPole, false, true); } @@ -847,7 +846,7 @@ public void testComplexPolygons() { * doesn't do it because it's almost impossible to come up with nested ones of the proper * clockwise/counterclockwise rotation that way. */ - protected Polygon makePoly(final PlanetModel pm, final GeoPoint pole, final boolean clockwiseDesired) { + protected Polygon makePoly(final PlanetModel pm, final GeoPoint pole, final boolean clockwiseDesired, final boolean createHoles) { // Polygon edges will be arranged around the provided pole, and holes will each have a pole selected within the parent // polygon. final int pointCount = TestUtil.nextInt(random(), 3, 10); @@ -857,86 +856,97 @@ protected Polygon makePoly(final PlanetModel pm, final GeoPoint pole, final bool // These are all picked in the context of the pole, final double[] angles = new double[pointCount]; final double[] arcDistance = new double[pointCount]; - double accumulatedAngle = 0.0; - for (int i = 0; i < pointCount; i++) { - final int remainingEdgeCount = pointCount - i; - final double remainingAngle = 2.0 * Math.PI - accumulatedAngle; - if (remainingEdgeCount == 1) { - angles[i] = remainingAngle; - } else { - // The maximum angle is 180 degrees, or what's left when you give a minimal amount to each edge. - double maximumAngle = remainingAngle - (remainingEdgeCount-1) * MINIMUM_EDGE_ANGLE; - if (maximumAngle > Math.PI) { - maximumAngle = Math.PI; - } - // The minimum angle is MINIMUM_EDGE_ANGLE, or enough to be sure nobody afterwards needs more than - // 180 degrees. And since we have three points to start with, we already know that. - final double minimumAngle = MINIMUM_EDGE_ANGLE; - // Pick the angle - final double angle = random().nextDouble() * (maximumAngle - minimumAngle) + minimumAngle; - angles[i] = angle; - accumulatedAngle += angle; - } - // Pick the arc distance randomly - arcDistance[i] = random().nextDouble() * (Math.PI - MINIMUM_ARC_ANGLE) + MINIMUM_ARC_ANGLE; - } - if (clockwiseDesired) { - // Reverse the signs + // Pick a set of points + while (true) { + double accumulatedAngle = 0.0; for (int i = 0; i < pointCount; i++) { - angles[i] = -angles[i]; + final int remainingEdgeCount = pointCount - i; + final double remainingAngle = 2.0 * Math.PI - accumulatedAngle; + if (remainingEdgeCount == 1) { + angles[i] = remainingAngle; + } else { + // The maximum angle is 180 degrees, or what's left when you give a minimal amount to each edge. + double maximumAngle = remainingAngle - (remainingEdgeCount-1) * MINIMUM_EDGE_ANGLE; + if (maximumAngle > Math.PI) { + maximumAngle = Math.PI; + } + // The minimum angle is MINIMUM_EDGE_ANGLE, or enough to be sure nobody afterwards needs more than + // 180 degrees. And since we have three points to start with, we already know that. + final double minimumAngle = MINIMUM_EDGE_ANGLE; + // Pick the angle + final double angle = random().nextDouble() * (maximumAngle - minimumAngle) + minimumAngle; + angles[i] = angle; + accumulatedAngle += angle; + } + // Pick the arc distance randomly + arcDistance[i] = random().nextDouble() * (Math.PI - MINIMUM_ARC_ANGLE) + MINIMUM_ARC_ANGLE; } - } - - // Now, use the pole's information plus angles and arcs to create GeoPoints in the right order. - final List polyPoints = convertToPoints(pm, pole, angles, arcDistance); - - // Create the geo3d polygon, so we can test out our poles. - GeoPolygon poly = GeoPolygonFactory.makeGeoPolygon(pm, polyPoints, null); - - // Next, do some holes. No more than 2 of these. The poles for holes must always be within the polygon, so we're - // going to use Geo3D to help us select those given the points we just made. - - final int holeCount = TestUtil.nextInt(random(), 0, 2); - - final Polygon[] holes = new Polygon[holeCount]; - - for (int i = 0; i < holeCount; i++) { - // Choose a pole. The poly has to be within the polygon, but it also cannot be on the polygon edge. - // We try indefinitely to find a good pole... - while (true) { - final GeoPoint poleChoice = new GeoPoint(pm, toRadians(randomLat()), toRadians(randomLon())); - if (!poly.isWithin(poleChoice)) { - continue; + if (clockwiseDesired) { + // Reverse the signs + for (int i = 0; i < pointCount; i++) { + angles[i] = -angles[i]; } - // We have a pole within the polygon. Now try 100 times to build a polygon that does not intersect the outside ring. - // After that we give up and pick a new pole. - boolean foundOne = false; - for (int j = 0; j < 100; j++) { - final Polygon insidePoly = makePoly(pm, poleChoice, !clockwiseDesired); - // Verify that the inside polygon is OK. If not, discard and repeat. - if (!verifyPolygon(pm, insidePoly, poly)) { + } + + // Now, use the pole's information plus angles and arcs to create GeoPoints in the right order. + final List polyPoints = convertToPoints(pm, pole, angles, arcDistance); + + // Create the geo3d polygon, so we can test out our poles. + final GeoPolygon poly; + try { + poly = GeoPolygonFactory.makeGeoPolygon(pm, polyPoints, null); + } catch (IllegalArgumentException e) { + // This is what happens when three adjacent points are colinear, so try again. + continue; + } + + // Next, do some holes. No more than 2 of these. The poles for holes must always be within the polygon, so we're + // going to use Geo3D to help us select those given the points we just made. + + final int holeCount = createHoles?TestUtil.nextInt(random(), 0, 2):0; + + final List holeList = new ArrayList<>(); + + for (int i = 0; i < holeCount; i++) { + // Choose a pole. The poly has to be within the polygon, but it also cannot be on the polygon edge. + // If we can't find a good pole we have to give it up and not do the hole. + for (int k = 0; k < 500; k++) { + final GeoPoint poleChoice = new GeoPoint(pm, toRadians(randomLat()), toRadians(randomLon())); + if (!poly.isWithin(poleChoice)) { continue; } - holes[i] = insidePoly; - foundOne = true; - } - if (foundOne) { - break; + // We have a pole within the polygon. Now try 100 times to build a polygon that does not intersect the outside ring. + // After that we give up and pick a new pole. + boolean foundOne = false; + for (int j = 0; j < 100; j++) { + final Polygon insidePoly = makePoly(pm, poleChoice, !clockwiseDesired, false); + // Verify that the inside polygon is OK. If not, discard and repeat. + if (!verifyPolygon(pm, insidePoly, poly)) { + continue; + } + holeList.add(insidePoly); + foundOne = true; + } + if (foundOne) { + break; + } } } - } - - // Finally, build the polygon and return it - final double[] lats = new double[polyPoints.size() + 1]; - final double[] lons = new double[polyPoints.size() + 1]; + + final Polygon[] holes = holeList.toArray(new Polygon[0]); - for (int i = 0; i < polyPoints.size(); i++) { - lats[i] = polyPoints.get(i).getLatitude() * 180.0 / Math.PI; - lons[i] = polyPoints.get(i).getLongitude() * 180.0 / Math.PI; + // Finally, build the polygon and return it + final double[] lats = new double[polyPoints.size() + 1]; + final double[] lons = new double[polyPoints.size() + 1]; + + for (int i = 0; i < polyPoints.size(); i++) { + lats[i] = polyPoints.get(i).getLatitude() * 180.0 / Math.PI; + lons[i] = polyPoints.get(i).getLongitude() * 180.0 / Math.PI; + } + lats[polyPoints.size()] = lats[0]; + lons[polyPoints.size()] = lons[0]; + return new Polygon(lats, lons, holes); } - lats[polyPoints.size()] = lats[0]; - lons[polyPoints.size()] = lons[0]; - return new Polygon(lats, lons, holes); } protected static List convertToPoints(final PlanetModel pm, final GeoPoint pole, final double[] angles, final double[] arcDistances) { @@ -1035,7 +1045,7 @@ protected static boolean verifyPolygon(final PlanetModel pm, final Polygon polyg } protected static int legalIndex(int index, int size) { - if (index > size) { + if (index >= size) { index -= size; } if (index < 0) { From 6868a8cd74d2d21dd679dfbb867d0823253ac2a5 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Wed, 6 Apr 2016 19:18:56 -0400 Subject: [PATCH 0292/1113] LUCENE-7168: improve encode and quantization testing for geo3d --- lucene/CHANGES.txt | 3 + .../apache/lucene/document/LatLonPoint.java | 4 +- .../apache/lucene/spatial3d/Geo3DPoint.java | 5 +- .../apache/lucene/spatial3d/Geo3DUtil.java | 75 +- .../spatial3d/PointInGeo3DShapeQuery.java | 71 +- .../PointInShapeIntersectVisitor.java | 107 +++ .../lucene/spatial3d/TestGeo3DPoint.java | 682 ++++++++++++------ 7 files changed, 622 insertions(+), 325 deletions(-) create mode 100644 lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index da5be98b8d71..f28254dc460c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -63,6 +63,9 @@ Bug Fixes * LUCENE-7166: Fix corner case bugs in LatLonPoint/GeoPointField bounding box queries. (Robert Muir) +* LUCENE-7168: Switch to stable encode for geo3d, remove quantization + test leniency, remove dead code (Mike McCandless) + Other * LUCENE-7174: Upgrade randomizedtesting to 2.3.4. (Uwe Schindler, Dawid Weiss) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 107d83cb8772..b2c39c21b7fd 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -199,7 +199,7 @@ public static int encodeLongitudeCeil(double longitude) { */ public static double decodeLatitude(int encoded) { double result = encoded * LATITUDE_DECODE; - assert result >= GeoUtils.MIN_LAT_INCL && result <= GeoUtils.MAX_LAT_INCL; + assert result >= GeoUtils.MIN_LAT_INCL && result < GeoUtils.MAX_LAT_INCL; return result; } @@ -220,7 +220,7 @@ public static double decodeLatitude(byte[] src, int offset) { */ public static double decodeLongitude(int encoded) { double result = encoded * LONGITUDE_DECODE; - assert result >= GeoUtils.MIN_LON_INCL && result <= GeoUtils.MAX_LON_INCL; + assert result >= GeoUtils.MIN_LON_INCL && result < GeoUtils.MAX_LON_INCL; return result; } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index be8990f95081..1e07aa715073 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.geo.Polygon; import org.apache.lucene.geo.GeoUtils; -import org.apache.lucene.spatial3d.geom.Vector; import org.apache.lucene.spatial3d.geom.GeoPoint; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; @@ -231,12 +230,12 @@ private void fillFieldsData(double x, double y, double z) { /** Encode single dimension */ public static void encodeDimension(double value, byte bytes[], int offset) { - NumericUtils.intToSortableBytes(Geo3DUtil.encodeValue(PlanetModel.WGS84.getMaximumMagnitude(), value), bytes, offset); + NumericUtils.intToSortableBytes(Geo3DUtil.encodeValue(value), bytes, offset); } /** Decode single dimension */ public static double decodeDimension(byte value[], int offset) { - return Geo3DUtil.decodeValueCenter(PlanetModel.WGS84.getMaximumMagnitude(), NumericUtils.sortableBytesToInt(value, offset)); + return Geo3DUtil.decodeValue(NumericUtils.sortableBytesToInt(value, offset)); } /** Returns a query matching all points inside the provided shape. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java index 0a0bf30039c8..cc11294e6c2d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java @@ -16,44 +16,59 @@ */ package org.apache.lucene.spatial3d; +import org.apache.lucene.spatial3d.geom.PlanetModel; + class Geo3DUtil { - /** Clips the incoming value to the allowed min/max range before encoding, instead of throwing an exception. */ - public static int encodeValueLenient(double planetMax, double x) { - if (x > planetMax) { - x = planetMax; - } else if (x < -planetMax) { - x = -planetMax; - } - return encodeValue(planetMax, x); - } + private static final double MAX_VALUE = PlanetModel.WGS84.getMaximumMagnitude(); + private static final int BITS = 32; + private static final double MUL = (0x1L< planetMax) { - throw new IllegalArgumentException("value=" + x + " is out-of-bounds (greater than planetMax=" + planetMax + ")"); + public static int encodeValue(double x) { + if (x > MAX_VALUE) { + throw new IllegalArgumentException("value=" + x + " is out-of-bounds (greater than WGS84's planetMax=" + MAX_VALUE + ")"); } - if (x < -planetMax) { - throw new IllegalArgumentException("value=" + x + " is out-of-bounds (less than than -planetMax=" + -planetMax + ")"); + if (x < -MAX_VALUE) { + throw new IllegalArgumentException("value=" + x + " is out-of-bounds (less than than WGS84's -planetMax=" + -MAX_VALUE + ")"); } - long y = Math.round (x * (Integer.MAX_VALUE / planetMax)); - assert y >= Integer.MIN_VALUE; - assert y <= Integer.MAX_VALUE; - - return (int) y; + long result = (long) Math.floor(x / DECODE); + //System.out.println(" enc: " + x + " -> " + result); + assert result >= Integer.MIN_VALUE; + assert result <= Integer.MAX_VALUE; + return (int) result; } - /** Center decode */ - public static double decodeValueCenter(double planetMax, int x) { - return x * (planetMax / Integer.MAX_VALUE); + public static double decodeValue(int x) { + double result; + if (x == MIN_ENCODED_VALUE) { + // We must special case this, because -MAX_VALUE is not guaranteed to land precisely at a floor value, and we don't ever want to + // return a value outside of the planet's range (I think?). The max value is "safe" because we floor during encode: + result = -MAX_VALUE; + } else { + result = x * DECODE; + } + assert result >= -MAX_VALUE && result <= MAX_VALUE; + return result; } - /** More negative decode, at bottom of cell */ - public static double decodeValueMin(double planetMax, int x) { - return (((double)x) - 0.5) * (planetMax / Integer.MAX_VALUE); - } - - /** More positive decode, at top of cell */ - public static double decodeValueMax(double planetMax, int x) { - return (((double)x) + 0.5) * (planetMax / Integer.MAX_VALUE); + /** Returns a double value >= x such that if you multiply that value by an int, and then + * divide it by that int again, you get precisely the same value back */ + private static double getNextSafeDouble(double x) { + + // Move to double space: + long bits = Double.doubleToLongBits(x); + + // Make sure we are beyond the actual maximum value: + bits += Integer.MAX_VALUE; + + // Clear the bottom 32 bits: + bits &= ~((long) Integer.MAX_VALUE); + + // Convert back to double: + double result = Double.longBitsToDouble(bits); + assert result > x; + return result; } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java index 3cc95304a67d..ef0fe47f1e13 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInGeo3DShapeQuery.java @@ -19,13 +19,9 @@ import java.io.IOException; import org.apache.lucene.spatial3d.geom.BasePlanetObject; -import org.apache.lucene.spatial3d.geom.GeoArea; -import org.apache.lucene.spatial3d.geom.GeoAreaFactory; import org.apache.lucene.spatial3d.geom.GeoShape; import org.apache.lucene.spatial3d.geom.PlanetModel; -import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; @@ -35,7 +31,6 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.util.DocIdSetBuilder; -import org.apache.lucene.util.NumericUtils; /** Finds all previously indexed points that fall within the specified polygon. * @@ -98,73 +93,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException { assert xyzSolid.getRelationship(shape) == GeoArea.WITHIN || xyzSolid.getRelationship(shape) == GeoArea.OVERLAPS: "expected WITHIN (1) or OVERLAPS (2) but got " + xyzSolid.getRelationship(shape) + "; shape="+shape+"; XYZSolid="+xyzSolid; */ - double planetMax = PlanetModel.WGS84.getMaximumMagnitude(); - DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc()); - values.intersect(field, - new IntersectVisitor() { - - @Override - public void visit(int docID) { - result.add(docID); - } - - @Override - public void visit(int docID, byte[] packedValue) { - assert packedValue.length == 12; - double x = Geo3DPoint.decodeDimension(packedValue, 0); - double y = Geo3DPoint.decodeDimension(packedValue, Integer.BYTES); - double z = Geo3DPoint.decodeDimension(packedValue, 2 * Integer.BYTES); - if (shape.isWithin(x, y, z)) { - result.add(docID); - } - } - - @Override - public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - // Because the dimensional format operates in quantized (64 bit -> 32 bit) space, and the cell bounds - // here are inclusive, we need to extend the bounds to the largest un-quantized values that - // could quantize into these bounds. The encoding (Geo3DUtil.encodeValue) does - // a Math.round from double to long, so e.g. 1.4 -> 1, and -1.4 -> -1: - double xMin = Geo3DUtil.decodeValueMin(planetMax, NumericUtils.sortableBytesToInt(minPackedValue, 0)); - double xMax = Geo3DUtil.decodeValueMax(planetMax, NumericUtils.sortableBytesToInt(maxPackedValue, 0)); - double yMin = Geo3DUtil.decodeValueMin(planetMax, NumericUtils.sortableBytesToInt(minPackedValue, 1 * Integer.BYTES)); - double yMax = Geo3DUtil.decodeValueMax(planetMax, NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); - double zMin = Geo3DUtil.decodeValueMin(planetMax, NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); - double zMax = Geo3DUtil.decodeValueMax(planetMax, NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); - - //System.out.println(" compare: x=" + cellXMin + "-" + cellXMax + " y=" + cellYMin + "-" + cellYMax + " z=" + cellZMin + "-" + cellZMax); - assert xMin <= xMax; - assert yMin <= yMax; - assert zMin <= zMax; - - GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, xMin, xMax, yMin, yMax, zMin, zMax); - - switch(xyzSolid.getRelationship(shape)) { - case GeoArea.CONTAINS: - // Shape fully contains the cell - //System.out.println(" inside"); - return Relation.CELL_INSIDE_QUERY; - case GeoArea.OVERLAPS: - // They do overlap but neither contains the other: - //System.out.println(" crosses1"); - return Relation.CELL_CROSSES_QUERY; - case GeoArea.WITHIN: - // Cell fully contains the shape: - //System.out.println(" crosses2"); - // return Relation.SHAPE_INSIDE_CELL; - return Relation.CELL_CROSSES_QUERY; - case GeoArea.DISJOINT: - // They do not overlap at all - //System.out.println(" outside"); - return Relation.CELL_OUTSIDE_QUERY; - default: - assert false; - return Relation.CELL_CROSSES_QUERY; - } - } - }); + values.intersect(field, new PointInShapeIntersectVisitor(result, shape)); return new ConstantScoreScorer(this, score(), result.build().iterator()); } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java new file mode 100644 index 000000000000..25dea721c116 --- /dev/null +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.spatial3d; + +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.PlanetModel; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.NumericUtils; + +class PointInShapeIntersectVisitor implements IntersectVisitor { + private final DocIdSetBuilder hits; + private final GeoShape shape; + + public PointInShapeIntersectVisitor(DocIdSetBuilder hits, GeoShape shape) { + this.hits = hits; + this.shape = shape; + } + + @Override + public void visit(int docID) { + hits.add(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) { + assert packedValue.length == 12; + double x = Geo3DPoint.decodeDimension(packedValue, 0); + double y = Geo3DPoint.decodeDimension(packedValue, Integer.BYTES); + double z = Geo3DPoint.decodeDimension(packedValue, 2 * Integer.BYTES); + if (shape.isWithin(x, y, z)) { + hits.add(docID); + } + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + // Because the dimensional format operates in quantized (64 bit -> 32 bit) space, and the cell bounds + // here are inclusive, we need to extend the bounds to the largest un-quantized values that + // could quantize into these bounds. The encoding (Geo3DUtil.encodeValue) does + // a Math.round from double to long, so e.g. 1.4 -> 1, and -1.4 -> -1: + double xMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 0)); + double xMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 0)); + double yMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 1 * Integer.BYTES)); + double yMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); + double zMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); + double zMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); + + //System.out.println(" compare: x=" + cellXMin + "-" + cellXMax + " y=" + cellYMin + "-" + cellYMax + " z=" + cellZMin + "-" + cellZMax); + assert xMin <= xMax; + assert yMin <= yMax; + assert zMin <= zMax; + + GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, xMin, xMax, yMin, yMax, zMin, zMax); + + switch(xyzSolid.getRelationship(shape)) { + case GeoArea.CONTAINS: + // Shape fully contains the cell + //System.out.println(" inside"); + return Relation.CELL_INSIDE_QUERY; + case GeoArea.OVERLAPS: + // They do overlap but neither contains the other: + //System.out.println(" crosses1"); + return Relation.CELL_CROSSES_QUERY; + case GeoArea.WITHIN: + // Cell fully contains the shape: + //System.out.println(" crosses2"); + // return Relation.SHAPE_INSIDE_CELL; + return Relation.CELL_CROSSES_QUERY; + case GeoArea.DISJOINT: + // They do not overlap at all + //System.out.println(" outside"); + return Relation.CELL_OUTSIDE_QUERY; + default: + assert false; + return Relation.CELL_CROSSES_QUERY; + } + } + + /** More negative decode, at bottom of cell */ + static double decodeValueMin(int x) { + return (((double)x) - 0.5) * Geo3DUtil.DECODE; + } + + /** More positive decode, at top of cell */ + static double decodeValueMax(int x) { + return (((double)x) + 0.5) * Geo3DUtil.DECODE; + } +} diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index d0d3388be485..b751125bced6 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -20,11 +20,10 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; @@ -36,55 +35,52 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.spatial3d.geom.GeoArea; -import org.apache.lucene.spatial3d.geom.GeoAreaFactory; -import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; -import org.apache.lucene.spatial3d.geom.GeoCircleFactory; -import org.apache.lucene.spatial3d.geom.GeoPathFactory; -import org.apache.lucene.spatial3d.geom.GeoPoint; -import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; -import org.apache.lucene.spatial3d.geom.GeoShape; -import org.apache.lucene.spatial3d.geom.PlanetModel; -import org.apache.lucene.spatial3d.geom.XYZBounds; -import org.apache.lucene.spatial3d.geom.SidedPlane; -import org.apache.lucene.spatial3d.geom.Plane; -import org.apache.lucene.spatial3d.geom.GeoPolygon; +import org.apache.lucene.geo.GeoTestUtil; +import org.apache.lucene.geo.Polygon; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.spatial3d.geom.GeoArea; +import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; +import org.apache.lucene.spatial3d.geom.GeoCircleFactory; +import org.apache.lucene.spatial3d.geom.GeoPathFactory; +import org.apache.lucene.spatial3d.geom.GeoPoint; +import org.apache.lucene.spatial3d.geom.GeoPolygon; +import org.apache.lucene.spatial3d.geom.GeoPolygonFactory; +import org.apache.lucene.spatial3d.geom.GeoShape; +import org.apache.lucene.spatial3d.geom.Plane; +import org.apache.lucene.spatial3d.geom.PlanetModel; +import org.apache.lucene.spatial3d.geom.SidedPlane; +import org.apache.lucene.spatial3d.geom.XYZBounds; import org.apache.lucene.store.Directory; -import org.apache.lucene.geo.Polygon; +import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; -import org.junit.BeforeClass; import org.junit.Ignore; import com.carrotsearch.randomizedtesting.generators.RandomInts; public class TestGeo3DPoint extends LuceneTestCase { - private static boolean smallBBox; - - @BeforeClass - public static void beforeClass() { - smallBBox = random().nextBoolean(); - if (VERBOSE) { - System.err.println("TEST: smallBBox=" + smallBBox); - } - } - private static Codec getCodec() { if (Codec.getDefault().getName().equals("Lucene60")) { int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048); @@ -133,22 +129,7 @@ public void testBasic() throws Exception { } private static double toRadians(double degrees) { - return Math.PI*(degrees/180.0); - } - - private static PlanetModel getPlanetModel() { - if (random().nextBoolean()) { - // Use one of the earth models: - if (random().nextBoolean()) { - return PlanetModel.WGS84; - } else { - return PlanetModel.SPHERE; - } - } else { - // Make a randomly squashed planet: - double oblateness = random().nextDouble() * 0.5 - 0.25; - return new PlanetModel(1.0 + oblateness, 1.0 - oblateness); - } + return Math.toRadians(degrees); } private static class Cell { @@ -178,10 +159,10 @@ public Cell(Cell parent, } /** Returns true if the quantized point lies within this cell, inclusive on all bounds. */ - public boolean contains(double planetMax, GeoPoint point) { - int docX = Geo3DUtil.encodeValue(planetMax, point.x); - int docY = Geo3DUtil.encodeValue(planetMax, point.y); - int docZ = Geo3DUtil.encodeValue(planetMax, point.z); + public boolean contains(GeoPoint point) { + int docX = Geo3DUtil.encodeValue(point.x); + int docY = Geo3DUtil.encodeValue(point.y); + int docZ = Geo3DUtil.encodeValue(point.z); return docX >= xMinEnc && docX <= xMaxEnc && docY >= yMinEnc && docY <= yMaxEnc && @@ -194,17 +175,17 @@ public String toString() { } } - private static GeoPoint quantize(double planetMax, GeoPoint point) { - return new GeoPoint(Geo3DUtil.decodeValueCenter(planetMax, Geo3DUtil.encodeValue(planetMax, point.x)), - Geo3DUtil.decodeValueCenter(planetMax, Geo3DUtil.encodeValue(planetMax, point.y)), - Geo3DUtil.decodeValueCenter(planetMax, Geo3DUtil.encodeValue(planetMax, point.z))); + private static double quantize(double xyzValue) { + return Geo3DUtil.decodeValue(Geo3DUtil.encodeValue(xyzValue)); + } + + private static GeoPoint quantize(GeoPoint point) { + return new GeoPoint(quantize(point.x), quantize(point.y), quantize(point.z)); } /** Tests consistency of GeoArea.getRelationship vs GeoShape.isWithin */ public void testGeo3DRelations() throws Exception { - PlanetModel planetModel = getPlanetModel(); - int numDocs = atLeast(1000); if (VERBOSE) { System.out.println("TEST: " + numDocs + " docs"); @@ -212,14 +193,12 @@ public void testGeo3DRelations() throws Exception { GeoPoint[] docs = new GeoPoint[numDocs]; for(int docID=0;docID deleted = new HashSet<>(); // RandomIndexWriter is too slow here: IndexWriter w = new IndexWriter(dir, iwc); - for(int id=0;id 0 && random().nextInt(100) == 42) { @@ -688,119 +649,89 @@ private static void verify(double[] lats, double[] lons) throws Exception { w.forceMerge(1); } final IndexReader r = DirectoryReader.open(w); + if (VERBOSE) { + System.out.println("TEST: using reader " + r); + } w.close(); // We can't wrap with "exotic" readers because the geo3d query must see the Geo3DDVFormat: IndexSearcher s = newSearcher(r, false); - int numThreads = TestUtil.nextInt(random(), 2, 5); - - List threads = new ArrayList<>(); final int iters = atLeast(100); - final CountDownLatch startingGun = new CountDownLatch(1); - final AtomicBoolean failed = new AtomicBoolean(); - - for(int i=0;i", point.toString()); + assertEquals("Geo3DPoint ", point.toString()); } public void testShapeQueryToString() { @@ -813,7 +744,7 @@ private static Directory getDirectory() { } public void testEquals() { - GeoShape shape = randomShape(PlanetModel.WGS84); + GeoShape shape = randomShape(); Query q = Geo3DPoint.newShapeQuery("point", shape); assertEquals(q, Geo3DPoint.newShapeQuery("point", shape)); assertFalse(q.equals(Geo3DPoint.newShapeQuery("point2", shape))); @@ -821,7 +752,7 @@ public void testEquals() { // make a different random shape: GeoShape shape2; do { - shape2 = randomShape(PlanetModel.WGS84); + shape2 = randomShape(); } while (shape.equals(shape2)); assertFalse(q.equals(Geo3DPoint.newShapeQuery("point", shape2))); @@ -830,12 +761,11 @@ public void testEquals() { public void testComplexPolygons() { final PlanetModel pm = PlanetModel.WGS84; // Pick a random pole - final GeoPoint randomPole = new GeoPoint(pm, Math.toRadians(randomLat()), Math.toRadians(randomLon())); + final GeoPoint randomPole = new GeoPoint(pm, Math.toRadians(GeoTestUtil.nextLatitude()), Math.toRadians(GeoTestUtil.nextLongitude())); // Create a polygon that's less than 180 degrees final Polygon clockWise = makePoly(pm, randomPole, true, true); // Create a polygon that's greater than 180 degrees final Polygon counterClockWise = makePoly(pm, randomPole, false, true); - } protected static double MINIMUM_EDGE_ANGLE = Math.toRadians(5.0); @@ -911,7 +841,7 @@ protected Polygon makePoly(final PlanetModel pm, final GeoPoint pole, final bool // Choose a pole. The poly has to be within the polygon, but it also cannot be on the polygon edge. // If we can't find a good pole we have to give it up and not do the hole. for (int k = 0; k < 500; k++) { - final GeoPoint poleChoice = new GeoPoint(pm, toRadians(randomLat()), toRadians(randomLon())); + final GeoPoint poleChoice = new GeoPoint(pm, toRadians(GeoTestUtil.nextLatitude()), toRadians(GeoTestUtil.nextLongitude())); if (!poly.isWithin(poleChoice)) { continue; } @@ -1053,4 +983,316 @@ protected static int legalIndex(int index, int size) { } return index; } + + public void testEncodeDecodeCeil() throws Exception { + + // just for testing quantization error + final double ENCODING_TOLERANCE = Geo3DUtil.DECODE; + + int iters = atLeast(10000); + for(int iter=0;iter start+1) { + assertEquals(Geo3DUtil.DECODE, x - Geo3DUtil.decodeValue(i-1), 0.0d); + } + } + } + + /** Clips the incoming value to the allowed min/max range before encoding, instead of throwing an exception. */ + private static int encodeValueLenient(double x) { + double planetMax = PlanetModel.WGS84.getMaximumMagnitude(); + if (x > planetMax) { + x = planetMax; + } else if (x < -planetMax) { + x = -planetMax; + } + return Geo3DUtil.encodeValue(x); + } + + private static class ExplainingVisitor implements IntersectVisitor { + + final IntersectVisitor in; + final List stack = new ArrayList<>(); + private List stackToTargetDoc; + final int targetDocID; + final int numDims; + final int bytesPerDim; + private int targetStackUpto; + final StringBuilder b; + + // In the first phase, we always return CROSSES to do a full scan of the BKD tree to see which leaf block the document lives in + boolean firstPhase = true; + + public ExplainingVisitor(IntersectVisitor in, int targetDocID, int numDims, int bytesPerDim, StringBuilder b) { + this.in = in; + this.targetDocID = targetDocID; + this.numDims = numDims; + this.bytesPerDim = bytesPerDim; + this.b = b; + } + + public void startSecondPhase() { + if (firstPhase == false) { + throw new IllegalStateException("already started second phase"); + } + if (stackToTargetDoc == null) { + b.append("target docID=" + targetDocID + " was never seen in points!\n"); + } + firstPhase = false; + stack.clear(); + } + + @Override + public void visit(int docID) throws IOException { + assert firstPhase == false; + if (docID == targetDocID) { + b.append("leaf visit docID=" + docID + "\n"); + } + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + if (firstPhase) { + if (docID == targetDocID) { + assert stackToTargetDoc == null; + stackToTargetDoc = new ArrayList<>(stack); + b.append(" full BKD path to target doc:\n"); + for(Cell cell : stack) { + b.append(" " + cell + "\n"); + } + } + } else { + if (docID == targetDocID) { + double x = Geo3DPoint.decodeDimension(packedValue, 0); + double y = Geo3DPoint.decodeDimension(packedValue, Integer.BYTES); + double z = Geo3DPoint.decodeDimension(packedValue, 2 * Integer.BYTES); + b.append("leaf visit docID=" + docID + " x=" + x + " y=" + y + " z=" + z + "\n"); + in.visit(docID, packedValue); + } + } + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + Cell cell = new Cell(minPackedValue, maxPackedValue); + //System.out.println("compare: " + cell); + + // TODO: this is a bit hacky, having to reverse-engineer where we are in the BKD tree's recursion ... but it's the lesser evil vs e.g. + // polluting this visitor API, or implementing this "under the hood" in BKDReader instead? + if (firstPhase) { + + // Pop stack: + while (stack.size() > 0 && stack.get(stack.size()-1).contains(cell)) { + stack.remove(stack.size()-1); + //System.out.println(" pop"); + } + + // Push stack: + stack.add(cell); + //System.out.println(" push"); + + return Relation.CELL_CROSSES_QUERY; + } else { + Relation result = in.compare(minPackedValue, maxPackedValue); + if (targetStackUpto < stackToTargetDoc.size() && cell.equals(stackToTargetDoc.get(targetStackUpto))) { + b.append(" on cell " + stackToTargetDoc.get(targetStackUpto) + ", wrapped visitor returned " + result); + targetStackUpto++; + } + return result; + } + } + + private class Cell { + private final byte[] minPackedValue; + private final byte[] maxPackedValue; + + public Cell(byte[] minPackedValue, byte[] maxPackedValue) { + this.minPackedValue = minPackedValue.clone(); + this.maxPackedValue = maxPackedValue.clone(); + } + + /** Returns true if this cell fully contains the other one */ + public boolean contains(Cell other) { + for(int dim=0;dim 0) { + return false; + } + } + + return true; + } + + @Override + public String toString() { + double xMin = PointInShapeIntersectVisitor.decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 0)); + double xMax = PointInShapeIntersectVisitor.decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 0)); + double yMin = PointInShapeIntersectVisitor.decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 1 * Integer.BYTES)); + double yMax = PointInShapeIntersectVisitor.decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); + double zMin = PointInShapeIntersectVisitor.decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); + double zMax = PointInShapeIntersectVisitor.decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); + return "Cell(x=" + xMin + " TO " + xMax + " y=" + yMin + " TO " + yMax + " z=" + zMin + " TO " + zMax + ")"; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Cell == false) { + return false; + } + + Cell otherCell = (Cell) other; + return Arrays.equals(minPackedValue, otherCell.minPackedValue) && Arrays.equals(maxPackedValue, otherCell.maxPackedValue); + } + + @Override + public int hashCode() { + return Arrays.hashCode(minPackedValue) + Arrays.hashCode(maxPackedValue); + } + } + } + + public static String explain(String fieldName, GeoShape shape, IndexReader reader, int docID) throws Exception { + + // First find the leaf reader that owns this doc: + int subIndex = ReaderUtil.subIndex(docID, reader.leaves()); + LeafReader leafReader = reader.leaves().get(subIndex).reader(); + + StringBuilder b = new StringBuilder(); + b.append("target is in leaf " + leafReader + " of full reader " + reader + "\n"); + + DocIdSetBuilder hits = new DocIdSetBuilder(leafReader.maxDoc()); + ExplainingVisitor visitor = new ExplainingVisitor(new PointInShapeIntersectVisitor(hits, shape), docID - reader.leaves().get(subIndex).docBase, 3, Integer.BYTES, b); + + // Do first phase, where we just figure out the "path" that leads to the target docID: + leafReader.getPointValues().intersect(fieldName, visitor); + + // Do second phase, where we we see how the wrapped visitor responded along that path: + visitor.startSecondPhase(); + leafReader.getPointValues().intersect(fieldName, visitor); + + return b.toString(); + } + + @Ignore("https://issues.apache.org/jira/browse/LUCENE-7168") + public void testCuriousFailure() throws Exception { + GeoShape shape = GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, -0.8971654677124566, -0.3398482030102755, 1.4775317506492547); + GeoPoint point = new GeoPoint(0.8653002868649471, 0.50134342478497, 0.046203414829601996); + + // point is inside our circle shape: + assertTrue(shape.isWithin(point)); + + double xMin = 0.8653002866318559; + double xMax = 0.8653002870980383; + double yMin = 0.5013434245518787; + double yMax = 0.5013434250180612; + double zMin = 0.04620341459651078; + double zMax = 0.04620341506269321; + GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, xMin, xMax, yMin, yMax, zMin, zMax); + + // point is also inside our wee tiny box: + assertTrue(xyzSolid.isWithin(point)); + + assertTrue(xyzSolid.getRelationship(shape) != GeoArea.DISJOINT); + } } From e75db9476764b9a6a426686a58e18f38a26883a5 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 6 Apr 2016 21:48:33 -0400 Subject: [PATCH 0293/1113] LUCENE-7185: make an empty grid the simple way --- .../org/apache/lucene/document/LatLonGrid.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java index 5d594c68723d..9b4e981ac817 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java @@ -74,11 +74,17 @@ final class LatLonGrid { } long latitudeRange = maxLat - (long) minLat; long longitudeRange = maxLon - (long) minLon; - // we spill over the edge of the bounding box in each direction a bit, - // but it prevents edge case bugs. - latPerCell = latitudeRange / (GRID_SIZE - 1); - lonPerCell = longitudeRange / (GRID_SIZE - 1); - fill(polygons, 0, GRID_SIZE, 0, GRID_SIZE); + + if (latitudeRange < GRID_SIZE || longitudeRange < GRID_SIZE) { + // don't complicate fill right now if you pass e.g. emptyish stuff: make an "empty grid" + latPerCell = lonPerCell = Long.MAX_VALUE; + } else { + // we spill over the edge of the bounding box in each direction a bit, + // but it prevents edge case bugs. + latPerCell = latitudeRange / (GRID_SIZE - 1); + lonPerCell = longitudeRange / (GRID_SIZE - 1); + fill(polygons, 0, GRID_SIZE, 0, GRID_SIZE); + } } /** fills a 2D range of grid cells [minLatIndex .. maxLatIndex) X [minLonIndex .. maxLonIndex) */ From cebac848a999db0b341d717c8edd39ecc9184671 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 7 Apr 2016 00:08:03 -0400 Subject: [PATCH 0294/1113] LUCENE-7185: fix edge case bug in test logic (min=max=180), don't leak Directory for edge cases! --- .../java/org/apache/lucene/geo/Rectangle.java | 2 +- .../spatial/util/BaseGeoPointTestCase.java | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java index 527395ef3ddb..592e202fa87b 100644 --- a/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java +++ b/lucene/core/src/java/org/apache/lucene/geo/Rectangle.java @@ -66,7 +66,7 @@ public Rectangle(double minLat, double maxLat, double minLon, double maxLon) { @Override public String toString() { StringBuilder b = new StringBuilder(); - b.append("GeoRect(lat="); + b.append("Rectangle(lat="); b.append(minLat); b.append(" TO "); b.append(maxLat); diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index 6ae4d20142eb..adca4666fafa 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -738,7 +738,7 @@ protected Rectangle randomRect(boolean small) { static final boolean rectContainsPoint(Rectangle rect, double pointLat, double pointLon) { assert Double.isNaN(pointLat) == false; - if (rect.minLon < rect.maxLon) { + if (rect.minLon <= rect.maxLon) { return GeoRelationUtils.pointInRectPrecise(pointLat, pointLon, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon); } else { // Rect crosses dateline: @@ -1223,26 +1223,29 @@ public void testRectBoundariesAreInclusive() throws Exception { // exact edge cases assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon))); - // expand 1 ulp in each direction - assumeFalse("can't expand box, its too big already", rect.minLat == -90); - assumeFalse("can't expand box, its too big already", rect.maxLat == 90); - assumeFalse("can't expand box, its too big already", rect.minLon == -180); - assumeFalse("can't expand box, its too big already", rect.maxLon == 180); - assertEquals(8, s.count(newRectQuery(FIELD_NAME, Math.nextDown(rect.minLat), rect.maxLat, rect.minLon, rect.maxLon))); - assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, Math.nextUp(rect.maxLat), rect.minLon, rect.maxLon))); - assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, Math.nextDown(rect.minLon), rect.maxLon))); - assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, Math.nextUp(rect.maxLon)))); + // expand 1 ulp in each direction if possible and test a slightly larger box! + if (rect.minLat != -90) { + assertEquals(8, s.count(newRectQuery(FIELD_NAME, Math.nextDown(rect.minLat), rect.maxLat, rect.minLon, rect.maxLon))); + } + if (rect.maxLat != 90) { + assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, Math.nextUp(rect.maxLat), rect.minLon, rect.maxLon))); + } + if (rect.minLon != -180) { + assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, Math.nextDown(rect.minLon), rect.maxLon))); + } + if (rect.maxLon != 180) { + assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, Math.nextUp(rect.maxLon)))); + } - // now shrink 1 ulp in each direction: it should not include bogus stuff - assumeFalse("can't shrink box, its too small already", rect.minLat == 90); - assumeFalse("can't shrink box, its too small already", rect.maxLat == -90); - assumeFalse("can't shrink box, its too small already", rect.minLon == 180); - assumeFalse("can't shrink box, its too small already", rect.maxLon == -180); - // note we put points on "sides" not just "corners" so we just shrink all 4 at once for now: it should exclude all points! - assertEquals(0, s.count(newRectQuery(FIELD_NAME, Math.nextUp(rect.minLat), + // now shrink 1 ulp in each direction if possible: it should not include bogus stuff + if (rect.minLat != 90 && rect.maxLat != -90 && rect.minLon != 80 && rect.maxLon != -180) { + // note we put points on "sides" not just "corners" so we just shrink all 4 at once for now: it should exclude all points! + assertEquals(0, s.count(newRectQuery(FIELD_NAME, Math.nextUp(rect.minLat), Math.nextDown(rect.maxLat), Math.nextUp(rect.minLon), Math.nextDown(rect.maxLon)))); + } + r.close(); w.close(); dir.close(); From 6d537d2bad64a27927aa7c67c600dee97f79aa2b Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 7 Apr 2016 00:09:56 -0400 Subject: [PATCH 0295/1113] LUCENE-7185: fix edge case bugs in LatLonPoint bounding box query --- .../org/apache/lucene/document/LatLonPoint.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index b2c39c21b7fd..0b96936d8967 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; @@ -285,6 +286,21 @@ static void checkCompatible(FieldInfo fieldInfo) { * @throws IllegalArgumentException if {@code field} is null, or the box has invalid coordinates. */ public static Query newBoxQuery(String field, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) { + // exact double values of lat=90.0D and lon=180.0D must be treated special as they are not represented in the encoding + // and should not drag in extra bogus junk! TODO: should encodeCeil just throw ArithmeticException to be less trappy here? + if (minLatitude == 90.0) { + // range cannot match as 90.0 can never exist + return new MatchNoDocsQuery(); + } + if (minLongitude == 180.0) { + if (maxLongitude == 180.0) { + // range cannot match as 180.0 can never exist + return new MatchNoDocsQuery(); + } else if (maxLongitude < minLongitude) { + // encodeCeil() with dateline wrapping! + minLongitude = -180.0; + } + } byte[] lower = encodeCeil(minLatitude, minLongitude); byte[] upper = encode(maxLatitude, maxLongitude); // Crosses date line: we just rewrite into OR of two bboxes, with longitude as an open range: From 66859dcc66ee2ca707f4d82ffc96c8b7b1435753 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 7 Apr 2016 05:29:09 -0400 Subject: [PATCH 0296/1113] fix wrong description of how to run IndexFiles demo --- lucene/demo/src/java/overview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/demo/src/java/overview.html b/lucene/demo/src/java/overview.html index bb3752332df2..c20b55b481ad 100644 --- a/lucene/demo/src/java/overview.html +++ b/lucene/demo/src/java/overview.html @@ -71,7 +71,7 @@

    Indexing Files

    Once you've gotten this far you're probably itching to go. Let's build an index! Assuming you've set your CLASSPATH correctly, just type:

    -    java org.apache.lucene.demo.IndexFiles -docs {path-to-lucene}/src
+    java org.apache.lucene.demo.IndexFiles -docs {path-to-lucene}
    This will produce a subdirectory called index which will contain an index of all of the Lucene source code. From c1517c4a9cd1b82c7863f523424f97681b624633 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Thu, 7 Apr 2016 07:40:07 -0400 Subject: [PATCH 0297/1113] Fix up internal method signatures to make some degree of greater sense, in GeoPolygonFactory. --- .../spatial3d/geom/GeoPolygonFactory.java | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index fdd51c5744ed..a58056fcf0bf 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -64,13 +64,17 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final List holes) { // The basic operation uses a set of points, two points determining one particular edge, and a sided plane // describing membership. - return buildPolygonShape(new GeoCompositePolygon(), + final GeoCompositePolygon rval = new GeoCompositePolygon(); + if (buildPolygonShape(rval, planetModel, pointList, new BitSet(), convexPointIndex, getLegalIndex(convexPointIndex + 1, pointList.size()), new SidedPlane(pointList.get(getLegalIndex(convexPointIndex - 1, pointList.size())), pointList.get(convexPointIndex), pointList.get(getLegalIndex(convexPointIndex + 1, pointList.size()))), holes, - null); + null) == false) { + return null; + } + return rval; } /** Create a GeoPolygon using the specified points and holes, using order to determine @@ -138,23 +142,29 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final SidedPlane initialPlane = new SidedPlane(testPoint, pointList.get(0), pointList.get(1)); // We don't know if this is the correct siding choice. We will only know as we build the complex polygon. // So we need to be prepared to try both possibilities. - final GeoPolygon trial = buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, testPoint); - if (trial == null) { + GeoCompositePolygon rval = new GeoCompositePolygon(); + if (buildPolygonShape(rval, planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, testPoint) == false) { // The testPoint was within the shape. Was that intended? if (testPointInside) { // Yes: build it for real - return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, null); + rval = new GeoCompositePolygon(); + buildPolygonShape(rval, planetModel, pointList, new BitSet(), 0, 1, initialPlane, holes, null); + return rval; } // No: do the complement and return that. - return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); + rval = new GeoCompositePolygon(); + buildPolygonShape(rval, planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); + return rval; } else { // The testPoint was outside the shape. Was that intended? if (!testPointInside) { // Yes: return what we just built - return trial; + return rval; } // No: return the complement - return buildPolygonShape(new GeoCompositePolygon(), planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); + rval = new GeoCompositePolygon(); + buildPolygonShape(rval, planetModel, pointList, new BitSet(), 0, 1, new SidedPlane(initialPlane), holes, null); + return rval; } } @@ -296,12 +306,12 @@ protected static Double computeAngle(final GeoPoint point, * which result to use. If the test point is supposed to be within the shape, then it must be outside of the * complement shape. If the test point is supposed to be outside the shape, then it must be outside of the * original shape. Either way, we can figure out the right thing to use. - * @return the GeoPolygon passed in in the rval parameter, or null if what was specified + * @return false if what was specified * was inconsistent with what we generated. Specifically, if we specify an exterior point that is - * found in the interior of the shape we create here we return null, which is a signal that we chose + * found in the interior of the shape we create here we return false, which is a signal that we chose * our initial plane sidedness backwards. */ - public static GeoPolygon buildPolygonShape( + public static boolean buildPolygonShape( final GeoCompositePolygon rval, final PlanetModel planetModel, final List pointsList, @@ -345,7 +355,7 @@ public static GeoPolygon buildPolygonShape( // Find convexity around the current edge, if any final Boolean foundIt = findConvexPolygon(planetModel, currentEdge, rval, edgeBuffer, holes, testPoint); if (foundIt == null) { - return null; + return false; } if (foundIt) { @@ -409,7 +419,7 @@ public static GeoPolygon buildPolygonShape( thirdPartPoints.add(thePoint); thirdPartInternal.set(2, true); //System.out.println("Doing convex part..."); - final GeoPolygon thirdPoly = buildPolygonShape(rval, + if (buildPolygonShape(rval, planetModel, thirdPartPoints, thirdPartInternal, @@ -417,11 +427,10 @@ public static GeoPolygon buildPolygonShape( 1, checkEdge.plane, holes, - testPoint); - //System.out.println("...done convex part."); - if (thirdPoly == null) { - return null; + testPoint) == false) { + return false; } + //System.out.println("...done convex part."); // The part preceding the bad edge, back to thePoint, needs to be recursively // processed. So, assemble what we need, which is basically a list of edges. @@ -439,7 +448,7 @@ public static GeoPolygon buildPolygonShape( } firstPartInternal.set(i, true); //System.out.println("Doing first part..."); - final GeoPolygon firstPoly = buildPolygonShape(rval, + if (buildPolygonShape(rval, planetModel, firstPartPoints, firstPartInternal, @@ -447,11 +456,10 @@ public static GeoPolygon buildPolygonShape( 0, new SidedPlane(checkEdge.endPoint, false, checkEdge.startPoint, thePoint), holes, - testPoint); - //System.out.println("...done first part."); - if (firstPoly == null) { - return null; + testPoint) == false) { + return false; } + //System.out.println("...done first part."); final List secondPartPoints = new ArrayList<>(); final BitSet secondPartInternal = new BitSet(); loopEdge = edgeBuffer.getNext(checkEdge); @@ -466,7 +474,7 @@ public static GeoPolygon buildPolygonShape( } secondPartInternal.set(i, true); //System.out.println("Doing second part..."); - final GeoPolygon secondPoly = buildPolygonShape(rval, + if (buildPolygonShape(rval, planetModel, secondPartPoints, secondPartInternal, @@ -474,13 +482,12 @@ public static GeoPolygon buildPolygonShape( 0, new SidedPlane(checkEdge.endPoint, true, checkEdge.startPoint, thePoint), holes, - testPoint); - //System.out.println("... done second part"); - if (secondPoly == null) { - return null; + testPoint) == false) { + return false; } + //System.out.println("... done second part"); - return rval; + return true; } } } @@ -489,10 +496,10 @@ public static GeoPolygon buildPolygonShape( // If there's anything left in the edge buffer, convert to concave polygon. if (makeConcavePolygon(planetModel, rval, edgeBuffer, holes, testPoint) == false) { - return null; + return false; } - return rval; + return true; } /** Look for a concave polygon in the remainder of the edgebuffer. From 16c6dd9a2d67d88e18a43f24ff636a4ddca0f123 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 7 Apr 2016 09:27:48 -0400 Subject: [PATCH 0298/1113] LUCENE-7168: fix ceil/floor decode to match encode --- .../apache/lucene/spatial3d/Geo3DUtil.java | 13 ++++ .../PointInShapeIntersectVisitor.java | 22 ++----- .../lucene/spatial3d/TestGeo3DPoint.java | 63 ++++++------------- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java index cc11294e6c2d..1ff56a77a6f6 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java @@ -71,4 +71,17 @@ private static double getNextSafeDouble(double x) { assert result > x; return result; } + + /** Returns smallest double that would encode to int x. */ + // NOTE: keep this package private!! + static double decodeValueFloor(int x) { + return x * DECODE; + } + + /** Returns largest double that would encode to int x. */ + // NOTE: keep this package private!! + static double decodeValueCeil(int x) { + assert x < Integer.MAX_VALUE; + return Math.nextDown((x+1) * DECODE); + } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java index 25dea721c116..b9d4e70a6c81 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/PointInShapeIntersectVisitor.java @@ -57,12 +57,12 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { // here are inclusive, we need to extend the bounds to the largest un-quantized values that // could quantize into these bounds. The encoding (Geo3DUtil.encodeValue) does // a Math.round from double to long, so e.g. 1.4 -> 1, and -1.4 -> -1: - double xMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 0)); - double xMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 0)); - double yMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 1 * Integer.BYTES)); - double yMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); - double zMin = decodeValueMin(NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); - double zMax = decodeValueMax(NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); + double xMin = Geo3DUtil.decodeValueFloor(NumericUtils.sortableBytesToInt(minPackedValue, 0)); + double xMax = Geo3DUtil.decodeValueCeil(NumericUtils.sortableBytesToInt(maxPackedValue, 0)); + double yMin = Geo3DUtil.decodeValueFloor(NumericUtils.sortableBytesToInt(minPackedValue, 1 * Integer.BYTES)); + double yMax = Geo3DUtil.decodeValueCeil(NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); + double zMin = Geo3DUtil.decodeValueFloor(NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); + double zMax = Geo3DUtil.decodeValueCeil(NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); //System.out.println(" compare: x=" + cellXMin + "-" + cellXMax + " y=" + cellYMin + "-" + cellYMax + " z=" + cellZMin + "-" + cellZMax); assert xMin <= xMax; @@ -94,14 +94,4 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return Relation.CELL_CROSSES_QUERY; } } - - /** More negative decode, at bottom of cell */ - static double decodeValueMin(int x) { - return (((double)x) - 0.5) * Geo3DUtil.DECODE; - } - - /** More positive decode, at top of cell */ - static double decodeValueMax(int x) { - return (((double)x) + 0.5) * Geo3DUtil.DECODE; - } } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index b751125bced6..410618c829e6 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -75,7 +75,6 @@ import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; -import org.junit.Ignore; import com.carrotsearch.randomizedtesting.generators.RandomInts; @@ -266,14 +265,14 @@ public void testGeo3DRelations() throws Exception { } else { GeoArea xyzSolid = GeoAreaFactory.makeGeoArea(PlanetModel.WGS84, - PointInShapeIntersectVisitor.decodeValueMin(cell.xMinEnc), PointInShapeIntersectVisitor.decodeValueMax(cell.xMaxEnc), - PointInShapeIntersectVisitor.decodeValueMin(cell.yMinEnc), PointInShapeIntersectVisitor.decodeValueMax(cell.yMaxEnc), - PointInShapeIntersectVisitor.decodeValueMin(cell.zMinEnc), PointInShapeIntersectVisitor.decodeValueMax(cell.zMaxEnc)); + Geo3DUtil.decodeValueFloor(cell.xMinEnc), Geo3DUtil.decodeValueCeil(cell.xMaxEnc), + Geo3DUtil.decodeValueFloor(cell.yMinEnc), Geo3DUtil.decodeValueCeil(cell.yMaxEnc), + Geo3DUtil.decodeValueFloor(cell.zMinEnc), Geo3DUtil.decodeValueCeil(cell.zMaxEnc)); if (VERBOSE) { - log.println(" minx="+PointInShapeIntersectVisitor.decodeValueMin(cell.xMinEnc)+" maxx="+PointInShapeIntersectVisitor.decodeValueMax(cell.xMaxEnc)+ - " miny="+PointInShapeIntersectVisitor.decodeValueMin(cell.yMinEnc)+" maxy="+PointInShapeIntersectVisitor.decodeValueMax(cell.yMaxEnc)+ - " minz="+PointInShapeIntersectVisitor.decodeValueMin(cell.zMinEnc)+" maxz="+PointInShapeIntersectVisitor.decodeValueMax(cell.zMaxEnc)); + log.println(" minx="+Geo3DUtil.decodeValueFloor(cell.xMinEnc)+" maxx="+Geo3DUtil.decodeValueCeil(cell.xMaxEnc)+ + " miny="+Geo3DUtil.decodeValueFloor(cell.yMinEnc)+" maxy="+Geo3DUtil.decodeValueCeil(cell.yMaxEnc)+ + " minz="+Geo3DUtil.decodeValueFloor(cell.zMinEnc)+" maxz="+Geo3DUtil.decodeValueCeil(cell.zMaxEnc)); } switch (xyzSolid.getRelationship(shape)) { @@ -312,9 +311,7 @@ public void testGeo3DRelations() throws Exception { log.println(" GeoArea.DISJOINT: drop this cell"); for(int docID=0;docID Date: Thu, 7 Apr 2016 10:44:08 +0100 Subject: [PATCH 0299/1113] SOLR-8938: Add optional -excluderegex argument to ZkCLI. --- solr/CHANGES.txt | 1 + .../src/java/org/apache/solr/cloud/ZkCLI.java | 13 ++++-- .../test/org/apache/solr/cloud/ZkCLITest.java | 41 ++++++++++++++----- .../solr/common/cloud/ZkConfigManager.java | 27 ++++++++++-- .../common/cloud/TestZkConfigManager.java | 7 ++++ 5 files changed, 72 insertions(+), 17 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d0c1c361ac70..b2f9a84f6564 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -60,6 +60,7 @@ New Features graph traversal to find the shortest path(s) in an unweighted, directed graph stored in a SolrCloud collection. (Joel Bernstein) +* SOLR-8938: Add optional -excluderegex argument to ZkCLI. (Christine Poerschke) Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index 7ba3a8fb4106..402a83fdfba6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -29,7 +29,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; import org.apache.solr.common.cloud.ZkStateReader; @@ -48,6 +47,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; public class ZkCLI { @@ -66,7 +66,9 @@ public class ZkCLI { private static final String RUNZK = "runzk"; private static final String SOLRHOME = "solrhome"; private static final String BOOTSTRAP = "bootstrap"; - private static final String UPCONFIG = "upconfig"; + static final String UPCONFIG = "upconfig"; + static final String EXCLUDE_REGEX = "excluderegex"; + static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX; private static final String COLLECTION = "collection"; private static final String CLEAR = "clear"; private static final String LIST = "list"; @@ -120,6 +122,9 @@ public static void main(String[] args) throws InterruptedException, options.addOption("c", COLLECTION, true, "for " + LINKCONFIG + ": name of the collection"); + options.addOption(EXCLUDE_REGEX, true, + "for " + UPCONFIG + ": files matching this regular expression won't be uploaded"); + options .addOption( "r", @@ -211,13 +216,15 @@ public static void main(String[] args) throws InterruptedException, } String confDir = line.getOptionValue(CONFDIR); String confName = line.getOptionValue(CONFNAME); + final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT); if(!ZkController.checkChrootPath(zkServerAddress, true)) { System.out.println("A chroot was specified in zkHost but the znode doesn't exist. "); System.exit(1); } ZkConfigManager configManager = new ZkConfigManager(zkClient); - configManager.uploadConfigDir(Paths.get(confDir), confName); + final Pattern excludePattern = Pattern.compile(excludeExpr); + configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern); } else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) { if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) { System.out.println("-" + CONFDIR + " and -" + CONFNAME diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java index 19cea70d582b..81b810bf5bd9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java @@ -100,6 +100,13 @@ public void setUp() throws Exception { log.info("####SETUP_END " + getTestName()); } + @Test + public void testCmdConstants() throws Exception { + assertEquals("upconfig", ZkCLI.UPCONFIG); + assertEquals("excluderegex", ZkCLI.EXCLUDE_REGEX); + assertEquals(ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT); + } + @Test public void testBootstrapWithChroot() throws Exception { String chroot = "/foo/bar"; @@ -193,14 +200,22 @@ public void testUpConfigLinkConfigClearZk() throws Exception { // test upconfig String confsetname = "confsetone"; - String[] args = new String[] { - "-zkhost", - zkServer.getZkAddress(), - "-cmd", - "upconfig", - "-confdir", - ExternalPaths.TECHPRODUCTS_CONFIGSET, "-confname", confsetname}; - ZkCLI.main(args); + final String[] upconfigArgs; + if (random().nextBoolean()) { + upconfigArgs = new String[] { + "-zkhost", zkServer.getZkAddress(), + "-cmd", ZkCLI.UPCONFIG, + "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, + "-confname", confsetname}; + } else { + upconfigArgs = new String[] { + "-zkhost", zkServer.getZkAddress(), + "-cmd", ZkCLI.UPCONFIG, + "-"+ZkCLI.EXCLUDE_REGEX, ZkCLI.EXCLUDE_REGEX_DEFAULT, + "-confdir", ExternalPaths.TECHPRODUCTS_CONFIGSET, + "-confname", confsetname}; + } + ZkCLI.main(upconfigArgs); assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true)); @@ -208,7 +223,7 @@ public void testUpConfigLinkConfigClearZk() throws Exception { // ZkCLI.main(new String[0]); // test linkconfig - args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", + String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "linkconfig", "-collection", "collection1", "-confname", confsetname}; ZkCLI.main(args); @@ -236,8 +251,12 @@ public void testUpConfigLinkConfigClearZk() throws Exception { int indexOfRelativePath = sourceFile.getAbsolutePath().lastIndexOf("sample_techproducts_configs" + File.separator + "conf"); String relativePathofFile = sourceFile.getAbsolutePath().substring(indexOfRelativePath + 33, sourceFile.getAbsolutePath().length()); File downloadedFile = new File(confDir,relativePathofFile); - assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists()); - assertTrue("Content didn't change",FileUtils.contentEquals(sourceFile,downloadedFile)); + if (ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_PATTERN.matcher(relativePathofFile).matches()) { + assertFalse(sourceFile.getAbsolutePath() + " exists in ZK, downloaded:" + downloadedFile.getAbsolutePath(), downloadedFile.exists()); + } else { + assertTrue(downloadedFile.getAbsolutePath() + " does not exist source:" + sourceFile.getAbsolutePath(), downloadedFile.exists()); + assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile)); + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java index 9123c5391981..eb87c34782e6 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java @@ -30,6 +30,8 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Class that manages named configs in Zookeeper @@ -41,6 +43,9 @@ public class ZkConfigManager { /** ZkNode where named configs are stored */ public static final String CONFIGS_ZKNODE = "/configs"; + public static final String UPLOAD_FILENAME_EXCLUDE_REGEX = "^\\..*$"; + public static final Pattern UPLOAD_FILENAME_EXCLUDE_PATTERN = Pattern.compile(UPLOAD_FILENAME_EXCLUDE_REGEX); + private final SolrZkClient zkClient; /** @@ -51,7 +56,8 @@ public ZkConfigManager(SolrZkClient zkClient) { this.zkClient = zkClient; } - private void uploadToZK(final Path rootPath, final String zkPath) throws IOException { + private void uploadToZK(final Path rootPath, final String zkPath, + final Pattern filenameExclusions) throws IOException { if (!Files.exists(rootPath)) throw new IOException("Path " + rootPath + " does not exist"); @@ -60,8 +66,10 @@ private void uploadToZK(final Path rootPath, final String zkPath) throws IOExcep @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String filename = file.getFileName().toString(); - if (filename.startsWith(".")) + if (filenameExclusions != null && filenameExclusions.matcher(filename).matches()) { + logger.info("uploadToZK skipping '{}' due to filenameExclusions '{}'", filename, filenameExclusions); return FileVisitResult.CONTINUE; + } String zkNode = createZkNodeName(zkPath, rootPath, file); try { zkClient.makePath(zkNode, file.toFile(), false, true); @@ -118,7 +126,20 @@ private void downloadFromZK(String zkPath, Path dir) throws IOException { * if an I/O error occurs or the path does not exist */ public void uploadConfigDir(Path dir, String configName) throws IOException { - uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName); + uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, UPLOAD_FILENAME_EXCLUDE_PATTERN); + } + + /** + * Upload matching files from a given path to a config in Zookeeper + * @param dir {@link java.nio.file.Path} to the files + * @param configName the name to give the config + * @param filenameExclusions files matching this pattern will not be uploaded + * @throws IOException + * if an I/O error occurs or the path does not exist + */ + public void uploadConfigDir(Path dir, String configName, + Pattern filenameExclusions) throws IOException { + uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName, filenameExclusions); } /** diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java index fd7a0acd0bad..4727bcc86415 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestZkConfigManager.java @@ -19,6 +19,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.ZkCLI; import org.apache.solr.cloud.ZkTestServer; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; @@ -53,6 +54,12 @@ public static void shutdownZkServer() throws IOException, InterruptedException { zkServer = null; } + @Test + public void testConstants() throws Exception { + assertEquals("/configs", ZkConfigManager.CONFIGS_ZKNODE); + assertEquals("^\\..*$", ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX); + } + @Test public void testUploadConfig() throws IOException { From 01867a5b318d6509731feea118b3f82764310380 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 7 Apr 2016 11:26:46 -0400 Subject: [PATCH 0300/1113] LUCENE-7185: fix tie-breaker sort bug --- .../org/apache/lucene/util/SloppyMath.java | 8 +++- .../apache/lucene/util/TestSloppyMath.java | 41 ++++++++++++++----- .../document/TestLatLonPointDistanceSort.java | 4 +- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java index d8b7056d077d..7be9be1c4ad4 100644 --- a/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java +++ b/lucene/core/src/java/org/apache/lucene/util/SloppyMath.java @@ -37,7 +37,9 @@ public class SloppyMath { * specified in decimal degrees (latitude/longitude). This works correctly * even if the dateline is between the two points. *

    - * Error is around 1E-5 (0.01mm) from the actual haversine distance. + * Error is at most 2E-1 (20cm) from the actual haversine distance, but is typically + * much smaller for reasonable distances: around 1E-5 (0.01mm) for distances less than + * 1000km. * * @param lat1 Latitude of the first point. * @param lon1 Longitude of the first point. @@ -87,7 +89,9 @@ public static double haversinSortKey(double lat1, double lon1, double lat2, doub double x2 = lat2 * TO_RADIANS; double h1 = 1 - cos(x1 - x2); double h2 = 1 - cos((lon1 - lon2) * TO_RADIANS); - return h1 + cos(x1) * cos(x2) * h2; + double h = h1 + cos(x1) * cos(x2) * h2; + // clobber crazy precision so subsequent rounding does not create ties. + return Double.longBitsToDouble(Double.doubleToRawLongBits(h) & 0xFFFFFFFFFFFFFFF8L); } /** diff --git a/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java b/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java index f11d35fb95c3..ea5597183c85 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestSloppyMath.java @@ -24,6 +24,8 @@ import java.util.Random; +import org.apache.lucene.geo.GeoTestUtil; + public class TestSloppyMath extends LuceneTestCase { // accuracy for cos() @@ -31,7 +33,9 @@ public class TestSloppyMath extends LuceneTestCase { // accuracy for asin() static double ASIN_DELTA = 1E-7; // accuracy for haversinMeters() - static double HAVERSIN_DELTA = 1E-5; + static double HAVERSIN_DELTA = 2E-1; + // accuracy for haversinMeters() for "reasonable" distances (< 1000km) + static double REASONABLE_HAVERSIN_DELTA = 1E-5; public void testCos() { assertTrue(Double.isNaN(cos(Double.NaN))); @@ -127,14 +131,14 @@ public void testHaversin() { /** Test this method sorts the same way as real haversin */ public void testHaversinSortKey() { for (int i = 0; i < 100000; i++) { - double centerLat = -90 + 180.0 * random().nextDouble(); - double centerLon = -180 + 360.0 * random().nextDouble(); + double centerLat = GeoTestUtil.nextLatitude(); + double centerLon = GeoTestUtil.nextLongitude(); - double lat1 = -90 + 180.0 * random().nextDouble(); - double lon1 = -180 + 360.0 * random().nextDouble(); + double lat1 = GeoTestUtil.nextLatitude(); + double lon1 = GeoTestUtil.nextLongitude(); - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); int expected = Integer.signum(Double.compare(haversinMeters(centerLat, centerLon, lat1, lon1), haversinMeters(centerLat, centerLon, lat2, lon2))); @@ -148,10 +152,10 @@ public void testHaversinSortKey() { public void testAgainstSlowVersion() { for (int i = 0; i < 100_000; i++) { - double lat1 = -90 + 180.0 * random().nextDouble(); - double lon1 = -180 + 360.0 * random().nextDouble(); - double lat2 = -90 + 180.0 * random().nextDouble(); - double lon2 = -180 + 360.0 * random().nextDouble(); + double lat1 = GeoTestUtil.nextLatitude(); + double lon1 = GeoTestUtil.nextLongitude(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); double expected = haversinMeters(lat1, lon1, lat2, lon2); double actual = slowHaversin(lat1, lon1, lat2, lon2); @@ -159,6 +163,21 @@ public void testAgainstSlowVersion() { } } + public void testAgainstSlowVersionReasonable() { + for (int i = 0; i < 100_000; i++) { + double lat1 = GeoTestUtil.nextLatitude(); + double lon1 = GeoTestUtil.nextLongitude(); + double lat2 = GeoTestUtil.nextLatitude(); + double lon2 = GeoTestUtil.nextLongitude(); + + double expected = haversinMeters(lat1, lon1, lat2, lon2); + if (expected < 1_000_000) { + double actual = slowHaversin(lat1, lon1, lat2, lon2); + assertEquals(expected, actual, REASONABLE_HAVERSIN_DELTA); + } + } + } + // simple incorporation of the wikipedia formula private static double slowHaversin(double lat1, double lon1, double lat2, double lon2) { double h1 = (1 - StrictMath.cos(StrictMath.toRadians(lat2) - StrictMath.toRadians(lat1))) / 2; diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 5ce819c98e93..0003b302d40f 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -64,7 +64,7 @@ public void testDistanceSort() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.1028401330432, (Double)d.fields[0], 0.0D); + assertEquals(462.1028401330431, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; assertEquals(1054.9842850974826, (Double)d.fields[0], 0.0D); @@ -101,7 +101,7 @@ public void testMissingLast() throws Exception { TopDocs td = searcher.search(new MatchAllDocsQuery(), 3, sort); FieldDoc d = (FieldDoc) td.scoreDocs[0]; - assertEquals(462.1028401330432D, (Double)d.fields[0], 0.0D); + assertEquals(462.1028401330431D, (Double)d.fields[0], 0.0D); d = (FieldDoc) td.scoreDocs[1]; assertEquals(1054.9842850974826, (Double)d.fields[0], 0.0D); From 6a6f73a26c53a02d6a5267fae88f2ad2e86a6980 Mon Sep 17 00:00:00 2001 From: nknize Date: Wed, 6 Apr 2016 13:06:17 -0500 Subject: [PATCH 0301/1113] LUCENE-7184: Refactor LatLonPoint encoding methods to new GeoEncodingUtils helper class in core geo package. Also refactors LatLonPointTests to TestGeoEncodingUtils. --- .../apache/lucene/geo/GeoEncodingUtils.java | 147 +++++++++++++++++ .../lucene/geo/TestGeoEncodingUtils.java | 151 ++++++++++++++++++ .../apache/lucene/document/LatLonGrid.java | 17 +- .../apache/lucene/document/LatLonPoint.java | 116 ++------------ .../LatLonPointDistanceComparator.java | 25 +-- .../document/LatLonPointDistanceQuery.java | 29 ++-- .../document/LatLonPointInPolygonQuery.java | 29 ++-- .../lucene/document/TestLatLonGrid.java | 17 +- .../lucene/document/TestLatLonPoint.java | 118 +------------- .../document/TestLatLonPointDistanceSort.java | 9 +- .../lucene/search/TestLatLonPointQueries.java | 5 +- 11 files changed, 392 insertions(+), 271 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/geo/GeoEncodingUtils.java create mode 100644 lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java diff --git a/lucene/core/src/java/org/apache/lucene/geo/GeoEncodingUtils.java b/lucene/core/src/java/org/apache/lucene/geo/GeoEncodingUtils.java new file mode 100644 index 000000000000..671f779ecff6 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/geo/GeoEncodingUtils.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.geo; + +import org.apache.lucene.util.NumericUtils; + +import static org.apache.lucene.geo.GeoUtils.MAX_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MAX_LON_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LON_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.checkLatitude; +import static org.apache.lucene.geo.GeoUtils.checkLongitude; + +/** + * reusable geopoint encoding methods + * + * @lucene.experimental + */ +public final class GeoEncodingUtils { + /** number of bits used for quantizing latitude and longitude values */ + public static final short BITS = 32; + + private static final double LAT_SCALE = (0x1L<= MIN_LAT_INCL && result < MAX_LAT_INCL; + return result; + } + + /** + * Turns quantized value from byte array back into a double. + * @param src byte array containing 4 bytes to decode at {@code offset} + * @param offset offset into {@code src} to decode from. + * @return decoded latitude value. + */ + public static double decodeLatitude(byte[] src, int offset) { + return decodeLatitude(NumericUtils.sortableBytesToInt(src, offset)); + } + + /** + * Turns quantized value from {@link #encodeLongitude} back into a double. + * @param encoded encoded value: 32-bit quantized value. + * @return decoded longitude value. + */ + public static double decodeLongitude(int encoded) { + double result = encoded * LON_DECODE; + assert result >= MIN_LON_INCL && result < MAX_LON_INCL; + return result; + } + + /** + * Turns quantized value from byte array back into a double. + * @param src byte array containing 4 bytes to decode at {@code offset} + * @param offset offset into {@code src} to decode from. + * @return decoded longitude value. + */ + public static double decodeLongitude(byte[] src, int offset) { + return decodeLongitude(NumericUtils.sortableBytesToInt(src, offset)); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java new file mode 100644 index 000000000000..3b71c6388e14 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.geo; + +import java.util.Random; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.TestUtil; + +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil; +import static org.apache.lucene.geo.GeoUtils.MAX_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MAX_LON_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LAT_INCL; +import static org.apache.lucene.geo.GeoUtils.MIN_LON_INCL; + +/** + Tests methods in {@link GeoEncodingUtils} + */ +public class TestGeoEncodingUtils extends LuceneTestCase { + + /** + * step through some integers, ensuring they decode to their expected double values. + * double values start at -90 and increase by LATITUDE_DECODE for each integer. + * check edge cases within the double range and random doubles within the range too. + */ + public void testLatitudeQuantization() throws Exception { + final double LATITUDE_DECODE = 180.0D/(0x1L<<32); + Random random = random(); + for (int i = 0; i < 10000; i++) { + int encoded = random.nextInt(); + double min = MIN_LAT_INCL + (encoded - (long)Integer.MIN_VALUE) * LATITUDE_DECODE; + double decoded = decodeLatitude(encoded); + // should exactly equal expected value + assertEquals(min, decoded, 0.0D); + // should round-trip + assertEquals(encoded, encodeLatitude(decoded)); + assertEquals(encoded, encodeLatitudeCeil(decoded)); + // test within the range + if (i != Integer.MAX_VALUE) { + // this is the next representable value + // all double values between [min .. max) should encode to the current integer + // all double values between (min .. max] should encodeCeil to the next integer. + double max = min + LATITUDE_DECODE; + assertEquals(max, decodeLatitude(encoded+1), 0.0D); + assertEquals(encoded+1, encodeLatitude(max)); + assertEquals(encoded+1, encodeLatitudeCeil(max)); + + // first and last doubles in range that will be quantized + double minEdge = Math.nextUp(min); + double maxEdge = Math.nextDown(max); + assertEquals(encoded, encodeLatitude(minEdge)); + assertEquals(encoded+1, encodeLatitudeCeil(minEdge)); + assertEquals(encoded, encodeLatitude(maxEdge)); + assertEquals(encoded+1, encodeLatitudeCeil(maxEdge)); + + // check random values within the double range + long minBits = NumericUtils.doubleToSortableLong(minEdge); + long maxBits = NumericUtils.doubleToSortableLong(maxEdge); + for (int j = 0; j < 100; j++) { + double value = NumericUtils.sortableLongToDouble(TestUtil.nextLong(random, minBits, maxBits)); + // round down + assertEquals(encoded, encodeLatitude(value)); + // round up + assertEquals(encoded+1, encodeLatitudeCeil(value)); + } + } + } + } + + /** + * step through some integers, ensuring they decode to their expected double values. + * double values start at -180 and increase by LONGITUDE_DECODE for each integer. + * check edge cases within the double range and a random doubles within the range too. + */ + public void testLongitudeQuantization() throws Exception { + final double LONGITUDE_DECODE = 360.0D/(0x1L<<32); + Random random = random(); + for (int i = 0; i < 10000; i++) { + int encoded = random.nextInt(); + double min = MIN_LON_INCL + (encoded - (long)Integer.MIN_VALUE) * LONGITUDE_DECODE; + double decoded = decodeLongitude(encoded); + // should exactly equal expected value + assertEquals(min, decoded, 0.0D); + // should round-trip + assertEquals(encoded, encodeLongitude(decoded)); + assertEquals(encoded, encodeLongitudeCeil(decoded)); + // test within the range + if (i != Integer.MAX_VALUE) { + // this is the next representable value + // all double values between [min .. max) should encode to the current integer + // all double values between (min .. max] should encodeCeil to the next integer. + double max = min + LONGITUDE_DECODE; + assertEquals(max, decodeLongitude(encoded+1), 0.0D); + assertEquals(encoded+1, encodeLongitude(max)); + assertEquals(encoded+1, encodeLongitudeCeil(max)); + + // first and last doubles in range that will be quantized + double minEdge = Math.nextUp(min); + double maxEdge = Math.nextDown(max); + assertEquals(encoded, encodeLongitude(minEdge)); + assertEquals(encoded+1, encodeLongitudeCeil(minEdge)); + assertEquals(encoded, encodeLongitude(maxEdge)); + assertEquals(encoded+1, encodeLongitudeCeil(maxEdge)); + + // check random values within the double range + long minBits = NumericUtils.doubleToSortableLong(minEdge); + long maxBits = NumericUtils.doubleToSortableLong(maxEdge); + for (int j = 0; j < 100; j++) { + double value = NumericUtils.sortableLongToDouble(TestUtil.nextLong(random, minBits, maxBits)); + // round down + assertEquals(encoded, encodeLongitude(value)); + // round up + assertEquals(encoded+1, encodeLongitudeCeil(value)); + } + } + } + } + + // check edge/interesting cases explicitly + public void testEncodeEdgeCases() { + assertEquals(Integer.MIN_VALUE, encodeLatitude(MIN_LAT_INCL)); + assertEquals(Integer.MIN_VALUE, encodeLatitudeCeil(MIN_LAT_INCL)); + assertEquals(Integer.MAX_VALUE, encodeLatitude(MAX_LAT_INCL)); + assertEquals(Integer.MAX_VALUE, encodeLatitudeCeil(MAX_LAT_INCL)); + + assertEquals(Integer.MIN_VALUE, encodeLongitude(MIN_LON_INCL)); + assertEquals(Integer.MIN_VALUE, encodeLongitudeCeil(MIN_LON_INCL)); + assertEquals(Integer.MAX_VALUE, encodeLongitude(MAX_LON_INCL)); + assertEquals(Integer.MAX_VALUE, encodeLongitudeCeil(MAX_LON_INCL)); + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java index 9b4e981ac817..7cb047affcdf 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonGrid.java @@ -20,6 +20,11 @@ import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.util.FixedBitSet; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** * This is a temporary hack, until some polygon methods have better performance! *

    @@ -99,10 +104,10 @@ void fill(Polygon[] polygons, int minLatIndex, int maxLatIndex, int minLonIndex, assert cellMaxLat >= cellMinLat; assert cellMaxLon >= cellMinLon; - Relation relation = Polygon.relate(polygons, LatLonPoint.decodeLatitude((int) cellMinLat), - LatLonPoint.decodeLatitude((int) cellMaxLat), - LatLonPoint.decodeLongitude((int) cellMinLon), - LatLonPoint.decodeLongitude((int) cellMaxLon)); + Relation relation = Polygon.relate(polygons, decodeLatitude((int) cellMinLat), + decodeLatitude((int) cellMaxLat), + decodeLongitude((int) cellMinLon), + decodeLongitude((int) cellMaxLon)); if (relation != Relation.CELL_CROSSES_QUERY) { // we know the answer for this region, fill the cell range for (int i = minLatIndex; i < maxLatIndex; i++) { @@ -140,8 +145,8 @@ boolean contains(int latitude, int longitude) { } // the grid is unsure (boundary): do a real test. - double docLatitude = LatLonPoint.decodeLatitude(latitude); - double docLongitude = LatLonPoint.decodeLongitude(longitude); + double docLatitude = decodeLatitude(latitude); + double docLongitude = decodeLongitude(longitude); return Polygon.contains(polygons, docLatitude, docLongitude); } diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java index 0b96936d8967..b1e0b44a19be 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPoint.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.document; -import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.index.DocValuesType; @@ -32,6 +31,13 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.geo.Polygon; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil; + /** * An indexed location field. *

    @@ -123,117 +129,13 @@ public String toString() { /** * Returns a 64-bit long, where the upper 32 bits are the encoded latitude, * and the lower 32 bits are the encoded longitude. - * @see #decodeLatitude(int) - * @see #decodeLongitude(int) + * @see org.apache.lucene.geo.GeoEncodingUtils#decodeLatitude(int) + * @see org.apache.lucene.geo.GeoEncodingUtils#decodeLongitude(int) */ @Override public Number numericValue() { return currentValue; } - - // public helper methods (e.g. for queries) - - /** - * Quantizes double (64 bit) latitude into 32 bits (rounding down: in the direction of -90) - * @param latitude latitude value: must be within standard +/-90 coordinate bounds. - * @return encoded value as a 32-bit {@code int} - * @throws IllegalArgumentException if latitude is out of bounds - */ - public static int encodeLatitude(double latitude) { - GeoUtils.checkLatitude(latitude); - // the maximum possible value cannot be encoded without overflow - if (latitude == 90.0D) { - latitude = Math.nextDown(latitude); - } - return (int) Math.floor(latitude / LATITUDE_DECODE); - } - - /** - * Quantizes double (64 bit) latitude into 32 bits (rounding up: in the direction of +90) - * @param latitude latitude value: must be within standard +/-90 coordinate bounds. - * @return encoded value as a 32-bit {@code int} - * @throws IllegalArgumentException if latitude is out of bounds - */ - public static int encodeLatitudeCeil(double latitude) { - GeoUtils.checkLatitude(latitude); - // the maximum possible value cannot be encoded without overflow - if (latitude == 90.0D) { - latitude = Math.nextDown(latitude); - } - return (int) Math.ceil(latitude / LATITUDE_DECODE); - } - - /** - * Quantizes double (64 bit) longitude into 32 bits (rounding down: in the direction of -180) - * @param longitude longitude value: must be within standard +/-180 coordinate bounds. - * @return encoded value as a 32-bit {@code int} - * @throws IllegalArgumentException if longitude is out of bounds - */ - public static int encodeLongitude(double longitude) { - GeoUtils.checkLongitude(longitude); - // the maximum possible value cannot be encoded without overflow - if (longitude == 180.0D) { - longitude = Math.nextDown(longitude); - } - return (int) Math.floor(longitude / LONGITUDE_DECODE); - } - - /** - * Quantizes double (64 bit) longitude into 32 bits (rounding up: in the direction of +180) - * @param longitude longitude value: must be within standard +/-180 coordinate bounds. - * @return encoded value as a 32-bit {@code int} - * @throws IllegalArgumentException if longitude is out of bounds - */ - public static int encodeLongitudeCeil(double longitude) { - GeoUtils.checkLongitude(longitude); - // the maximum possible value cannot be encoded without overflow - if (longitude == 180.0D) { - longitude = Math.nextDown(longitude); - } - return (int) Math.ceil(longitude / LONGITUDE_DECODE); - } - - /** - * Turns quantized value from {@link #encodeLatitude} back into a double. - * @param encoded encoded value: 32-bit quantized value. - * @return decoded latitude value. - */ - public static double decodeLatitude(int encoded) { - double result = encoded * LATITUDE_DECODE; - assert result >= GeoUtils.MIN_LAT_INCL && result < GeoUtils.MAX_LAT_INCL; - return result; - } - - /** - * Turns quantized value from byte array back into a double. - * @param src byte array containing 4 bytes to decode at {@code offset} - * @param offset offset into {@code src} to decode from. - * @return decoded latitude value. - */ - public static double decodeLatitude(byte[] src, int offset) { - return decodeLatitude(NumericUtils.sortableBytesToInt(src, offset)); - } - - /** - * Turns quantized value from {@link #encodeLongitude} back into a double. - * @param encoded encoded value: 32-bit quantized value. - * @return decoded longitude value. - */ - public static double decodeLongitude(int encoded) { - double result = encoded * LONGITUDE_DECODE; - assert result >= GeoUtils.MIN_LON_INCL && result < GeoUtils.MAX_LON_INCL; - return result; - } - - /** - * Turns quantized value from byte array back into a double. - * @param src byte array containing 4 bytes to decode at {@code offset} - * @param offset offset into {@code src} to decode from. - * @return decoded longitude value. - */ - public static double decodeLongitude(byte[] src, int offset) { - return decodeLongitude(NumericUtils.sortableBytesToInt(src, offset)); - } /** sugar encodes a single point as a byte array */ private static byte[] encode(double latitude, double longitude) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java index 58db5e57e67c..0306489da3c0 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceComparator.java @@ -29,6 +29,11 @@ import org.apache.lucene.geo.Rectangle; import org.apache.lucene.util.SloppyMath; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** * Compares documents by distance from an origin point *

    @@ -85,17 +90,17 @@ public void setBottom(int slot) { Rectangle box = Rectangle.fromPointDistance(latitude, longitude, haversin2(bottom)); // pre-encode our box to our integer encoding, so we don't have to decode // to double values for uncompetitive hits. This has some cost! - minLat = LatLonPoint.encodeLatitude(box.minLat); - maxLat = LatLonPoint.encodeLatitude(box.maxLat); + minLat = encodeLatitude(box.minLat); + maxLat = encodeLatitude(box.maxLat); if (box.crossesDateline()) { // box1 minLon = Integer.MIN_VALUE; - maxLon = LatLonPoint.encodeLongitude(box.maxLon); + maxLon = encodeLongitude(box.maxLon); // box2 - minLon2 = LatLonPoint.encodeLongitude(box.minLon); + minLon2 = encodeLongitude(box.minLon); } else { - minLon = LatLonPoint.encodeLongitude(box.minLon); - maxLon = LatLonPoint.encodeLongitude(box.maxLon); + minLon = encodeLongitude(box.minLon); + maxLon = encodeLongitude(box.maxLon); // disable box2 minLon2 = Integer.MAX_VALUE; } @@ -132,8 +137,8 @@ public int compareBottom(int doc) throws IOException { } // only compute actual distance if its inside "competitive bounding box" - double docLatitude = LatLonPoint.decodeLatitude(latitudeBits); - double docLongitude = LatLonPoint.decodeLongitude(longitudeBits); + double docLatitude = decodeLatitude(latitudeBits); + double docLongitude = decodeLongitude(longitudeBits); cmp = Math.max(cmp, Double.compare(bottom, SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude))); // once we compete in the PQ, no need to continue. if (cmp > 0) { @@ -178,8 +183,8 @@ public int compareTop(int doc) throws IOException { int numValues = currentDocs.count(); for (int i = 0; i < numValues; i++) { long encoded = currentDocs.valueAt(i); - double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); - double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); + double docLatitude = decodeLatitude((int)(encoded >> 32)); + double docLongitude = decodeLongitude((int)(encoded & 0xFFFFFFFF)); minValue = Math.min(minValue, SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude)); } return minValue; diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java index 1b667dd25008..843421bbaa90 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java @@ -45,6 +45,11 @@ import org.apache.lucene.util.SparseFixedBitSet; import org.apache.lucene.util.StringHelper; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** * Distance query for {@link LatLonPoint}. */ @@ -81,19 +86,19 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I // second set of longitude ranges to check (for cross-dateline case) final byte minLon2[] = new byte[Integer.BYTES]; - NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.minLat), minLat, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.maxLat), maxLat, 0); + NumericUtils.intToSortableBytes(encodeLatitude(box.minLat), minLat, 0); + NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0); // crosses dateline: split if (box.crossesDateline()) { // box1 NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minLon, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0); // box2 - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon2, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon2, 0); } else { - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0); // disable box2 NumericUtils.intToSortableBytes(Integer.MAX_VALUE, minLon2, 0); } @@ -191,10 +196,10 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return Relation.CELL_OUTSIDE_QUERY; } - double latMin = LatLonPoint.decodeLatitude(minPackedValue, 0); - double lonMin = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES); - double latMax = LatLonPoint.decodeLatitude(maxPackedValue, 0); - double lonMax = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); + double latMin = decodeLatitude(minPackedValue, 0); + double lonMin = decodeLongitude(minPackedValue, Integer.BYTES); + double latMax = decodeLatitude(maxPackedValue, 0); + double lonMax = decodeLongitude(maxPackedValue, Integer.BYTES); if ((longitude < lonMin || longitude > lonMax) && (axisLat+ Rectangle.AXISLAT_ERROR < latMin || axisLat- Rectangle.AXISLAT_ERROR > latMax)) { // circle not fully inside / crossing axis @@ -240,8 +245,8 @@ public boolean matches() throws IOException { int count = docValues.count(); for (int i = 0; i < count; i++) { long encoded = docValues.valueAt(i); - double docLatitude = LatLonPoint.decodeLatitude((int)(encoded >> 32)); - double docLongitude = LatLonPoint.decodeLongitude((int)(encoded & 0xFFFFFFFF)); + double docLatitude = decodeLatitude((int)(encoded >> 32)); + double docLongitude = decodeLongitude((int)(encoded & 0xFFFFFFFF)); // first check the partial distance, if its more than that, it can't be <= radiusMeters double h1 = SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude); diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java index 670db3de67a3..7c504e2c7e24 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java @@ -45,6 +45,11 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.geo.Polygon; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** Finds all previously indexed points that fall within the specified polygons. * *

    The field must be indexed with using {@link org.apache.lucene.document.LatLonPoint} added per document. @@ -88,10 +93,10 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I final byte maxLat[] = new byte[Integer.BYTES]; final byte minLon[] = new byte[Integer.BYTES]; final byte maxLon[] = new byte[Integer.BYTES]; - NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.minLat), minLat, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLatitude(box.maxLat), maxLat, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.minLon), minLon, 0); - NumericUtils.intToSortableBytes(LatLonPoint.encodeLongitude(box.maxLon), maxLon, 0); + NumericUtils.intToSortableBytes(encodeLatitude(box.minLat), minLat, 0); + NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon, 0); + NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0); // TODO: make this fancier, but currently linear with number of vertices float cumulativeCost = 0; @@ -100,10 +105,10 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I } final float matchCost = cumulativeCost; - final LatLonGrid grid = new LatLonGrid(LatLonPoint.encodeLatitude(box.minLat), - LatLonPoint.encodeLatitude(box.maxLat), - LatLonPoint.encodeLongitude(box.minLon), - LatLonPoint.encodeLongitude(box.maxLon), polygons); + final LatLonGrid grid = new LatLonGrid(encodeLatitude(box.minLat), + encodeLatitude(box.maxLat), + encodeLongitude(box.minLon), + encodeLongitude(box.maxLon), polygons); return new ConstantScoreWeight(this) { @@ -164,10 +169,10 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { return Relation.CELL_OUTSIDE_QUERY; } - double cellMinLat = LatLonPoint.decodeLatitude(minPackedValue, 0); - double cellMinLon = LatLonPoint.decodeLongitude(minPackedValue, Integer.BYTES); - double cellMaxLat = LatLonPoint.decodeLatitude(maxPackedValue, 0); - double cellMaxLon = LatLonPoint.decodeLongitude(maxPackedValue, Integer.BYTES); + double cellMinLat = decodeLatitude(minPackedValue, 0); + double cellMinLon = decodeLongitude(minPackedValue, Integer.BYTES); + double cellMaxLat = decodeLatitude(maxPackedValue, 0); + double cellMaxLon = decodeLongitude(maxPackedValue, Integer.BYTES); return Polygon.relate(polygons, cellMinLat, cellMaxLat, cellMinLon, cellMaxLon); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java index 2de5a433d6f1..0b739aee25ec 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonGrid.java @@ -22,6 +22,11 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** tests against LatLonGrid (avoiding indexing/queries) */ public class TestLatLonGrid extends LuceneTestCase { @@ -30,18 +35,18 @@ public void testRandom() throws Exception { for (int i = 0; i < 100; i++) { Polygon polygon = GeoTestUtil.nextPolygon(); Rectangle box = Rectangle.fromPolygon(new Polygon[] { polygon }); - int minLat = LatLonPoint.encodeLatitude(box.minLat); - int maxLat = LatLonPoint.encodeLatitude(box.maxLat); - int minLon = LatLonPoint.encodeLongitude(box.minLon); - int maxLon = LatLonPoint.encodeLongitude(box.maxLon); + int minLat = encodeLatitude(box.minLat); + int maxLat = encodeLatitude(box.maxLat); + int minLon = encodeLongitude(box.minLon); + int maxLon = encodeLongitude(box.maxLon); LatLonGrid grid = new LatLonGrid(minLat, maxLat, minLon, maxLon, polygon); // we are in integer space... but exhaustive testing is slow! for (int j = 0; j < 10000; j++) { int lat = TestUtil.nextInt(random(), minLat, maxLat); int lon = TestUtil.nextInt(random(), minLon, maxLon); - boolean expected = polygon.contains(LatLonPoint.decodeLatitude(lat), - LatLonPoint.decodeLongitude(lon)); + boolean expected = polygon.contains(decodeLatitude(lat), + decodeLongitude(lon)); boolean actual = grid.contains(lat, lon); assertEquals(expected, actual); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java index fb8e0dc65892..acc9186b14f5 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPoint.java @@ -16,13 +16,12 @@ */ package org.apache.lucene.document; -import java.util.Random; - import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TestUtil; -/** Simple tests for {@link LatLonPoint} */ +/** + * Simple tests for {@link LatLonPoint} + * TODO: move this lone test and remove class? + * */ public class TestLatLonPoint extends LuceneTestCase { public void testToString() throws Exception { @@ -38,113 +37,4 @@ public void testToString() throws Exception { // sort field assertEquals("", LatLonPoint.newDistanceSort("field", 18.0, 19.0).toString()); } - - /** - * step through some integers, ensuring they decode to their expected double values. - * double values start at -90 and increase by LATITUDE_DECODE for each integer. - * check edge cases within the double range and random doubles within the range too. - */ - public void testLatitudeQuantization() throws Exception { - Random random = random(); - for (int i = 0; i < 10000; i++) { - int encoded = random.nextInt(); - double min = -90.0 + (encoded - (long)Integer.MIN_VALUE) * LatLonPoint.LATITUDE_DECODE; - double decoded = LatLonPoint.decodeLatitude(encoded); - // should exactly equal expected value - assertEquals(min, decoded, 0.0D); - // should round-trip - assertEquals(encoded, LatLonPoint.encodeLatitude(decoded)); - assertEquals(encoded, LatLonPoint.encodeLatitudeCeil(decoded)); - // test within the range - if (i != Integer.MAX_VALUE) { - // this is the next representable value - // all double values between [min .. max) should encode to the current integer - // all double values between (min .. max] should encodeCeil to the next integer. - double max = min + LatLonPoint.LATITUDE_DECODE; - assertEquals(max, LatLonPoint.decodeLatitude(encoded+1), 0.0D); - assertEquals(encoded+1, LatLonPoint.encodeLatitude(max)); - assertEquals(encoded+1, LatLonPoint.encodeLatitudeCeil(max)); - - // first and last doubles in range that will be quantized - double minEdge = Math.nextUp(min); - double maxEdge = Math.nextDown(max); - assertEquals(encoded, LatLonPoint.encodeLatitude(minEdge)); - assertEquals(encoded+1, LatLonPoint.encodeLatitudeCeil(minEdge)); - assertEquals(encoded, LatLonPoint.encodeLatitude(maxEdge)); - assertEquals(encoded+1, LatLonPoint.encodeLatitudeCeil(maxEdge)); - - // check random values within the double range - long minBits = NumericUtils.doubleToSortableLong(minEdge); - long maxBits = NumericUtils.doubleToSortableLong(maxEdge); - for (int j = 0; j < 100; j++) { - double value = NumericUtils.sortableLongToDouble(TestUtil.nextLong(random, minBits, maxBits)); - // round down - assertEquals(encoded, LatLonPoint.encodeLatitude(value)); - // round up - assertEquals(encoded+1, LatLonPoint.encodeLatitudeCeil(value)); - } - } - } - } - - /** - * step through some integers, ensuring they decode to their expected double values. - * double values start at -180 and increase by LONGITUDE_DECODE for each integer. - * check edge cases within the double range and a random doubles within the range too. - */ - public void testLongitudeQuantization() throws Exception { - Random random = random(); - for (int i = 0; i < 10000; i++) { - int encoded = random.nextInt(); - double min = -180.0 + (encoded - (long)Integer.MIN_VALUE) * LatLonPoint.LONGITUDE_DECODE; - double decoded = LatLonPoint.decodeLongitude(encoded); - // should exactly equal expected value - assertEquals(min, decoded, 0.0D); - // should round-trip - assertEquals(encoded, LatLonPoint.encodeLongitude(decoded)); - assertEquals(encoded, LatLonPoint.encodeLongitudeCeil(decoded)); - // test within the range - if (i != Integer.MAX_VALUE) { - // this is the next representable value - // all double values between [min .. max) should encode to the current integer - // all double values between (min .. max] should encodeCeil to the next integer. - double max = min + LatLonPoint.LONGITUDE_DECODE; - assertEquals(max, LatLonPoint.decodeLongitude(encoded+1), 0.0D); - assertEquals(encoded+1, LatLonPoint.encodeLongitude(max)); - assertEquals(encoded+1, LatLonPoint.encodeLongitudeCeil(max)); - - // first and last doubles in range that will be quantized - double minEdge = Math.nextUp(min); - double maxEdge = Math.nextDown(max); - assertEquals(encoded, LatLonPoint.encodeLongitude(minEdge)); - assertEquals(encoded+1, LatLonPoint.encodeLongitudeCeil(minEdge)); - assertEquals(encoded, LatLonPoint.encodeLongitude(maxEdge)); - assertEquals(encoded+1, LatLonPoint.encodeLongitudeCeil(maxEdge)); - - // check random values within the double range - long minBits = NumericUtils.doubleToSortableLong(minEdge); - long maxBits = NumericUtils.doubleToSortableLong(maxEdge); - for (int j = 0; j < 100; j++) { - double value = NumericUtils.sortableLongToDouble(TestUtil.nextLong(random, minBits, maxBits)); - // round down - assertEquals(encoded, LatLonPoint.encodeLongitude(value)); - // round up - assertEquals(encoded+1, LatLonPoint.encodeLongitudeCeil(value)); - } - } - } - } - - // check edge/interesting cases explicitly - public void testEncodeEdgeCases() { - assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLatitude(-90.0)); - assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLatitudeCeil(-90.0)); - assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLatitude(90.0)); - assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLatitudeCeil(90.0)); - - assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLongitude(-180.0)); - assertEquals(Integer.MIN_VALUE, LatLonPoint.encodeLongitudeCeil(-180.0)); - assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLongitude(180.0)); - assertEquals(Integer.MAX_VALUE, LatLonPoint.encodeLongitudeCeil(180.0)); - } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java index 0003b302d40f..e40f33d864b9 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java +++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointDistanceSort.java @@ -35,6 +35,11 @@ import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude; +import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude; + /** Simple tests for {@link LatLonPoint#newDistanceSort} */ public class TestLatLonPointDistanceSort extends LuceneTestCase { @@ -191,8 +196,8 @@ private void doRandomTest(int numDocs, int numQueries) throws IOException { double latRaw = GeoTestUtil.nextLatitude(); double lonRaw = GeoTestUtil.nextLongitude(); // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons - double lat = LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); - double lon = LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); + double lat = decodeLatitude(encodeLatitude(latRaw)); + double lon = decodeLongitude(encodeLongitude(lonRaw)); doc.add(new LatLonPoint("field", lat, lon)); doc.add(new StoredField("lat", lat)); diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java index 08b9ebf6d7c9..8c909e6b5931 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/TestLatLonPointQueries.java @@ -20,6 +20,7 @@ import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.GeoEncodingUtils; public class TestLatLonPointQueries extends BaseGeoPointTestCase { @@ -45,11 +46,11 @@ protected Query newPolygonQuery(String field, Polygon... polygons) { @Override protected double quantizeLat(double latRaw) { - return LatLonPoint.decodeLatitude(LatLonPoint.encodeLatitude(latRaw)); + return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(latRaw)); } @Override protected double quantizeLon(double lonRaw) { - return LatLonPoint.decodeLongitude(LatLonPoint.encodeLongitude(lonRaw)); + return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lonRaw)); } } From f7a25f07ad69ddf1b441fd0d60614d49597dd660 Mon Sep 17 00:00:00 2001 From: nknize Date: Thu, 7 Apr 2016 15:23:04 -0500 Subject: [PATCH 0302/1113] fix bug in testLat/LonQuantization --- .../src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java index 3b71c6388e14..f13e6c7d21ba 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoEncodingUtils.java @@ -56,7 +56,7 @@ public void testLatitudeQuantization() throws Exception { assertEquals(encoded, encodeLatitude(decoded)); assertEquals(encoded, encodeLatitudeCeil(decoded)); // test within the range - if (i != Integer.MAX_VALUE) { + if (encoded != Integer.MAX_VALUE) { // this is the next representable value // all double values between [min .. max) should encode to the current integer // all double values between (min .. max] should encodeCeil to the next integer. @@ -105,7 +105,7 @@ public void testLongitudeQuantization() throws Exception { assertEquals(encoded, encodeLongitude(decoded)); assertEquals(encoded, encodeLongitudeCeil(decoded)); // test within the range - if (i != Integer.MAX_VALUE) { + if (encoded != Integer.MAX_VALUE) { // this is the next representable value // all double values between [min .. max) should encode to the current integer // all double values between (min .. max] should encodeCeil to the next integer. From 347b9f0d7f81ab881ac28b34b5f4477a82811321 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 7 Apr 2016 17:54:39 -0400 Subject: [PATCH 0303/1113] LUCENE-7189: make it easier to write WebGL earth HTML for debugging geo failures --- .../org/apache/lucene/geo/TestGeoUtils.java | 5 +- .../org/apache/lucene/geo/EarthDebugger.java | 267 ++++++++++++++++++ .../org/apache/lucene/geo/GeoTestUtil.java | 228 --------------- 3 files changed, 271 insertions(+), 229 deletions(-) create mode 100644 lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java diff --git a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java index 8727b42afa6b..48db105b0847 100644 --- a/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java +++ b/lucene/core/src/test/org/apache/lucene/geo/TestGeoUtils.java @@ -308,7 +308,10 @@ public void testCircleOpto() throws Exception { centerLat, centerLon, lat, lon, distance, Rectangle.fromPointDistance(centerLat, centerLon, radius)), distance > radius); } catch (AssertionError e) { - GeoTestUtil.toWebGLEarth(latMin, latMax, lonMin, lonMax, centerLat, centerLon, radius); + EarthDebugger ed = new EarthDebugger(); + ed.addRect(latMin, latMax, lonMin, lonMax); + ed.addCircle(centerLat, centerLon, radius, true); + System.out.println(ed.finish()); throw e; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java new file mode 100644 index 000000000000..14956f2c1675 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java @@ -0,0 +1,267 @@ +package org.apache.lucene.geo; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.util.SloppyMath; + +/** Draws shapes on the earth surface and renders using the very cool http://www.webglearth.org. + * + * Just instantiate this class, add the things you want plotted, and call {@link #finish} to get the + * resulting HTML that you should save and load with a browser. */ + +public class EarthDebugger { + final StringBuilder b = new StringBuilder(); + private int nextShape; + + public EarthDebugger() { + b.append("\n"); + b.append("\n"); + b.append(" \n"); + b.append(" \n"); + b.append(" \n"); + b.append(" \n"); + b.append(" WebGL Earth API: Hello World\n"); + b.append(" \n"); + b.append(" \n"); + b.append("

    \n"); + b.append(" \n"); + b.append("\n"); + + return b.toString(); + } + + private static void inverseHaversin(StringBuilder b, double centerLat, double centerLon, double radiusMeters) { + double angle = 0; + int steps = 100; + + newAngle: + while (angle < 360) { + double x = Math.cos(Math.toRadians(angle)); + double y = Math.sin(Math.toRadians(angle)); + double factor = 2.0; + double step = 1.0; + int last = 0; + double lastDistanceMeters = 0.0; + //System.out.println("angle " + angle + " slope=" + slope); + while (true) { + double lat = wrapLat(centerLat + y * factor); + double lon = wrapLon(centerLon + x * factor); + double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, lat, lon); + + if (last == 1 && distanceMeters < lastDistanceMeters) { + // For large enough circles, some angles are not possible: + //System.out.println(" done: give up on angle " + angle); + angle += 360./steps; + continue newAngle; + } + if (last == -1 && distanceMeters > lastDistanceMeters) { + // For large enough circles, some angles are not possible: + //System.out.println(" done: give up on angle " + angle); + angle += 360./steps; + continue newAngle; + } + lastDistanceMeters = distanceMeters; + + //System.out.println(" iter lat=" + lat + " lon=" + lon + " distance=" + distanceMeters + " vs " + radiusMeters); + if (Math.abs(distanceMeters - radiusMeters) < 0.1) { + b.append(" [" + lat + ", " + lon + "],\n"); + break; + } + if (distanceMeters > radiusMeters) { + // too big + //System.out.println(" smaller"); + factor -= step; + if (last == 1) { + //System.out.println(" half-step"); + step /= 2.0; + } + last = -1; + } else if (distanceMeters < radiusMeters) { + // too small + //System.out.println(" bigger"); + factor += step; + if (last == -1) { + //System.out.println(" half-step"); + step /= 2.0; + } + last = 1; + } + } + angle += 360./steps; + } + } + // craziness for plotting stuff :) + + private static double wrapLat(double lat) { + //System.out.println("wrapLat " + lat); + if (lat > 90) { + //System.out.println(" " + (180 - lat)); + return 180 - lat; + } else if (lat < -90) { + //System.out.println(" " + (-180 - lat)); + return -180 - lat; + } else { + //System.out.println(" " + lat); + return lat; + } + } + + private static double wrapLon(double lon) { + //System.out.println("wrapLon " + lon); + if (lon > 180) { + //System.out.println(" " + (lon - 360)); + return lon - 360; + } else if (lon < -180) { + //System.out.println(" " + (lon + 360)); + return lon + 360; + } else { + //System.out.println(" " + lon); + return lon; + } + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java index 0d241f6dc66e..23e34164668d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java @@ -281,232 +281,4 @@ private static double normalizeLongitude(double longitude) { private static Random random() { return RandomizedContext.current().getRandom(); } - - // craziness for plotting stuff :) - - private static double wrapLat(double lat) { - //System.out.println("wrapLat " + lat); - if (lat > 90) { - //System.out.println(" " + (180 - lat)); - return 180 - lat; - } else if (lat < -90) { - //System.out.println(" " + (-180 - lat)); - return -180 - lat; - } else { - //System.out.println(" " + lat); - return lat; - } - } - - private static double wrapLon(double lon) { - //System.out.println("wrapLon " + lon); - if (lon > 180) { - //System.out.println(" " + (lon - 360)); - return lon - 360; - } else if (lon < -180) { - //System.out.println(" " + (lon + 360)); - return lon + 360; - } else { - //System.out.println(" " + lon); - return lon; - } - } - - private static void drawRectApproximatelyOnEarthSurface(String name, String color, double minLat, double maxLat, double minLon, double maxLon) { - int steps = 20; - System.out.println(" var " + name + " = WE.polygon(["); - System.out.println(" // min -> max lat, min lon"); - for(int i=0;i max lon"); - for(int i=0;i min lat, max lon"); - for(int i=0;i min lon"); - for(int i=0;i=minLon;lon -= (maxLon-minLon)/36) { - System.out.println(" [" + lat + ", " + lon + "],"); - } - System.out.println(" ], {color: \"" + color + "\", fillColor: \"#ffffff\", opacity: " + (color.equals("#ffffff") ? "0.3" : "1") + ", fillOpacity: 0.0001});"); - System.out.println(" " + name + ".addTo(earth);"); - } - - private static void plotLonApproximatelyOnEarthSurface(String name, String color, double lon, double minLat, double maxLat) { - System.out.println(" var " + name + " = WE.polygon(["); - double lat; - for(lat = minLat;lat<=maxLat;lat += (maxLat-minLat)/36) { - System.out.println(" [" + lat + ", " + lon + "],"); - } - System.out.println(" [" + maxLat + ", " + lon + "],"); - lat -= (maxLat-minLat)/36; - for(;lat>=minLat;lat -= (maxLat-minLat)/36) { - System.out.println(" [" + lat + ", " + lon + "],"); - } - System.out.println(" ], {color: \"" + color + "\", fillColor: \"#ffffff\", opacity: " + (color.equals("#ffffff") ? "0.3" : "1") + ", fillOpacity: 0.0001});"); - System.out.println(" " + name + ".addTo(earth);"); - } - - // http://www.webglearth.org has API details: - public static void polysToWebGLEarth(List polys) { - System.out.println(""); - System.out.println(""); - System.out.println(" "); - System.out.println(" "); - System.out.println(" "); - System.out.println(" "); - System.out.println(" WebGL Earth API: Hello World"); - System.out.println(" "); - System.out.println(" "); - System.out.println("
    "); - System.out.println(" "); - System.out.println(""); - } - - // http://www.webglearth.org has API details: - public static void toWebGLEarth(double rectMinLatitude, double rectMaxLatitude, - double rectMinLongitude, double rectMaxLongitude, - double centerLatitude, double centerLongitude, - double radiusMeters) { - Rectangle box = Rectangle.fromPointDistance(centerLatitude, centerLongitude, radiusMeters); - System.out.println(""); - System.out.println(""); - System.out.println(" "); - System.out.println(" "); - System.out.println(" "); - System.out.println(" "); - System.out.println(" WebGL Earth API: Hello World"); - System.out.println(" "); - System.out.println(" "); - System.out.println("
    "); - System.out.println(" "); - System.out.println(""); - } - - private static void inverseHaversin(StringBuilder b, double centerLat, double centerLon, double radiusMeters) { - double angle = 0; - int steps = 100; - - newAngle: - while (angle < 360) { - double x = Math.cos(Math.toRadians(angle)); - double y = Math.sin(Math.toRadians(angle)); - double factor = 2.0; - double step = 1.0; - int last = 0; - double lastDistanceMeters = 0.0; - //System.out.println("angle " + angle + " slope=" + slope); - while (true) { - double lat = wrapLat(centerLat + y * factor); - double lon = wrapLon(centerLon + x * factor); - double distanceMeters = SloppyMath.haversinMeters(centerLat, centerLon, lat, lon); - - if (last == 1 && distanceMeters < lastDistanceMeters) { - // For large enough circles, some angles are not possible: - //System.out.println(" done: give up on angle " + angle); - angle += 360./steps; - continue newAngle; - } - if (last == -1 && distanceMeters > lastDistanceMeters) { - // For large enough circles, some angles are not possible: - //System.out.println(" done: give up on angle " + angle); - angle += 360./steps; - continue newAngle; - } - lastDistanceMeters = distanceMeters; - - //System.out.println(" iter lat=" + lat + " lon=" + lon + " distance=" + distanceMeters + " vs " + radiusMeters); - if (Math.abs(distanceMeters - radiusMeters) < 0.1) { - b.append(" [" + lat + ", " + lon + "],\n"); - break; - } - if (distanceMeters > radiusMeters) { - // too big - //System.out.println(" smaller"); - factor -= step; - if (last == 1) { - //System.out.println(" half-step"); - step /= 2.0; - } - last = -1; - } else if (distanceMeters < radiusMeters) { - // too small - //System.out.println(" bigger"); - factor += step; - if (last == -1) { - //System.out.println(" half-step"); - step /= 2.0; - } - last = 1; - } - } - angle += 360./steps; - } - } } From ecbf1a4d83d4816241679c37f196c1f5b3d1b88c Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Thu, 7 Apr 2016 17:59:33 -0400 Subject: [PATCH 0304/1113] LUCENE-7173: Iterate at least 100 times each for polygon construction when testing. --- .../apache/lucene/spatial3d/TestGeo3DPoint.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 410618c829e6..5aaa8357a44b 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -759,10 +759,16 @@ public void testComplexPolygons() { final PlanetModel pm = PlanetModel.WGS84; // Pick a random pole final GeoPoint randomPole = new GeoPoint(pm, Math.toRadians(GeoTestUtil.nextLatitude()), Math.toRadians(GeoTestUtil.nextLongitude())); - // Create a polygon that's less than 180 degrees - final Polygon clockWise = makePoly(pm, randomPole, true, true); - // Create a polygon that's greater than 180 degrees - final Polygon counterClockWise = makePoly(pm, randomPole, false, true); + int iters = atLeast(100); + for (int i = 0; i < iters; i++) { + // Create a polygon that's less than 180 degrees + final Polygon clockWise = makePoly(pm, randomPole, true, true); + } + iters = atLeast(100); + for (int i = 0; i < iters; i++) { + // Create a polygon that's greater than 180 degrees + final Polygon counterClockWise = makePoly(pm, randomPole, false, true); + } } protected static double MINIMUM_EDGE_ANGLE = Math.toRadians(5.0); From 48f1d56c7e39d718ddda5a2e9b5e4e45423222c5 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Thu, 7 Apr 2016 18:31:19 -0400 Subject: [PATCH 0305/1113] LUCENE-7190: Make some methods private in public classes, and make a very few constants public. --- .../spatial3d/geom/GeoPolygonFactory.java | 20 ++++----- .../lucene/spatial3d/geom/LatLonBounds.java | 18 ++++---- .../apache/lucene/spatial3d/geom/Plane.java | 44 +++++-------------- .../lucene/spatial3d/geom/XYZBounds.java | 20 ++++----- 4 files changed, 40 insertions(+), 62 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index a58056fcf0bf..144c4a530095 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -175,7 +175,7 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, * @return null if the point is illegal, otherwise false if the point is inside and true if the point is outside * of the polygon. */ - protected static Boolean isInsidePolygon(final GeoPoint point, final List polyPoints) { + private static Boolean isInsidePolygon(final GeoPoint point, final List polyPoints) { // First, compute sine and cosine of pole point latitude and longitude final double norm = 1.0 / point.magnitude(); final double xyDenom = Math.sqrt(point.x * point.x + point.y * point.y); @@ -255,7 +255,7 @@ protected static Boolean isInsidePolygon(final GeoPoint point, final List holes, @@ -572,7 +572,7 @@ protected static boolean makeConcavePolygon(final PlanetModel planetModel, * @param testPoint is the optional test point. * @return null if the testPoint is within any polygon detected, otherwise true if a convex polygon was created. */ - protected static Boolean findConvexPolygon(final PlanetModel planetModel, + private static Boolean findConvexPolygon(final PlanetModel planetModel, final Edge currentEdge, final GeoCompositePolygon rval, final EdgeBuffer edgeBuffer, @@ -786,7 +786,7 @@ protected static Boolean findConvexPolygon(final PlanetModel planetModel, * @param returnBoundary is the return edge * @return true if within */ - protected static boolean isWithin(final GeoPoint point, final Set edgeSet, final Edge extension, final SidedPlane returnBoundary) { + private static boolean isWithin(final GeoPoint point, final Set edgeSet, final Edge extension, final SidedPlane returnBoundary) { if (!extension.plane.isWithin(point)) { return false; } @@ -801,7 +801,7 @@ protected static boolean isWithin(final GeoPoint point, final Set edgeSet, * @param edgeSet is the set of edges * @return true if within */ - protected static boolean isWithin(final GeoPoint point, final Set edgeSet) { + private static boolean isWithin(final GeoPoint point, final Set edgeSet) { for (final Edge edge : edgeSet) { if (!edge.plane.isWithin(point)) { return false; @@ -815,7 +815,7 @@ protected static boolean isWithin(final GeoPoint point, final Set edgeSet) *@param size is the array size. *@return an updated index. */ - protected static int getLegalIndex(int index, int size) { + private static int getLegalIndex(int index, int size) { while (index < 0) { index += size; } @@ -827,7 +827,7 @@ protected static int getLegalIndex(int index, int size) { /** Class representing a single (unused) edge. */ - protected static class Edge { + private static class Edge { /** Plane */ public final SidedPlane plane; /** Start point */ @@ -863,7 +863,7 @@ public boolean equals(final Object o) { /** Class representing an iterator over an EdgeBuffer. */ - protected static class EdgeBufferIterator implements Iterator { + private static class EdgeBufferIterator implements Iterator { /** Edge buffer */ protected final EdgeBuffer edgeBuffer; /** First edge */ @@ -905,7 +905,7 @@ public void remove() { /** Class representing a pool of unused edges, all linked together by vertices. */ - protected static class EdgeBuffer { + private static class EdgeBuffer { /** Starting edge */ protected Edge oneEdge; /** Full set of edges */ diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java index 627fdaec996f..8f57efd5841d 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java @@ -24,16 +24,16 @@ public class LatLonBounds implements Bounds { /** Set to true if no longitude bounds can be stated */ - protected boolean noLongitudeBound = false; + private boolean noLongitudeBound = false; /** Set to true if no top latitude bound can be stated */ - protected boolean noTopLatitudeBound = false; + private boolean noTopLatitudeBound = false; /** Set to true if no bottom latitude bound can be stated */ - protected boolean noBottomLatitudeBound = false; + private boolean noBottomLatitudeBound = false; /** If non-null, the minimum latitude bound */ - protected Double minLatitude = null; + private Double minLatitude = null; /** If non-null, the maximum latitude bound */ - protected Double maxLatitude = null; + private Double maxLatitude = null; // For longitude bounds, this class needs to worry about keeping track of the distinction // between left-side bounds and right-side bounds. Points are always submitted in pairs @@ -88,9 +88,9 @@ public class LatLonBounds implements Bounds { // and can recognize that, we can set "unconstrained in longitude".) /** If non-null, the left longitude bound */ - protected Double leftLongitude = null; + private Double leftLongitude = null; /** If non-null, the right longitude bound */ - protected Double rightLongitude = null; + private Double rightLongitude = null; /** Construct an empty bounds object */ public LatLonBounds() { @@ -252,7 +252,7 @@ public Bounds noBottomLatitudeBound() { /** Update latitude bound. *@param latitude is the latitude. */ - protected void addLatitudeBound(double latitude) { + private void addLatitudeBound(double latitude) { if (!noTopLatitudeBound && (maxLatitude == null || latitude > maxLatitude)) maxLatitude = latitude; if (!noBottomLatitudeBound && (minLatitude == null || latitude < minLatitude)) @@ -262,7 +262,7 @@ protected void addLatitudeBound(double latitude) { /** Update longitude bound. *@param longitude is the new longitude value. */ - protected void addLongitudeBound(double longitude) { + private void addLongitudeBound(double longitude) { // If this point is within the current bounds, we're done; otherwise // expand one side or the other. if (leftLongitude == null && rightLongitude == null) { diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java index 1f2c054791ba..29c400a3f162 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java @@ -24,15 +24,15 @@ */ public class Plane extends Vector { /** An array with no points in it */ - protected final static GeoPoint[] NO_POINTS = new GeoPoint[0]; + public final static GeoPoint[] NO_POINTS = new GeoPoint[0]; /** An array with no bounds in it */ - protected final static Membership[] NO_BOUNDS = new Membership[0]; + public final static Membership[] NO_BOUNDS = new Membership[0]; /** A vertical plane normal to the Y axis */ - protected final static Plane normalYPlane = new Plane(0.0,1.0,0.0,0.0); + public final static Plane normalYPlane = new Plane(0.0,1.0,0.0,0.0); /** A vertical plane normal to the X axis */ - protected final static Plane normalXPlane = new Plane(1.0,0.0,0.0,0.0); + public final static Plane normalXPlane = new Plane(1.0,0.0,0.0,0.0); /** A vertical plane normal to the Z axis */ - protected final static Plane normalZPlane = new Plane(0.0,0.0,1.0,0.0); + public final static Plane normalZPlane = new Plane(0.0,0.0,1.0,0.0); /** Ax + By + Cz + D = 0 */ public final double D; @@ -1463,7 +1463,7 @@ public void recordBounds(final PlanetModel planetModel, final LatLonBounds bound * @param bounds is the area that the point must be within. * @param point is the point. */ - protected static void addPoint(final Bounds boundsInfo, final Membership[] bounds, final GeoPoint point) { + private static void addPoint(final Bounds boundsInfo, final Membership[] bounds, final GeoPoint point) { // Make sure the discovered point is within the bounds for (Membership bound : bounds) { if (!bound.isWithin(point)) @@ -1473,28 +1473,6 @@ protected static void addPoint(final Bounds boundsInfo, final Membership[] bound boundsInfo.addPoint(point); } - /** Add a point to boundsInfo if within a specifically bounded area. - * @param boundsInfo is the object to be modified. - * @param bounds is the area that the point must be within. - * @param x is the x value. - * @param y is the y value. - * @param z is the z value. - */ - /* - protected static void addPoint(final Bounds boundsInfo, final Membership[] bounds, final double x, final double y, final double z) { - //System.err.println(" Want to add point x="+x+" y="+y+" z="+z); - // Make sure the discovered point is within the bounds - for (Membership bound : bounds) { - if (!bound.isWithin(x, y, z)) - return; - } - // Add the point - //System.err.println(" point added"); - //System.out.println("Adding point x="+x+" y="+y+" z="+z); - boundsInfo.addPoint(x, y, z); - } - */ - /** * Determine whether the plane intersects another plane within the * bounds provided. @@ -1540,7 +1518,7 @@ public boolean intersects(final PlanetModel planetModel, final Plane q, final Ge * @param p is the plane to compare against. * @return true if the planes are numerically identical. */ - protected boolean isNumericallyIdentical(final Plane p) { + public boolean isNumericallyIdentical(final Plane p) { // We can get the correlation by just doing a parallel plane check. If that passes, then compute a point on the plane // (using D) and see if it also on the other plane. if (Math.abs(this.y * p.z - this.z * p.y) >= MINIMUM_RESOLUTION) @@ -1573,7 +1551,7 @@ protected boolean isNumericallyIdentical(final Plane p) { * @param bounds are the bounds. * @return true if the vector describes a point within the bounds. */ - protected static boolean meetsAllBounds(final Vector p, final Membership[] bounds) { + private static boolean meetsAllBounds(final Vector p, final Membership[] bounds) { return meetsAllBounds(p.x, p.y, p.z, bounds); } @@ -1585,7 +1563,7 @@ protected static boolean meetsAllBounds(final Vector p, final Membership[] bound * @param bounds are the bounds. * @return true if the vector describes a point within the bounds. */ - protected static boolean meetsAllBounds(final double x, final double y, final double z, final Membership[] bounds) { + private static boolean meetsAllBounds(final double x, final double y, final double z, final Membership[] bounds) { for (final Membership bound : bounds) { if (!bound.isWithin(x,y,z)) return false; @@ -1600,7 +1578,7 @@ protected static boolean meetsAllBounds(final double x, final double y, final do * @param moreBounds are an additional set of bounds. * @return true if the vector describes a point within the bounds. */ - protected static boolean meetsAllBounds(final Vector p, final Membership[] bounds, final Membership[] moreBounds) { + private static boolean meetsAllBounds(final Vector p, final Membership[] bounds, final Membership[] moreBounds) { return meetsAllBounds(p.x, p.y, p.z, bounds, moreBounds); } @@ -1613,7 +1591,7 @@ protected static boolean meetsAllBounds(final Vector p, final Membership[] bound * @param moreBounds are an additional set of bounds. * @return true if the vector describes a point within the bounds. */ - protected static boolean meetsAllBounds(final double x, final double y, final double z, final Membership[] bounds, + private static boolean meetsAllBounds(final double x, final double y, final double z, final Membership[] bounds, final Membership[] moreBounds) { return meetsAllBounds(x,y,z, bounds) && meetsAllBounds(x,y,z, moreBounds); } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java index c3ee53d4166c..3ab2701290ed 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java @@ -29,27 +29,27 @@ public class XYZBounds implements Bounds { * of the shape, and we cannot guarantee that without making MINIMUM_RESOLUTION * unacceptably large. */ - protected static final double FUDGE_FACTOR = Vector.MINIMUM_RESOLUTION * 2.0; + private static final double FUDGE_FACTOR = Vector.MINIMUM_RESOLUTION * 2.0; /** Minimum x */ - protected Double minX = null; + private Double minX = null; /** Maximum x */ - protected Double maxX = null; + private Double maxX = null; /** Minimum y */ - protected Double minY = null; + private Double minY = null; /** Maximum y */ - protected Double maxY = null; + private Double maxY = null; /** Minimum z */ - protected Double minZ = null; + private Double minZ = null; /** Maximum z */ - protected Double maxZ = null; + private Double maxZ = null; /** Set to true if no longitude bounds can be stated */ - protected boolean noLongitudeBound = false; + private boolean noLongitudeBound = false; /** Set to true if no top latitude bound can be stated */ - protected boolean noTopLatitudeBound = false; + private boolean noTopLatitudeBound = false; /** Set to true if no bottom latitude bound can be stated */ - protected boolean noBottomLatitudeBound = false; + private boolean noBottomLatitudeBound = false; /** Construct an empty bounds object */ public XYZBounds() { From dd9d5d06d2d5aef1f9367ff6459b325a44d0de4c Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 7 Apr 2016 18:48:47 -0400 Subject: [PATCH 0306/1113] LUCENE-7168: use center value on decode --- .../apache/lucene/spatial3d/Geo3DUtil.java | 44 +++++-------------- .../lucene/spatial3d/TestGeo3DPoint.java | 21 +-------- 2 files changed, 13 insertions(+), 52 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java index 1ff56a77a6f6..f91a6f63a619 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DUtil.java @@ -23,7 +23,7 @@ class Geo3DUtil { private static final double MAX_VALUE = PlanetModel.WGS84.getMaximumMagnitude(); private static final int BITS = 32; private static final double MUL = (0x1L< " + result); assert result >= Integer.MIN_VALUE; @@ -41,35 +45,8 @@ public static int encodeValue(double x) { } public static double decodeValue(int x) { - double result; - if (x == MIN_ENCODED_VALUE) { - // We must special case this, because -MAX_VALUE is not guaranteed to land precisely at a floor value, and we don't ever want to - // return a value outside of the planet's range (I think?). The max value is "safe" because we floor during encode: - result = -MAX_VALUE; - } else { - result = x * DECODE; - } - assert result >= -MAX_VALUE && result <= MAX_VALUE; - return result; - } - - /** Returns a double value >= x such that if you multiply that value by an int, and then - * divide it by that int again, you get precisely the same value back */ - private static double getNextSafeDouble(double x) { - - // Move to double space: - long bits = Double.doubleToLongBits(x); - - // Make sure we are beyond the actual maximum value: - bits += Integer.MAX_VALUE; - - // Clear the bottom 32 bits: - bits &= ~((long) Integer.MAX_VALUE); - - // Convert back to double: - double result = Double.longBitsToDouble(bits); - assert result > x; - return result; + // We decode to the center value; this keeps the encoding stable + return (x+0.5) * DECODE; } /** Returns smallest double that would encode to int x. */ @@ -81,7 +58,10 @@ static double decodeValueFloor(int x) { /** Returns largest double that would encode to int x. */ // NOTE: keep this package private!! static double decodeValueCeil(int x) { - assert x < Integer.MAX_VALUE; - return Math.nextDown((x+1) * DECODE); + if (x == Integer.MAX_VALUE) { + return MAX_VALUE; + } else { + return Math.nextDown((x+1) * DECODE); + } } } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 5aaa8357a44b..14f749b76bd5 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -728,7 +728,7 @@ public void collect(int doc) { public void testToString() { Geo3DPoint point = new Geo3DPoint("point", 44.244272, 7.769736); - assertEquals("Geo3DPoint ", point.toString()); + assertEquals("Geo3DPoint ", point.toString()); } public void testShapeQueryToString() { @@ -1078,25 +1078,6 @@ public void testEncodeDecodeIsStable() throws Exception { } } - // Takes ~35 seconds on modern-ish 2015 dev box: - @Nightly - public void testEncodeIsStableFromIntSide() throws Exception { - double max = PlanetModel.WGS84.getMaximumMagnitude(); - - // We can't test the full space of ints (Integer.MIN_VALUE to Integer.MAX_VALUE) because not all ints are allowed: - int start = Geo3DUtil.encodeValue(-max); - int end = Geo3DUtil.encodeValue(max); - // This prints: 99.99997175764292 - //System.out.println("PCTG INT SPACE USED: " + 100.*(((long) end)-(long) start)/(1L<<32)); - for (int i=start;i<=end;i++) { - double x = Geo3DUtil.decodeValue(i); - assertEquals(i, Geo3DUtil.encodeValue(x)); - if (i > start+1) { - assertEquals(Geo3DUtil.DECODE, x - Geo3DUtil.decodeValue(i-1), 0.0d); - } - } - } - /** Clips the incoming value to the allowed min/max range before encoding, instead of throwing an exception. */ private static int encodeValueLenient(double x) { double planetMax = PlanetModel.WGS84.getMaximumMagnitude(); From 646081c3dd5e605d4e3e9cf0d1dedd636016da5d Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 8 Apr 2016 11:02:48 +0200 Subject: [PATCH 0307/1113] LUCENE-7193 - add generic f1-measure metric to confusion matrix (cherry picked from commit 2507015) --- .../utils/ConfusionMatrixGenerator.java | 39 ++++++++++++++----- .../utils/ConfusionMatrixGeneratorTest.java | 30 +++++++++++--- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java index c9ecc4b3e822..3dd8ba83d048 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java @@ -52,16 +52,17 @@ private ConfusionMatrixGenerator() { * get the {@link org.apache.lucene.classification.utils.ConfusionMatrixGenerator.ConfusionMatrix} of a given {@link Classifier}, * generated on the given {@link LeafReader}, class and text fields. * - * @param reader the {@link LeafReader} containing the index used for creating the {@link Classifier} - * @param classifier the {@link Classifier} whose confusion matrix has to be generated - * @param classFieldName the name of the Lucene field used as the classifier's output - * @param textFieldName the nome the Lucene field used as the classifier's input - * @param the return type of the {@link ClassificationResult} returned by the given {@link Classifier} + * @param reader the {@link LeafReader} containing the index used for creating the {@link Classifier} + * @param classifier the {@link Classifier} whose confusion matrix has to be generated + * @param classFieldName the name of the Lucene field used as the classifier's output + * @param textFieldName the nome the Lucene field used as the classifier's input + * @param timeoutMilliseconds timeout to wait before stopping creating the confusion matrix + * @param the return type of the {@link ClassificationResult} returned by the given {@link Classifier} * @return a {@link org.apache.lucene.classification.utils.ConfusionMatrixGenerator.ConfusionMatrix} * @throws IOException if problems occurr while reading the index or using the classifier */ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifier classifier, String classFieldName, - String textFieldName) throws IOException { + String textFieldName, long timeoutMilliseconds) throws IOException { ExecutorService executorService = Executors.newFixedThreadPool(1, new NamedThreadFactory("confusion-matrix-gen-")); @@ -72,7 +73,13 @@ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifi TopDocs topDocs = indexSearcher.search(new TermRangeQuery(classFieldName, null, null, true, true), Integer.MAX_VALUE); double time = 0d; + int counter = 0; for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + + if (timeoutMilliseconds > 0 && time >= timeoutMilliseconds) { + break; + } + Document doc = reader.document(scoreDoc.doc); String[] correctAnswers = doc.getValues(classFieldName); @@ -91,6 +98,7 @@ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifi if (result != null) { T assignedClass = result.getAssignedClass(); if (assignedClass != null) { + counter++; String classified = assignedClass instanceof BytesRef ? ((BytesRef) assignedClass).utf8ToString() : assignedClass.toString(); String correctAnswer; @@ -117,7 +125,7 @@ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifi } } } catch (TimeoutException timeoutException) { - // add timeout + // add classification timeout time += 5000; } catch (ExecutionException | InterruptedException executionException) { throw new RuntimeException(executionException); @@ -126,7 +134,7 @@ public static ConfusionMatrix getConfusionMatrix(LeafReader reader, Classifi } } } - return new ConfusionMatrix(counts, time / topDocs.totalHits, topDocs.totalHits); + return new ConfusionMatrix(counts, time / counter, counter); } finally { executorService.shutdown(); } @@ -167,7 +175,7 @@ public Map> getLinearizedMatrix() { public double getPrecision(String klass) { Map classifications = linearizedMatrix.get(klass); double tp = 0; - double fp = -1; + double fp = 0; if (classifications != null) { for (Map.Entry entry : classifications.entrySet()) { if (klass.equals(entry.getKey())) { @@ -180,7 +188,7 @@ public double getPrecision(String klass) { } } } - return tp + fp > 0 ? tp / (tp + fp) : 0; + return tp > 0 ? tp / (tp + fp) : 0; } /** @@ -217,6 +225,17 @@ public double getF1Measure(String klass) { return precision > 0 && recall > 0 ? 2 * precision * recall / (precision + recall) : 0; } + /** + * get the F-1 measure on this confusion matrix + * + * @return the F-1 measure + */ + public double getF1Measure() { + double recall = getRecall(); + double precision = getPrecision(); + return precision > 0 && recall > 0 ? 2 * precision * recall / (precision + recall) : 0; + } + /** * Calculate accuracy on this confusion matrix using the formula: * {@literal accuracy = correctly-classified / (correctly-classified + wrongly-classified)} diff --git a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java index d1966ec7c815..63cce2a1c8e5 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/utils/ConfusionMatrixGeneratorTest.java @@ -59,7 +59,8 @@ public List> getClasses(String text, int max) thr return null; } }; - ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, classifier, categoryFieldName, textFieldName); + ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, + classifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); @@ -74,6 +75,9 @@ public List> getClasses(String text, int max) thr double recall = confusionMatrix.getRecall(); assertTrue(recall >= 0d); assertTrue(recall <= 1d); + double f1Measure = confusionMatrix.getF1Measure(); + assertTrue(f1Measure >= 0d); + assertTrue(f1Measure <= 1d); } finally { if (reader != null) { reader.close(); @@ -88,7 +92,8 @@ public void testGetConfusionMatrixWithSNB() throws Exception { MockAnalyzer analyzer = new MockAnalyzer(random()); reader = getSampleIndex(analyzer); Classifier classifier = new SimpleNaiveBayesClassifier(reader, analyzer, null, categoryFieldName, textFieldName); - ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, classifier, categoryFieldName, textFieldName); + ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, + classifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); @@ -102,6 +107,9 @@ public void testGetConfusionMatrixWithSNB() throws Exception { double recall = confusionMatrix.getRecall(); assertTrue(recall >= 0d); assertTrue(recall <= 1d); + double f1Measure = confusionMatrix.getF1Measure(); + assertTrue(f1Measure >= 0d); + assertTrue(f1Measure <= 1d); } finally { if (reader != null) { reader.close(); @@ -116,7 +124,8 @@ public void testGetConfusionMatrixWithCNB() throws Exception { MockAnalyzer analyzer = new MockAnalyzer(random()); reader = getSampleIndex(analyzer); Classifier classifier = new CachingNaiveBayesClassifier(reader, analyzer, null, categoryFieldName, textFieldName); - ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, classifier, categoryFieldName, textFieldName); + ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, + classifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); @@ -130,6 +139,9 @@ public void testGetConfusionMatrixWithCNB() throws Exception { double recall = confusionMatrix.getRecall(); assertTrue(recall >= 0d); assertTrue(recall <= 1d); + double f1Measure = confusionMatrix.getF1Measure(); + assertTrue(f1Measure >= 0d); + assertTrue(f1Measure <= 1d); } finally { if (reader != null) { reader.close(); @@ -144,7 +156,8 @@ public void testGetConfusionMatrixWithKNN() throws Exception { MockAnalyzer analyzer = new MockAnalyzer(random()); reader = getSampleIndex(analyzer); Classifier classifier = new KNearestNeighborClassifier(reader, null, analyzer, null, 1, 0, 0, categoryFieldName, textFieldName); - ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, classifier, categoryFieldName, textFieldName); + ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, + classifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); @@ -158,6 +171,9 @@ public void testGetConfusionMatrixWithKNN() throws Exception { double recall = confusionMatrix.getRecall(); assertTrue(recall >= 0d); assertTrue(recall <= 1d); + double f1Measure = confusionMatrix.getF1Measure(); + assertTrue(f1Measure >= 0d); + assertTrue(f1Measure <= 1d); } finally { if (reader != null) { reader.close(); @@ -172,7 +188,8 @@ public void testGetConfusionMatrixWithBP() throws Exception { MockAnalyzer analyzer = new MockAnalyzer(random()); reader = getSampleIndex(analyzer); Classifier classifier = new BooleanPerceptronClassifier(reader, analyzer, null, 1, null, booleanFieldName, textFieldName); - ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, classifier, booleanFieldName, textFieldName); + ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader, + classifier, booleanFieldName, textFieldName, -1); assertNotNull(confusionMatrix); assertNotNull(confusionMatrix.getLinearizedMatrix()); assertEquals(7, confusionMatrix.getNumberOfEvaluatedDocs()); @@ -186,6 +203,9 @@ public void testGetConfusionMatrixWithBP() throws Exception { double recall = confusionMatrix.getRecall(); assertTrue(recall >= 0d); assertTrue(recall <= 1d); + double f1Measure = confusionMatrix.getF1Measure(); + assertTrue(f1Measure >= 0d); + assertTrue(f1Measure <= 1d); assertTrue(confusionMatrix.getPrecision("true") >= 0d); assertTrue(confusionMatrix.getPrecision("true") <= 1d); assertTrue(confusionMatrix.getPrecision("false") >= 0d); From 585dddca7eaf31038632801f668f7c1702118295 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 8 Apr 2016 11:06:13 +0200 Subject: [PATCH 0308/1113] LUCENE-7193 - added missing timeout param to tests (cherry picked from commit c906b4f) --- .../lucene/classification/BooleanPerceptronClassifierTest.java | 2 +- .../lucene/classification/CachingNaiveBayesClassifierTest.java | 2 +- .../lucene/classification/KNearestNeighborClassifierTest.java | 2 +- .../lucene/classification/SimpleNaiveBayesClassifierTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java b/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java index 4b83dce68cbc..6ea92c03b088 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/BooleanPerceptronClassifierTest.java @@ -86,7 +86,7 @@ public void testPerformance() throws Exception { long evaluationStart = System.currentTimeMillis(); ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader, - classifier, categoryFieldName, textFieldName); + classifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); long evaluationEnd = System.currentTimeMillis(); long evaluationTime = evaluationEnd - evaluationStart; diff --git a/lucene/classification/src/test/org/apache/lucene/classification/CachingNaiveBayesClassifierTest.java b/lucene/classification/src/test/org/apache/lucene/classification/CachingNaiveBayesClassifierTest.java index 535c96f222e9..acbfe82be9a8 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/CachingNaiveBayesClassifierTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/CachingNaiveBayesClassifierTest.java @@ -104,7 +104,7 @@ public void testPerformance() throws Exception { long evaluationStart = System.currentTimeMillis(); ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader, - simpleNaiveBayesClassifier, categoryFieldName, textFieldName); + simpleNaiveBayesClassifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); long evaluationEnd = System.currentTimeMillis(); long evaluationTime = evaluationEnd - evaluationStart; diff --git a/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java b/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java index c03678481c7c..5e3ce7bfff33 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java @@ -134,7 +134,7 @@ public void testPerformance() throws Exception { long evaluationStart = System.currentTimeMillis(); ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader, - kNearestNeighborClassifier, categoryFieldName, textFieldName); + kNearestNeighborClassifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); long evaluationEnd = System.currentTimeMillis(); long evaluationTime = evaluationEnd - evaluationStart; diff --git a/lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java b/lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java index aa7de0cad623..ab1204f8b37b 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java @@ -106,7 +106,7 @@ public void testPerformance() throws Exception { long evaluationStart = System.currentTimeMillis(); ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader, - simpleNaiveBayesClassifier, categoryFieldName, textFieldName); + simpleNaiveBayesClassifier, categoryFieldName, textFieldName, -1); assertNotNull(confusionMatrix); long evaluationEnd = System.currentTimeMillis(); long evaluationTime = evaluationEnd - evaluationStart; From 7b9a4c69e5f2a1b0a01802d38132fcc50b287b2f Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Fri, 8 Apr 2016 01:33:46 -0400 Subject: [PATCH 0309/1113] LUCENE-7191: Tests now exercise the query factory methods in Geo3DPoint. --- .../lucene/spatial3d/TestGeo3DPoint.java | 81 +++++++++++++++++-- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 14f749b76bd5..d1aa416c79d9 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -37,6 +37,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.geo.GeoTestUtil; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -54,8 +55,10 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.spatial3d.geom.XYZSolid; import org.apache.lucene.spatial3d.geom.GeoArea; import org.apache.lucene.spatial3d.geom.GeoAreaFactory; +import org.apache.lucene.spatial3d.geom.GeoBBox; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; import org.apache.lucene.spatial3d.geom.GeoCircleFactory; import org.apache.lucene.spatial3d.geom.GeoPathFactory; @@ -514,6 +517,68 @@ private void doTestRandom(int count) throws Exception { verify(lats, lons); } + private static final double MEAN_EARTH_RADIUS_METERS = PlanetModel.WGS84_MEAN; + + private static Query random3DQuery(final String field) { + while (true) { + final int shapeType = random().nextInt(4); + switch (shapeType) { + case 0: { + // Polygons + final boolean isClockwise = random().nextDouble() < 0.5; + try { + return Geo3DPoint.newPolygonQuery(field, makePoly(PlanetModel.WGS84, + new GeoPoint(PlanetModel.WGS84, toRadians(GeoTestUtil.nextLatitude()), toRadians(GeoTestUtil.nextLongitude())), + isClockwise, + true)); + } catch (IllegalArgumentException e) { + continue; + } + } + + case 1: { + // Circles + final double widthMeters = random().nextDouble() * Math.PI * MEAN_EARTH_RADIUS_METERS; + try { + return Geo3DPoint.newDistanceQuery(field, GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), widthMeters); + } catch (IllegalArgumentException e) { + continue; + } + } + + case 2: { + // Rectangles + final Rectangle r = GeoTestUtil.nextBox(); + return Geo3DPoint.newBoxQuery(field, r.minLat, r.maxLat, r.minLon, r.maxLon); + } + + case 3: { + // Paths + // TBD: Need to rework generation to be realistic + final int pointCount = random().nextInt(5) + 1; + final double width = random().nextDouble() * Math.PI * 0.5 * MEAN_EARTH_RADIUS_METERS; + final double[] latitudes = new double[pointCount]; + final double[] longitudes = new double[pointCount]; + for (int i = 0; i < pointCount; i++) { + latitudes[i] = GeoTestUtil.nextLatitude(); + longitudes[i] = GeoTestUtil.nextLongitude(); + } + try { + return Geo3DPoint.newPathQuery(field, latitudes, longitudes, width); + } catch (IllegalArgumentException e) { + // This is what happens when we create a shape that is invalid. Although it is conceivable that there are cases where + // the exception is thrown incorrectly, we aren't going to be able to do that in this random test. + continue; + } + } + + default: + throw new IllegalStateException("Unexpected shape type"); + } + } + + } + // Poached from Geo3dRptTest.randomShape: private static GeoShape randomShape() { while (true) { @@ -660,13 +725,15 @@ private static void verify(double[] lats, double[] lons) throws Exception { for (int iter=0;iter Date: Fri, 8 Apr 2016 09:14:47 -0400 Subject: [PATCH 0310/1113] LUCENE-7192: Permit adjacent points in a polygon to be coplanar. --- .../spatial3d/geom/GeoConcavePolygon.java | 21 ++++++--- .../spatial3d/geom/GeoConvexPolygon.java | 21 ++++++--- .../spatial3d/geom/GeoPolygonFactory.java | 44 +++++++++++++++++-- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 375ec6088e44..05e58ee5734e 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -46,8 +46,6 @@ class GeoConcavePolygon extends GeoBasePolygon { protected GeoPoint[][] notableEdgePoints = null; /** A point which is on the boundary of the polygon */ protected GeoPoint[] edgePoints = null; - /** Tracking the maximum distance we go at any one time, so to be sure it's legal */ - protected double fullDistance = 0.0; /** Set to true when the polygon is complete */ protected boolean isDone = false; /** A bounds object for each sided plane */ @@ -189,10 +187,21 @@ public void done(final boolean isInternalReturnEdge) { for (int i = 0; i < points.size(); i++) { final GeoPoint start = points.get(i); final GeoPoint end = points.get(legalIndex(i + 1)); - final double distance = start.arcDistance(end); - if (distance > fullDistance) - fullDistance = distance; - final GeoPoint check = points.get(legalIndex(i + 2)); + // We have to find the next point that is not on the plane between start and end. + // If there is no such point, it's an error. + final Plane planeToFind = new Plane(start, end); + int endPointIndex = -1; + for (int j = 0; j < points.size(); j++) { + final int index = legalIndex(j + i + 2); + if (!planeToFind.evaluateIsZero(points.get(index))) { + endPointIndex = index; + break; + } + } + if (endPointIndex == -1) { + throw new IllegalArgumentException("Polygon points are all coplanar"); + } + final GeoPoint check = points.get(endPointIndex); // Here note the flip of the sense of the sided plane!! final SidedPlane sp = new SidedPlane(check, false, start, end); //System.out.println("Created edge "+sp+" using start="+start+" end="+end+" check="+check); diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index 0a48eeb3a37c..d1e009178f38 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -44,8 +44,6 @@ class GeoConvexPolygon extends GeoBasePolygon { protected GeoPoint[][] notableEdgePoints = null; /** A point which is on the boundary of the polygon */ protected GeoPoint[] edgePoints = null; - /** Tracking the maximum distance we go at any one time, so to be sure it's legal */ - protected double fullDistance = 0.0; /** Set to true when the polygon is complete */ protected boolean isDone = false; /** A bounds object for each sided plane */ @@ -185,10 +183,21 @@ public void done(final boolean isInternalReturnEdge) { for (int i = 0; i < points.size(); i++) { final GeoPoint start = points.get(i); final GeoPoint end = points.get(legalIndex(i + 1)); - final double distance = start.arcDistance(end); - if (distance > fullDistance) - fullDistance = distance; - final GeoPoint check = points.get(legalIndex(i + 2)); + // We have to find the next point that is not on the plane between start and end. + // If there is no such point, it's an error. + final Plane planeToFind = new Plane(start, end); + int endPointIndex = -1; + for (int j = 0; j < points.size(); j++) { + final int index = legalIndex(j + i + 2); + if (!planeToFind.evaluateIsZero(points.get(index))) { + endPointIndex = index; + break; + } + } + if (endPointIndex == -1) { + throw new IllegalArgumentException("Polygon points are all coplanar"); + } + final GeoPoint check = points.get(endPointIndex); final SidedPlane sp = new SidedPlane(check, start, end); //System.out.println("Created edge "+sp+" using start="+start+" end="+end+" check="+check); edges[i] = sp; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 144c4a530095..67804a2accf1 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -714,12 +714,17 @@ private static Boolean findConvexPolygon(final PlanetModel planetModel, // the start point of the first edge and the end point of the last edge. If the first edge start point is the same as the last edge end point, // it's a degenerate case and we want to just clean out the edge buffer entirely. - final List points = new ArrayList(includedEdges.size()); - final BitSet internalEdges = new BitSet(includedEdges.size()-1); + final List points = new ArrayList(includedEdges.size()+1); + final BitSet internalEdges = new BitSet(includedEdges.size()); final boolean returnIsInternal; if (firstEdge.startPoint == lastEdge.endPoint) { // Degenerate case!! There is no return edge -- or rather, we already have it. + if (includedEdges.size() < 3) { + // This means we found a degenerate cycle of edges. If we emit a polygon at this point it + // has no contents, so we've clearly done something wrong, but not sure what. + throw new IllegalArgumentException("polygon was illegal (degenerate illegal two-edge cyclical polygon encountered in processing)"); + } Edge edge = firstEdge; points.add(edge.startPoint); int i = 0; @@ -945,8 +950,39 @@ public EdgeBuffer(final List pointList, final BitSet internalEdges, fi // Get the next point final GeoPoint newPoint = pointList.get(endIndex); // Build the new edge - final boolean isNewPointWithin = currentEdge.plane.isWithin(newPoint); - final SidedPlane newPlane = new SidedPlane(currentEdge.startPoint, isNewPointWithin, pointList.get(startIndex), newPoint); + // We have to be sure that the point we use as a check does not lie on the plane. + // In order to meet that goal, we need to go hunting for a point that meets the criteria. If we don't + // find one, we've got a linear "polygon" that we cannot use. + + // We need to know the sidedness of the new plane. The point we're going to be presenting to it has + // a certain relationship with the sided plane we already have for the current edge. If the current edge + // is colinear with the new edge, then we want to maintain the same relationship. If the new edge + // is not colinear, then we can use the new point's relationship with the current edge as our guide. + + final boolean isNewPointWithin; + final GeoPoint pointToPresent; + if (currentEdge.plane.evaluateIsZero(newPoint)) { + // The new point is colinear with the current edge. We'll have to look for the first point that isn't. + int checkPointIndex = -1; + final Plane checkPlane = new Plane(pointList.get(startIndex), newPoint); + for (int i = 0; i < pointList.size(); i++) { + final int index = getLegalIndex(startIndex - 1 - i, pointList.size()); + if (!checkPlane.evaluateIsZero(pointList.get(index))) { + checkPointIndex = index; + break; + } + } + if (checkPointIndex == -1) { + throw new IllegalArgumentException("polygon is illegal (linear)"); + } + pointToPresent = pointList.get(checkPointIndex); + isNewPointWithin = currentEdge.plane.isWithin(pointToPresent); + } else { + isNewPointWithin = currentEdge.plane.isWithin(newPoint); + pointToPresent = currentEdge.startPoint; + } + + final SidedPlane newPlane = new SidedPlane(pointToPresent, isNewPointWithin, pointList.get(startIndex), newPoint); /* System.out.println("For next plane, the following points are in/out:"); for (final GeoPoint p: pointList) { From ebb2127cca54e49ed5c7462d11ee8dda6125e287 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 8 Apr 2016 12:26:35 -0400 Subject: [PATCH 0311/1113] LUCENE-7174: IntelliJ config: switch JUnit library to include all jars under lucene/test-framework/lib/, rather than using the exact jar names, which is brittle, and causes trouble when people forget to update when jars are upgraded (like on this issue) --- dev-tools/idea/.idea/libraries/JUnit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/idea/.idea/libraries/JUnit.xml b/dev-tools/idea/.idea/libraries/JUnit.xml index f6bdd9b6e334..4a6da924ae5f 100644 --- a/dev-tools/idea/.idea/libraries/JUnit.xml +++ b/dev-tools/idea/.idea/libraries/JUnit.xml @@ -1,10 +1,10 @@ - - + + \ No newline at end of file From ddc02603a7fc62695ff0985b56c82e45c1fc5049 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 8 Apr 2016 12:27:48 -0400 Subject: [PATCH 0312/1113] LUCENE-7174: IntelliJ config: remove trailing slashes on dir names to make IntelliJ happy --- dev-tools/idea/.idea/libraries/JUnit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/idea/.idea/libraries/JUnit.xml b/dev-tools/idea/.idea/libraries/JUnit.xml index 4a6da924ae5f..bb159b0b046e 100644 --- a/dev-tools/idea/.idea/libraries/JUnit.xml +++ b/dev-tools/idea/.idea/libraries/JUnit.xml @@ -1,10 +1,10 @@ - + - + \ No newline at end of file From 707960fb220d785705bea36c5bb213b3195f807b Mon Sep 17 00:00:00 2001 From: nknize Date: Fri, 8 Apr 2016 14:19:38 -0500 Subject: [PATCH 0313/1113] adds 6.0.0 back compat test indexes --- .../index/TestBackwardsCompatibility.java | 4 +++- .../org/apache/lucene/index/index.6.0.0-cfs.zip | Bin 0 -> 13744 bytes .../apache/lucene/index/index.6.0.0-nocfs.zip | Bin 0 -> 13749 bytes 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-cfs.zip create mode 100644 lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-nocfs.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 6121bca6e5c9..8e04143a4f03 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -235,7 +235,9 @@ public void testCreateIndexWithDocValuesUpdates() throws Exception { "5.4.1-cfs", "5.4.1-nocfs", "5.5.0-cfs", - "5.5.0-nocfs" + "5.5.0-nocfs", + "6.0.0-cfs", + "6.0.0-nocfs" }; final String[] unsupportedNames = { diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-cfs.zip new file mode 100644 index 0000000000000000000000000000000000000000..29939705f00cda890e63c90e4b768cdb8e653270 GIT binary patch literal 13744 zcmdU$Wl)^?wy$vy!8Jewjk~+MdypW(-Q6{~ySrW_z`3p@xY#2z>Z$j_(RwoGE$6Q$Jj)Fd;rwe+la(K@L~8>y;_@hKWZ zAP{D=OQfC@TrCe*d`h-q{tFk_H#eKQgPX4|70=_a0sOB88J>h<=mv&91T*dPn_ys| z-TFP~1+%AtDnBb=w6@fs!2~^0P1%@ZwARJL!K8tg#u7pd7lIXgz)t}`7NWoe+4Ct+ zI%BlPG8Ri{E@t~Z0R)OOmdfRW?AjyPKwaw)4v@XHVTQ|t4`R9{gBy^T0aza6mqNps zrGcE*Wc_*aLL$=HsZBmsBmS@_{!N}2pb-Of*>G5z5k$@)Ao9O;r(K`Y-~t*1#25dU z?%4lTcc#@mE#8QsXR+P4?p}D_yvf>RZv@J0GgCB?J_W;*M%vfgF;~2q$5IiGA70jO zIk=^5U9`6ZohS*AkET@23mqF{4K74dElu^7q{?T`M-fyY5hsQORh(`zTc@Kh7Tk!^ zGrBsQ`Ft3L>KK#Un!$Fl)4H0L!ggVFh4_>+m%y&^wS>eG;Mu_KUZY91%@kS*h&}nj z@#SVw!W*gy@xg60Y!47~@~MDqP(_O>qu8q;ci)UeY2>z~|I=ljY5GE>bIlvN z;Mg~2@SnwUt6ju$?Uht(-0epbS1J~LswYLIf;%#IpK%SlX4+*x>ns#UPZuG~nHs6*VAcK*=MAOGLKEnL353fN8XoP?h53%6qF^ zt};JwCt{d2cA3J;p$pzm?g2FzQc;s&CB>6K~2>#Gj0y&zDm077<Dk?W!-=v7_3H(2~S#U_ZQ6N#GWz7sLdMD0bpe~`nme%-9! zp44`Kn}9(EQnIW^buSF)-G}UyH9KYZ!QkVI4~&4qKLlF&fFcW5KFit_ioIuLjy-P`^cH`WLc4sMlIj7JL(PDeSHf0${bncywR+3 z#?=W~x?t|^ph%`m%_lpn?WkFDFt4bcQPg$a9<3t6I{COILg0&gQAvkSke`faK8lW8 zGTk9KoqRSx(IkzO;%g{v$!yz5qI2TzrMaF;pz1aoh3hsSwRlxc@cFTnfX!<@D$QeL z4Us4Q-3_Rwk4}f<4ZMfhvC5u}-0~_Neq6~_;`b$8j_Jb)H#~aD=wK)Cx2QzDirqgIVEs8TCiMb49BH$>cmyWo)gA$VTVrNcd@K!2{C2%WM zJlx4)C~>qm5;t4sPK4JpH5t$hYMa7_#tNXlc=a%|k+=oby&v4t z%GT}rRI=<$V^p!K3M8tK1dp+05^SZK3+7>7n9JD(MRNs%jTzq3;%|*%-XI#yR*Cgu)y=MWMYT$) z`5bs}Hz~(!Qk0-4Q^A{mAnBEHG0q!#M<64Hq+K5i$wB><^{xXwE>gqsVnM=WDi=14 zl%}aCmm=(hzQl8uU3e?49M3L%UU1&YCO~b*YQ#!UH%{!{HtI@nPR>22EALJz>Y%6C zn3k4xr!zUqZPG*SgF>43O0FvP-4_;d_mAU&^-{KSNwls_mxV`cscefjX#~&ypNzz# zQwa%Y2KdQNF2~7a@E>)fo(RW@x6TL+NcbKshVl+<0n`CD{c^5^TMAbVNVF3(!S4P` z51aF<@0OnuAStIezU)A`8_y1+!_Dl^TOKY8YjM4D;Od2?tq=q+K5?rGXc&~nQ~3gb5qo@w@=+3nL< z*@0SoVsNRV2=Dx%OoYOSFzZjsr_Tr?%U4iTEBS`zy7Gksm)sMaoJal%K5Z-L_O4^-d#T46-%1 zWIlETdR->QkANmg7DgcZ;3Vd{XSDZNn5?t1aJA(A={n)^e9C>5k zl!w@RCO!_KD`6t68zf&gr=|STxO;Y7>0(A1Zp}0mDvd}FRa*h}PEI-KCZAU)b@kap>`nj57=HTH(H}mAl%ATI zma4jul9V{w-rip9G;BGMRMYj<`k4v+GkY-)0eNqGuMdc3xTf@NPdRUjYXSO_dWd0& zn_-D7#jS|olSs$_M+w}>tqA0;NC-*Ftt+?uey$!Ms7roSa1^kz6hJ_f3Lm$|%s9=& zHZH?Rzws563zoFQtk5{yxII&+JV`A+y(2X_P0xr3Y(`T#RW&vRUbP>b3C95DovoLt zf*qx~hODVA6sA0ra*R(%W=y7UWp#)Z=U8O!kb!J8at}#JBVhwSMc|CQBO z*zG=zPd>d{mKQ?(hXko| zTf1EGI+%?sYP46flrTrmopbITMH?0otrHBTH`-qDi(-rA;vGyNF$?7+N-hPC4TS<^ zNRT2g35pA|vd4T=ffwkMdr>{^}gDghcA`3+3 zHB06*I`WJ@x*WxHqrarYY$;1AUQ=D3D2_P5h4LxAS0w&qLs3Tq8 z`JwW3W2jpo(vI=6~KRRcj~g@Q50EN*~|-`5f;?<@^sPV zJJPdx>*LG-jYGtM@@@$r3SN2K+)?itrn$&wWa*ZOpY=GAp3 zl}zrU$e?hVua~)_koab#NngQbCYM9;U|%5G~<_lg%2asro)cPWHff~td)0f^|?ON zVeYw?KLOyXMSja5ULWa?JgUmwMEy1f1_c$e!vRgyGTk}eUdHVe%E$8IIb!+ zIIQ}`t|qtq4IjlBBn+&lrR?qE)CecF#5wasRPK#D_!Z=NmM2XgvqKxr^kt8D$lXPweV)2-;yN zGY0U7d7-pPvwEK=;rlx=uyFNh%17qw@Y51&7$%Z~Cu0w?K0xY8EY%rU8~Hg%<}g-F za}Upvk39hohbd)>tBRFhiJEI=v>(TOU!H_Qs`y}6VOD>4S=R)@1X5xcyhcWs6nA)c z$qV60pF$Fg%?AvtD}q+j$=@7rvpW&1H5_@p@29PjASkl?cv>hPz?Z9n2K^kcKv{ zR*8~D_A5+qzn=MFMf>I0r~+}qoXKKw-4t=zzOPJwc7-=6uhT}L@BjYbEv5dM)udpt zZ>Fqk|8e^05i=b8{-*q`n^S2i`PN-dqN>!5IHElCXIwTw>xzGGv~8H~q>rxxj4;*A zf!V81SZ@eIQ0NKH1{4OwrF}XGv0_0kh?G0?wz}W|YA7*;^)EIMP9mT#E~y3vMAG)t z(5n@mw5%qr?96H(?_L56S}6?`%sZ&0S^ zAA=x74Uic#^b3Z~B@IpiSov951qK8h>Dmd@fe18|V5Oy;PrU+`TtmPW=CDiwq46Eo z2%+(v*0d10H`?x^u|`3Nx<1{j@u3KpkvQ9H1u5iD>#W|30K`Z_BjyOe zF2hKmF!%Pt5Qq{uewRwj3ULmgZ$j<^8n5CW8>=}`Y=0aVmtJYa-&5uF?siR}H~Hpm zzAUV!+G!&Yi(D`UwBFPf%|8BaizIdQV5H-s0(V#*K!GeYleI>|duz zw^Fr>{Nc?5b`W>9oC}E2*)C#u<6LKo8%Iv0o&cvc9EOuv9}17F*3i-ONP>B- zHy7q!z1VY?8ur~`Y8tbMByUD76-R23BQfuz`1&Loe&I)fbq2U3a_*I*Fnil1iCs-n zt>_jFd%^%g?MuFZ1(8l46{GzR)j;q#^8JpeFwF;I+9!r_m4R&`H5iuxx^B&KTIkLm z0Gt~G9hBBkRbx1pjO4qMa_>gZbEp)6a|O%(%mZfSczcIBSQc(&`*eQr^(J{{!^or{ zPW~Bh+NTke5*ombEwkrq4w}Mgjns6E$xE0N(s%BuNB7waZR2ytO7GDZg1VVGm@>+AEKDk$&N&q68Q<2dIIA}5eX;7m7pv|;qD(*+M>l31?D8)Iwvn^+ z&@i!(70l7Jk+W3`)3Nc9BzzrUVF&WEw@*)R|M1$6V~P&XmgEAX0i{L>;te=bE6N7OnDGFb^Od%5QACjxWKPUbAWut3Cx-A%x@;#@HNOJ{* zTqjxw0ofSd2geJ|0Ch?pQK2}8kUaHeo1Al=A}{kSza_<(lW12UWP$<8dx%iS*Z%0uUwbWC4 z_F3_$Nnsp%FjEs5iXsUMv{%|A8zZvEt((;KvG5drlIZJ{%18y05Wc_%iF`r-JAY;s zs(48i0pzj5(7hbe;3tz}`TQr7Nw((NL!NJ$_8~P1oXt$%j?=CgvaDB*9tno~@|rQ> zCDpMdzm-!TRZ%B~sjDX-=P&Z6YuwuHvstG9pC;CIuO)v2WWD3am1eN+nB`rD%LIlyel{Ce)<5<>rqR z7^v?kIA$9s;>cx}HE2?Jt5l*(7AUK0%ge536q{e{gQ7ihCO7ZC;cVp)3OlaD%I`s< z2{dI83{%hu&k!m)XPl0xuHPgGao?=lgw5Mzl62IKQuH~QwcF0EIhr%|^9_X}%%jpS zYq}Xv!p}c&v%p}Q(hdf!^7wn)eTG^5|sw~s|F)2S2uil^mLzleQRY&upB1AI7-8N= zI5U@rkoo=;&A$lSBdUXs@rJ=ai~dI4o?2GaLWa}Se1a=h$$DiUfG2y|=2TbO&6_v! zF|3OLu}^$R0Ke4YT%Yw8FE?UhS_|R*^8Kv^xvNOmsf=krgf991P zZ!M4YGGO1MpfGXp-fXovQMmw4C*HjERcxSAr*wKsbV_haDl_ONYgJrgVMHPCigYJZ zkO(!LKbxYRr*`G%?*hqUp8_5sq$*y+UI3@=PMyym@G9h#6l#NpSKbFrd2v_7xvXc7 zH9Mfh9cEz6o~K_cj=Gvg5I=P-e<@m^MmWuzRl@?-f3Vnwj8eUnc#7mVrbml(5Yb;g z6PB~q@wS`XF@k0bC5)WQe72^+{bW=bdf3Nm*2Q_RQ=A6Dc^@-yE4hA_Qkpwv-;hXy zuG|E-DBX-249+gb{QF)!@ljx;UZ!clFzkHYhBd|8Z07}9r*dS2CD8N8e!bsXo z*7Hd0&GIUQtql}RKe%m9u!pC$-l(&%Tj?I4(BBHL+cA1fAPUjVajeMS3lI2#xfDa*_dK4@Z%zG;q7LhT){ekRCmi#o3{@ z+RDaauueGx+|xsiIqC+)8S7f(h*B=!x9E1m{BRdq&Fb^F221-Q=Fv*dmlMu;t8f>1 zdI|=bO4|v2WtP(r2rsPj^3FRPZ>5iQwcKgoc@3RP>&nqQi*GScNO9maULD7j#c~-D z4GlRIlui(!jS=!SKw(mUm{Ie?cWlrnGn-Q=vol>yFa1ujcD~2(Zmdk!nL4DWz{2pX z+LdH)r0EJHsc?s?2FFgURbCU!8pTPBgGe{N z;#sGqy4>D1uj9zF2^sFDhHPc24_3h^%gHcKnTS0M-&Ra{(%ZLo%s&TOj6UvnIPuwC z*ez4o)oQL)44!;pJ}duzf6sR#vex#yR>*0gqWO^We>Uly)A(Rx$=O}tbo!B!*MEK>_H$@0lWa~wjew8v3!CSKijqd zq&g%a^R7|D87cp!(tFY~JYNmsNIg$UFGk_?XIAAE1d%U5RI@LDDUTgNdH#5Qb|QK{ zK9ELi9L`(`c>!(x9>-iV-Or7SltE;etS*d=W(q%M)FNVEYhQOx04W`VCFKC#3i zQRp3BSyib^pByY}r;0NeTh>39AlFwYi{J)?=8{tePl%S}k|#p(5t)Q!=fYHPy+&pp z89LFvvP%>0$ym;aV!QCN-WeWRbe+80=67^arLD}5Pwc(AT+4BwP1uWO!H%|h#LUpI z75dh8Gps{1&8DSx&22(bVLmj&qrsU6=R3#%D=kEF~6bN(yFY^5VNF`cqM_U zENNfn>#_EkOxu$(6RUqXo#RScGlj$cp)sD(^PEFHadSVuFu%@=B(Hd0pyvK{nT9vS z)KE6C9*n|OUZRM#f4-sg1mvl!{a8oh3$pyCHN@OAR`Pv*nm`8;@feHStZUO6WmtH- zl}H?F7`C4vyWh|fy=p%?>pMhDlW9ne{OB~|4(sEP^9%Ru@l54Cp2q4J?9Y|T6D!F+ z=vG3+27P`U#M!A3b)w`&vudMxO~w+$E>WK;2w>uccVCa4tkg-J6&3h9!)Ne4H>TQ!%tt`>)vf0h`;iV1*%t@%V6!NE9`%?m+QTHmlT3Vw@{vueZ?LPj^HX(Z~Eb-9Gta#TyiHmcPi@?3xdtO+`K{9XSJ8NjG?3 z!kF2OkqG=8Wpd3YpFal3NR-87*ID?6hPVv+o`toLF{s|SpD2&gSu{2p>p*-L5rN#x zc&46q*tT*OX?|NZ`pZS`;_p{;*SybcWr@<3z z4}~!=XBusQRYodU2lfD`V(A7wP(Tj)RogU&?H-=-F4}A~rtQAHg``WjY3(N>xiq3D@wcSPO#PfE$sm+AuHkJU0eUEQTx= zC&h60`IE21L*`0&uKX(7VovV>EI~HORF9SD+j*)qUFy6O?n?`Qe)^26bG>h2#Jq7n zzE3dspL8+UUyFG#27nGOX@82?k#P2@}ZlUN@9Wslhpo~m$ z33l!6~!_l7Lapa;SYR=Y0Yb%YOVN$-3>dOI+) zRsD8qitD%X5?c%gX4TKX3+CnTYBD&jwmYM`Uew?Vzg2&B+2~=ojLShpJ~K&$4`SfB zm}CXXwU3Y7QyzIPW|?D_+Nr2v?}HMr z+!HmCjlTrDDn;}Nq?r?Xi7XfjGG$6=gs`tOD?m#usVLuwq=}nEe^}BX9YqDKpSk?K zP?Qgcnpy&iR=#zY6H;iVHOTs6a6?4gnuh~wM5@UYH|)wuB8BkTbnk3)+g)NLot}Nt zrzJ;IrdHa=$j_aRb{i44aF~>zG@| zDq_kc*7|v`pfUhHzkk!LKH94h!E99M%ClUVG>lvF){a)oXToenVLCO!DSnjeoZI^C zx8e<571-+)l#@q8r=@!#lJa?!nHc^7EHQfXdQVY0t7k2_#qa}G|9Y8Zf=v^Vkm7L^ zg+7sHZTxG(EWF3~Dv0y#&go3pTQ?GDDun{P=feb&2ZF>?Z_0S)ySn_BJ36iKDFZrs0=wC1GZ@P%X6VKzb%_MB+Me6izsunw7q0zpa>(>g$|20Z z44mSDmEX5o2COUzz`5Ih4X;{~?Ex zKjpBm=0y&-1v!@GIl+nk7{c(2|YO4&*Ghz zGvXcU&%Qnc^Gka|P7?t84}hIiY@dw7m-vfL__RD}{JViz66wL;In)?5I0%_ zVg@@}M>94hN&5)OLHi{+f|I=A^8tp|u&2P(GzC2z3W0u7Cf`Jtv!+kP4m^w>AJb`Y zKyCz)LQ>|H1Y}^bv9_ccqm zgM8qSxZ2GJOvA){jK(c8eS368A9@OMMvB^xHP!YucD5I%=V$ORaIj&)p&?S@k`iD0 z2M353uRe5`BC_}7K($C2*}?kSKtoqEJBj%X1J;J1M2%#{NAyDV`Slnj7#Vr=j0_-* zxsVd|3=Z@KHiWcRf{tE=C@QfSe5qMK(EpR?TfGi^`fZ9Z|C3V$;$KrF{6A)jVC)#Z z9Gj5-2qPhtCjW1bP2Ac?zW+HO-VbjR@ANW7d|rO(;9k;R_J+omhE@*tI!yml9BD-1 zUr{f`p_iOj;h%C|t>NRz5ui{vzRBnTzJk(cP~j$KyY@Hnq(wUgso4C}On#@{%pi%n zp=`pltss@S>LiHEqI*r4vAK&_TGvF14r$qk+JY@sfj`nZ?S65GPgYJpKN@f^*9!78 z`2X|8A5TB7+}Y05!I0L%THpMyu3TOU6bv8yzq@?^>;FLOe1E+B_ss;5|GJsrPd5_$ zne}_v{#O?MOLu>=UPjs9WBs1O`<0~t`;+zK^ziSYe$SizimG|(=KoEf{XN?6`LSPV z^zbjVKhk7>kM(=r>Q|OD!XK=^-LT&eSbt>|z5KLauIukN?DwO@Ur{g^FQ`8b7XMy} zzaPE*$^v5k!TQ@J{=RSfmE}(GllAvY{C%_ZD{2Ywg8F06^!G~qeHZg9>x%FX*55Ai zcWeBWl}z!I_4iBs-H(1n-BP}w{;;LLSK{wl@+)hE>JQf6F7fw2X}_{oXnwN(eu=*i u?_W`TbRfTt^1mwczZ~fbh`&c=uoaC4z_m literal 0 HcmV?d00001 diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/index.6.0.0-nocfs.zip new file mode 100644 index 0000000000000000000000000000000000000000..55b5cc1d104e902f3045a4cc029c577a4834585e GIT binary patch literal 13749 zcmdU$Wl)^!wr+8EcMDE%m*DPBaCdityF=sd3GNbHgS(U9?g{RCN%mUzCb{SAlT)?+ z?4r7>`_JH;lV&oQ@AJt^fr6m{0Rce)-PwnV2S%+wt-QSZ2n+-S0fYyn%RpynVyvPJ z4FqZm3Z?e_;p_$r1PrzZ3Iz1yskS5Q72T0iT1Hy3g~eJ%wq}fOTJlDknqoq##t0DD zZkAlUdwC%d>72A?Uz@Q2=%avUuXEt2LHZmRthC|fD^MW$U%CTO(C;{d1OoCS_^CUN z|E@c;Z!N8~deO6(uf0AWdb#0OZgR|%`r;8x#(lAmtlHqfrO_m)sY1^Y4;nsNzM;Od z`O=0vN6V)gL!v=N0>6PyAgO}F8pbc2gTzmw5J)|MsEi^&$cMyAa-p`Zsxr%0p4wE| zw&KxdPhyilw#;q1Gs3x=n#$~Ga%6Mg?WDGGWFbpi0=kc%pX!xw!I-V!1Ri@a?@C`r z&}RJfCGm5nX*hY*S5D z@wi{CP^zgd-dH75d6+pdGHF_Z#%A5caPs1Y9tU)jdbj{#ih)Ss-No^E$D;w=DzA2? z&$rG)PUokO*@UzuBTG_ZBG{>bn|S?s@yH&^aDU?-9(W746xBej;-XTK%uK~$>N)DK z(#2&p0Lwe&^o%NCBQ##H{2q&W$Ytz*uSE?Z9O+$7)54 zMH9pGH>@Se3eAK4ncu!c@QE0;ak24bSwcwHW4`DHvhox70wd(my`I@&gai_Z9PqY> zm;hQ_SaF{lnX}`qh^ybG!vgWq0vy{)# zoC9xRp=;fu_YdP4iKqjBvbQMH=@9N}c%H&>JEo1WypJ|cMf~irLgKeh$-?Y9*X>^O z666)lIpp%hoF=lR-tfW;d1C3B9miYKE3HE5KPY5WEiv0hX6c{Cekcvu30z%|243{J zT20r;0Q%sPSpF5&533Lpio3XL6G-FXK^}(<$E_^B5g0-ksp;u(FqObheg1(+FP1_8 zDq!hh4J+dPD53}K@XFEyi!aVAt4Ih>#lu@CZbl}zeIjRr5r^R&4=I*tdgqN66}fMo z*M=nvmL5)m6#B3EL_g~oOBe6W8|aizu6yVWl#rzFBb?>gc17Lp^qxN#q~J%4qvICO zb_vd=REJ44OCzQFaY|dW*f+(uFI7G?eiGrUa2f>Qavk+sw`}xVcdYeC`E2#e5HL0d zTyL3V2N|9adm^lM`INQ6;2#yYoEMw&aN0W^2N zg~=C(5}y~KYk~-R!j@SUN32Xxm6+_NG0rrj!f{k&A$g%P9g25r8HQL4^fJbvwFd*7qPoAnv_cELolo zDV9GsnkrE2d~POsX1gP={UMX|O$F{Q>}TalM;Yf>%M#Q?QSHXdot8$z<=!Yy@55$T z+txP{-s?BupU^y;0;z5spp{OxQ3}@V#JmA9r|rt6QCd9j!W9>&Qdh~2I4))us_{l8 zEe#I|crr$gQJ615g*FO=y9S<+BUW!}Mk>A1?2?J;&VxTO^%%GNt!gaAX$5oO$2MFC zbBy!Vt_OFkA6`qZ#>=MeA|AU>HN@5-9eg&czhUK~%@UmgO;(v|tRk4VH$7br7bZo4 zEvpg|idD_^8)(GNU3q8KLx5xbb)ptkgg{Kti0qxIf-PxO<3QyEh(rfA`vgWv;svPB zUG=@gv>dOQE6udZM+ZO&%`MrO*geOA4+p-0TbCrpR?O?gy~Q0FE`!-oONv~D2=doy zb$iqi1+SaOK3r)Va>v@%)Gsrdz#1=hAseW-aJKjdp%Z-T=UJ%L=leE~063_8Q$mrK z)x7AakydE^76p5Ruie_j!)!TbC&AoNWIaZLV+oQ};EiW-GSc*4_^tb*w`E zGjPz=vJhPMg#cdjvpvLj7Yz{v!~jHMn%i|#Ljn~5s6JbukoQn6F`%YD$j-lBbO~4Z zVFS*_`CiSWSUKW84Y%yNyiJVqZJ}uctuw0gHTtMrY?tcE8Ct`I{fQ5U6%SQpR3ngGH!1tsD;_!c-K9@%w~)yq@7o8NX5F4MzXY@104itrwTOv7KSp`#N)Lp;tn6LW^s&rt6VU{CTUqgZ z-WcS%OsxN^HuVv*KmYFey_l(U;#Y0EDz!yPS>H?WV_n5D0YK_rB3zR@dS6{y8U-){ zq#or4Tb(A=a92v0rU|*Eai%wGJaBA4EOe)$Q!E)%UcKH|QY&tE^~o#g{l1h>z~foQ z5s@Wtu%ryUeF~JloHU}qYAb>so~rlb^27rCcF*o5Si-GwK zuc}hBQ%F2Wkl{Q-J8`*f1ZM}mGT_OTQ^}9Y$BK+;#o9|7iSYC@3qiFzAzU?VB|x6m zCzsXgqs}xTmoM+DbH0A&%$i%Z$OJi!0uQ;z2UP8fdZ)Yrds3MnHkD?LqZ zBQ-f`ytA{@?dX<;lVTaM#h}v(FbuD^g4PwzrHo^<6&tl6shNn7eWD zJE~g|!6%WhLC#{Bv0D-FTahrb)Z2GF^80xPL?PYstz}(Q}T^0qVIi{Uix@F046EeEeQqm1fNI>Sal+)DWQeo8wK$&rjpfv5h%@rJ| zEj47#?IAGbnU!OG!?I$t{3@!$Y`7+(^G1whV~~5v!kWNagUL-Gg2_db3W6aDf=8@Q zA>0Sei_7>+I8Z-PnIwcjs!4b&ScGeuSjb9NYkDi#7=-EAcuP_z$6MNod)oPymJWY+ zGQC_C-E%XT-;~UYS(B6JMOcM?DC~D()n%l!GI#kOd%X6yvc{_?9mHj8{gP4yXnRF3hAozde=vo_B9xmXxfDDx5>6yTh7^54 zSX7XmGvSv;dX7%L2iSQ439TxFopC<|#fjJDpJjp~WKjqDtv818uM_Z~fPesh%kuZ$ z{Cio(wzk3(b2hm;zdG_HY_Fxa&&)_iY$7W^!sCusD9_4_NPI8Bsn1?iGUMXW5}mx@ zx^6>XO%^CiTnAiC0tW8Ckf+L`5D!y)J6&)RKv@CF$MvpOA@+iD>w&wZ-Vy5BQ zaN|Kz_L(|Mp%fN!HcqE_uE=F?DjLU}$Zu_aZ=CS{y_4P9#bE8w8$dmY-g*?od29Wn zYY@9EZR?c(7S`)JU?31IWX3G=P=U@+iOx^~s1K^56z4nKjyxYX?%Y-8$7t}0LtuWtE)QIalbI~y?Y=F;{eE1&kt|f;&A9o`9edHBw z+ixzDhYrqCla18rREC;0=5}XFN@g*rRW6GP#`F1~Wmrm!&n4A$_;77JD|62e^>lA9jv~(98xcq#Xj{S)acW@)DN~ zV$6Y-(9u{{FYeO}=!9b78q$`H&DZIqC)F@crG!q$Eu_BZdzI89BjsXTAYOZ)jkZshao)3MAIa|_YR$i*q_@>7DxeQp?GFPiqqnww!l=c%Ge;Nq#!^O6PgLd|E zR1P9}tuWJOh*9j-1=3-J6VF{TeLH8RowqpVMM(k15Z7@Cy9kM=Prj%4i4a#MUaHoV zN^1wd<`sDal`a*2!tP;7!^t}9;rx_g!G#6^y;!Z%LS6~jNx`>(yilb{fhOwP_vEs( z(rP8mC0Ot<8~I(3GLslGjUvQ87o@M>xU0+|jP8r3f8xC+m<`gL-3E{=aK=^6WT&aU zdWy6Z;ACRejlC@5+u+=W(vdNEg6e0^DkHb|`LLNqKa^jnlE2uHm7Ue?NU{#NA;Pd7 zX*5@n1L?f0RZgWQsGu)WDW3@d1?9sNxSo#H8MtfEJeCx=*q3r8m~VI2tEm)Xh*;T-9!piEt{Ri zv0~)lMIqDU-PqiWi%kIv5Mm$&-oY*k?jH<6=dU6q5z>evW5ky7uZc;h0uv|lL{UXa zCrz`7U_HgXQR2wLf>f-zT}c942Tvff0DYB#Oj<)K371gLF{xLDOGEqeahM+2;utcK zXN#y2qHIAm4pDFcGj`4WN;z~SI?q_Vf49YMe*qDNi;rRF3kiZGkaNsBz(<9X3X}VAhVkmZWLuRF{WjJNt_simN_5* zC7M_qA7~OhI%bf8m@i3;(7qQ^+JF62hd>*hA*h;)iogr26U|JC10xd_rPyf=2`6FV zGKN&Z&?@p&|HA%rj?rzZn;6vlLwjk5tF}q+U=cM%1l*{et<1x&w_Bt5W2J%m9?zPG z`p5yA0Q(? znl2)L#rpt**WvYiUrpa$ty~pS70rJ#vCI)WSjv{A$aL^GirQREO_mqpd^PV2N6ORt9xBZ*Tfo|tVd(pN8Soap{UfO(y)y(Fbi zf0S>7c{ofrcdnqOn~@x;fog8?+Im+w`V&czoT6LGCx{>lN*;;zR1;mT$QJ45J*Iuy z!BSS+Ao+oar$YG_ajdGL*KG_*MtUe@Z*J^kBYZ}B`G}jTn>?Xlf$499QX*!z`5V!M z1?YG3b0ThylkImU%!e~CUum(5lMiTH$~3b+2U)V%*(G)E23y*(Qf)kPgi}l2F&HetCRn3Qic2H7(B2(Abh^;Yd(9ia4y^}6{d@?k~3HP zpE5l!JAT#bxsZqbU$mN$?jO`@tp7Nz#xQx&>fjfx?nREUlWg6-ZC?)NnWbLJ8 zW~IpgLfb*fUM)<|&PSFwJjlvH%FEF?JG=c|Y^x^Xv?_-{ev>!mgm{zXpT(Bx9~N7= z?|b0fA(@SrMevX!){jN-k*jc<1HycQs3M#NS`ib$0=IfPz)K?Y3Q6dZ z=rq?1<-NHFlw#Az05r;_9C&xCCH9uuVRToK;ot^-Q`em~uXEmWuX7VuZEuT(SLc;pZo3{ngrU#i-@uM2H;={>9PSftOElC0 z+H9PJa_i-Y1P^poDM*T74rmS(TXX}A1IsZUME%6qg0f50&*cmEx*vxvxiRm&PpAxv z4G$QjWwk0fN2fK4%XTPW4?NqMij`T|e7-O)f5aiz`y4{Iey9-kLCHGo&8%gA^aDqs z4);~&rU+|TTxlhq95v+A>C}9(Q{+vkK0SW#q7lTOeN~b|gh)Xe|c2!L8|v zDPaIZUdy#JZgATDy^5#2`EfL8aU#Y7Zvd{j(W97m@>|FZuFqsn`i^NBi{_JZ5x^R7 zQwE_J-E~+^fn#!1%Itkt(Z@)LXlDI}CjF)}XPOe1yr*_<4nkc!u^Gb4`7K z+P?O#HFbz_W;eesSL_G;03W>*R0zIuKyQx%382@d(_eVvl zZyR9KF4@FkWT9#7RRvW^R7=lP)WFiC_56Go87D3=dSF$&LWota~=WOisz?wavO=e zJ|*0vJ?@3QQ9Qe#Gmbjio1x6lTqm;eLBQS@5U>HH4Kd}dM0cbecxKS`!3&?h;m4Yp zs+F}+tfYUbYg*}wG6;PYz5A>=99*tmy0KFowNN$w^t5h0>~ytAu_M()VLDVIIleAg z5Hr0;ZN>D>y%pP4m!&xx_jNkX6~KTrbzS1{A#E)vU_vA=_EdZoEAg`DmJ^CJG_ujO z8V4_*kQd|A2}=cW5RBX=PF5c?twWD8(fc(8t#GS2^=7Kcf#XwdHx{PwEwjfqD)@H7 zSSZ#*LZpi?;f3o7tKk(+z@@ck)VDN@Y1sHdRS1H#akB9jE^$W``@nGor2E8i1mye3 zO}v+ryPVA(&>L%TNHI|Mxq%O-?A14c!gA=K!tfL3zI32Ha1++P1fV_e6PCV#3}X0X z&uDJ0HpuVwmKj)>cPwxR9UfRw#0?yaMXcOrf?XF*=x_;Y*O%E^`-A(f-74Yb#EU=p zg&Ed?hGzR0u(RApIj6O)J5xOed-xejg`XHD9v;SbjVlra4r2So?K9=3Vh;9ynn4K2 z8Fxz-r@Qn0q@Iu>&U*q1u8HPHQ3#?;qsfr~5i*&Gq2715fQ1~eRXfK!pJB?y-A>dr zg+O}$a#4GPjq)?3%}gg1yTvd%yTzfY&5VoM%9n6ie0Pq93liQiAHUUpwNsxjjT;E} z6Kywb_qqjB`fQ|&*GWYb+Q-c{m~SrkDzkF)wCR#_o#}l)Bn7f!am-&Gz0V%FC)eR} zXX>zi)&QezRoT>`@!Xf2Z%D$FEfeT7t5OiUI}>x-C#A|m1!9Eq-2?OadNv{4=Vlt+ zL&wegI9eJ){xDj~EcCQ9i;CUh@zK`;#hSFPvB1_TcfEz2#ZyYKn>}=;`apW9c*}coEw^Ptsd91S}n*nkvWwBVIF2@lgtLrpDr>?XWDR|aB zeCo3Fw=^l_eRBbe?!I+6hD%m~yeu_6z09*nD4g3pfO+888iRr9_xklX)a>U$a0rlHDx(JL~hJI&zp3|Nb0!$oQZ0UX1 zNGtof$E?uiQTb3V_;Ytj6eSCFOqT9$i9JA`*E?0nxczT)hNa4;rmztlLE^*!==Mi< z@cfcd1k|qr@$X7f&jo9HM7c&i^6-;R4w|w@cfslF?&}-`Mlo0?Bjyu|&fes22xMa} z^giT?KqS;OI}nyK$YxhU$z=OupcfY^RaO1D%P>t&cNOZt;u{^p$x&7Ym;bUU(nwwOjwGyXz#9#&CUvVgM2FL z1vGQWZKE-pXV-Dnr-Ds>rHVP@K>QhcvJDfpYi!z&Zp6b?1F?^x>2cv3vF4U-2hZwP z#EzORjHY?I6Bpyk_LJUMSI3z&pKg4Bf3@0T{~G3Btd{AYv|6zLG^-7UARj3^6?N}T zwc=;uq(v7*XZ~MU?Ud@B6~u4imNOv0IsdcOGXK+7i+%I5fK{aa?z561ReRfsFPmB! z`Gqt-0u#GdKOF&YMrBX>K!YuTM7grS@a({nbK^-GKkOk=h)HvTA_)cWv4DTEiSfP9l3wL0$N zLSFm+J&MEe(fd0+&OjO39c3n(tV9@AGLcDem zS!N5)=vc&FJ8P(D%w~G^yfLHjJB4Vr#@`mMpX`UYhQG?XvQ-wRj^2himK1D>4qC_vyP@`VEu6Bfu_3_7cQW3mq;Kw+93uL*&Rg$^$j=a>Z7J~k z+$_HH&V8yfrkAKpM04~~G=0dhu@-U%Nc+wOC9)`ai%F89Hj;@OtDv3jZ9+n)wg??UO5rrks<1S!yo(d59~A0|zclKT#%a(x zp*7H>YJ7TpeMj?_)hBz;!Yx|9GPh~Z0B)KE(4sX?pTa(6+&a$wMro);>s4uXbvY?3 zX{It&)F}8_0iP@3JK9u9OwrXZPbP}kUWWZ$kq-990_sWe^N|Hcb--kWMYjSq_YK7h z{9rRtdEOS_OlLW9qf5lQ@g>bLHx0lIK5Vza0+}7yde1a^Ul(m4z$vi&7DX${a$qam z{gK}mnmJfd3(3RCkubCOBs%1es&WdMM^!8%Z!T+i0hH zdl-V4G{Hy3;KTMs`W`eV;u*5I>Qi6?%4m;F!}M%Qi_^AYA0H2ptRTc?1PnNBx#*H$ zHJ+EDxF(FVNE~H}V1o-Z-NYsa@7iv<`uEXXK$gq5mGf!dH)W*A`1x0kny|HG^1J5O zlLxZ~y_I?Use(O~t7edwQ(OLPm!VQUO)jMKg(oB<vNG@tzy9L#o;0vbU<&Rmjd*I)iLjlP?X68o>aCO=4~T&*C~nYW zvJ(_^DanpFf^$uFlUHdHptAy+gkj#5J7iOk@oq0Gz2xrV(Q%*4deNPqH zi*}9;W@xNEz{#G&2*~bws9Z1ue^Ur`(sb*7^vR5nkVRG@FOFO$3n7M#H=O64lMWWh`TYDD@M_xKLa23Z;d$Xt`_d z9LCH-xQ1cMXyLW3ORcai>P^mud!+mIr8~+js)QWQ#Rp_W`Q4aR!6i|fP|A#4qz6e@ zq)v_-j0?#LEsV<~&zZR7*S8Q-?Fwc|7a25TN4PgOB(_>}wFA_LX%sqm0&4Kip`l`! zO?FuE%zW0ML|oGQp|aqQZkWx!=?Bgr(-!5hnNFxMYI(w1dkG=(dSU+W>QV8g09YO-c=KvyCW+M7wRPlzb?u9=V&d9*AR`3hfoIOb@TLc`Q>5zC{! zMEfzTo21hBn^{CFrF?eE?dUz%7X{Y^QNP89v1Iz?S!tQd(VGVYGkRjq<75c;hFBM4 zel0qRcTpav;x2LRz@IgZ(6pbA#`$6>2ERX*R3U}X0f*Qnx32C6{$%=E2&<-qu4BC) ze~xhDhEu;ef_zxLi|;55D`uD>dbQMa^P4CR`*dDb#l-MVlrlGGvaFyv`*G31sAws5 zKmNNc_ENRkld6J94@z8F3%BSToe*c^du3O`owZHPWBvWO*+lS7{bYyCw=4KZeYmNxeVpdH&50If-Dge(^)* zf6@=p{^f^d|1n{L;NZR&KkWO<56iIx^Zz$n&yq|gU98_Uh^wk0B=N-$VgF9Qvi!q- zIF8Nn-47*y_+fv|iyv+aaxTkrffO+zlHXuSK5qd+<}qw@)Z)~EkWE@a6Dz?}6F~Ht zMFhdi1zc45!3+7p0~Dks2jzl?_(FzubLYXFp&coepKgAnR;_|F*g5ZhD=O-(k;KGH zL9fC7a=Y(E{ad&DkwMw`J0J0Pq&Mzu+`#{z=N-rX>Q`c{R}vs~7NMxL#H8e?7`}`W zA(hjN#puM!!obAHz<7$s(KV2flHvv|&BXNGLj8eZwp80(fwAdeSeL%gcThHbVgE?V z2%fA-s_$<3VXf?$zY}0pp?iG{dTv3t6cS?GXch`R&#TIC)?<)o#h;rXh=fKEAC^S|v zoLSy$FFgfE-6F5Sfh0L6Y7baN`cRZ~4kh#&)zyqUh!|CMtzx(x&=I4)9w>)cbR=fC zSGY}MmP2SUv2hHq^w%+U64c`~w&JoQM^(guKYhp<_%QNmsBf%muy?e3xSyVglJqqR zH8~wI6&dX-8VZ2b;yBQpw_Ugpyj$|f77EBZ0)m10*J!YGfzDL;$dQbgj9#>U-yWkl zBO~{okpZYN7oyj_12g^pJt57li1WC$UNZ7|%IbzkuYUVH)N5OK|1w2b{>dqV_pd1u z`5!YyL?QZK&Q8d`N0N|AQvSDRC(W^5w3xp&_%&yErOV3{@qPKFgL%n(IU1W<8{0TJ z>N5XNp`N#N&o+N@z>M$D|dA;cQU54vNg2)apfj2>HqJK-|hgAmjVVM0R8XoA;9{- z&^o{GFaN%y0Q_Hf6#NgYUlaE~vj|?g`-Al|%KnV?YZmWkmOS(i*7qaCKSTYRI{O(_ z{nE|g$0o|lmo_*M KkRZeNcmD_LHHqB- literal 0 HcmV?d00001 From 30d612f84ed71b00ef981c24dd77d305b326a7e8 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Fri, 8 Apr 2016 15:53:12 -0400 Subject: [PATCH 0314/1113] LUCENE-7195: Clockwise/counterclockwise detection was rotating coordinates in the wrong direction. --- .../spatial3d/geom/GeoPolygonFactory.java | 45 +++++++++---------- .../lucene/spatial3d/TestGeo3DPoint.java | 34 +++++++++----- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 67804a2accf1..a68f908dc225 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -116,6 +116,7 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, final Boolean isPoleInside = isInsidePolygon(pole, pointList); if (isPoleInside != null) { // Legal pole + //System.out.println("Pole = "+pole+"; isInside="+isPoleInside+"; pointList = "+pointList); return makeGeoPolygon(planetModel, pointList, holes, pole, isPoleInside); } // If pole choice was illegal, try another one @@ -177,24 +178,17 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, */ private static Boolean isInsidePolygon(final GeoPoint point, final List polyPoints) { // First, compute sine and cosine of pole point latitude and longitude - final double norm = 1.0 / point.magnitude(); - final double xyDenom = Math.sqrt(point.x * point.x + point.y * point.y); - final double sinLatitude = point.z * norm; - final double cosLatitude = xyDenom * norm; - final double sinLongitude; - final double cosLongitude; - if (Math.abs(xyDenom) < Vector.MINIMUM_RESOLUTION) { - sinLongitude = 0.0; - cosLongitude = 1.0; - } else { - final double xyNorm = 1.0 / xyDenom; - sinLongitude = point.y * xyNorm; - cosLongitude = point.x * xyNorm; - } + final double latitude = point.getLatitude(); + final double longitude = point.getLongitude(); + final double sinLatitude = Math.sin(latitude); + final double cosLatitude = Math.cos(latitude); + final double sinLongitude = Math.sin(longitude); + final double cosLongitude = Math.cos(longitude); // Now, compute the incremental arc distance around the points of the polygon double arcDistance = 0.0; Double prevAngle = null; + //System.out.println("Computing angles:"); for (final GeoPoint polyPoint : polyPoints) { final Double angle = computeAngle(polyPoint, sinLatitude, cosLatitude, sinLongitude, cosLongitude); if (angle == null) { @@ -215,6 +209,7 @@ private static Boolean isInsidePolygon(final GeoPoint point, final List Date: Thu, 7 Apr 2016 15:43:24 -0700 Subject: [PATCH 0315/1113] SOLR-8097: Implement builder pattern design for constructing SolrJ clients and deprecate direct construction of clients --- solr/CHANGES.txt | 3 + .../dataimport/SolrEntityProcessor.java | 10 +- .../TestContentStreamDataSource.java | 4 +- .../TestSolrEntityProcessorEndToEnd.java | 2 +- .../java/org/apache/solr/hadoop/GoLive.java | 6 +- .../hadoop/MorphlineGoLiveMiniMRTest.java | 4 +- .../solr/SafeConcurrentUpdateSolrClient.java | 2 +- .../solr/morphlines/solr/SolrLocator.java | 5 +- .../solr/AbstractSolrMorphlineTestBase.java | 34 +-- .../cloud/LeaderInitiatedRecoveryThread.java | 2 +- .../OverseerAutoReplicaFailoverThread.java | 32 ++- .../OverseerCollectionMessageHandler.java | 2 +- .../apache/solr/cloud/RecoveryStrategy.java | 5 +- .../org/apache/solr/cloud/SyncStrategy.java | 2 +- .../org/apache/solr/cloud/ZkController.java | 3 +- .../apache/solr/cloud/rule/SnitchContext.java | 3 +- .../solr/handler/CdcrReplicatorManager.java | 18 +- .../solr/handler/CdcrRequestHandler.java | 2 +- .../handler/CdcrUpdateLogSynchronizer.java | 2 +- .../org/apache/solr/handler/IndexFetcher.java | 12 +- .../solr/handler/SolrConfigHandler.java | 3 +- .../handler/admin/CollectionsHandler.java | 3 +- .../handler/component/HttpShardHandler.java | 5 +- .../component/HttpShardHandlerFactory.java | 6 +- .../component/IterativeMergeStrategy.java | 7 +- .../solr/schema/ManagedIndexSchema.java | 2 +- .../solr/update/SolrCmdDistributor.java | 3 +- .../solr/update/StreamingSolrClients.java | 66 ++--- .../java/org/apache/solr/util/SolrCLI.java | 21 +- .../org/apache/solr/TestTolerantSearch.java | 6 +- .../client/solrj/ConnectionReuseTest.java | 6 +- .../solrj/embedded/TestJettySolrRunner.java | 3 +- .../solr/cloud/AliasIntegrationTest.java | 28 +-- .../solr/cloud/AsyncMigrateRouteKeyTest.java | 6 +- .../solr/cloud/BaseCdcrDistributedZkTest.java | 5 +- .../solr/cloud/BasicDistributedZk2Test.java | 6 +- .../solr/cloud/BasicDistributedZkTest.java | 15 +- .../cloud/ChaosMonkeyNothingIsSafeTest.java | 49 ++-- .../solr/cloud/CollectionReloadTest.java | 2 +- .../CollectionsAPIDistributedZkTest.java | 12 +- .../solr/cloud/CollectionsAPISolrJTest.java | 2 +- ...ncurrentDeleteAndCreateCollectionTest.java | 6 +- .../solr/cloud/CustomCollectionTest.java | 12 +- .../solr/cloud/DeleteInactiveReplicaTest.java | 2 +- .../apache/solr/cloud/DeleteReplicaTest.java | 4 +- .../apache/solr/cloud/DeleteShardTest.java | 6 +- .../cloud/DistributedVersionInfoTest.java | 8 +- .../apache/solr/cloud/ForceLeaderTest.java | 2 +- .../cloud/FullSolrCloudDistribCmdsTest.java | 2 +- .../apache/solr/cloud/HttpPartitionTest.java | 2 +- .../LeaderInitiatedRecoveryOnCommitTest.java | 2 +- ...erInitiatedRecoveryOnShardRestartTest.java | 2 +- .../solr/cloud/MigrateRouteKeyTest.java | 4 +- .../solr/cloud/ReplicationFactorTest.java | 2 +- .../apache/solr/cloud/SSLMigrationTest.java | 2 +- .../org/apache/solr/cloud/ShardSplitTest.java | 32 +-- .../org/apache/solr/cloud/SyncSliceTest.java | 2 +- .../solr/cloud/TestCloudDeleteByQuery.java | 10 +- .../apache/solr/cloud/TestConfigSetsAPI.java | 20 +- .../cloud/TestConfigSetsAPIExclusivity.java | 4 +- .../cloud/TestConfigSetsAPIZkFailure.java | 4 +- .../org/apache/solr/cloud/TestCryptoKeys.java | 2 +- .../cloud/TestMiniSolrCloudClusterSSL.java | 2 +- .../cloud/TestRandomRequestDistribution.java | 4 +- .../cloud/TestRequestStatusCollectionAPI.java | 2 +- .../TestTolerantUpdateProcessorCloud.java | 10 +- ...estTolerantUpdateProcessorRandomCloud.java | 2 +- .../solr/cloud/UnloadDistributedZkTest.java | 30 +-- .../HdfsWriteToMultipleCollectionsTest.java | 2 +- .../solr/cloud/hdfs/StressHdfsTest.java | 2 +- .../solr/core/OpenCloseCoreStressTest.java | 4 +- .../apache/solr/core/TestDynamicLoading.java | 2 +- .../solr/handler/TestReplicationHandler.java | 5 +- .../handler/TestReplicationHandlerBackup.java | 3 +- .../apache/solr/handler/TestRestoreCore.java | 3 +- .../handler/admin/CoreAdminHandlerTest.java | 4 +- .../DistributedDebugComponentTest.java | 6 +- ...istributedQueryElevationComponentTest.java | 3 +- .../org/apache/solr/search/TestSolrJ.java | 4 +- .../solr/search/stats/TestDistribIDF.java | 8 +- .../security/BasicAuthIntegrationTest.java | 3 +- .../processor/TestNamedUpdateProcessors.java | 3 +- .../solr/util/TestSolrCLIRunExample.java | 4 +- .../client/solrj/impl/CloudSolrClient.java | 229 +++++++++++++++--- .../impl/ConcurrentUpdateSolrClient.java | 146 +++++++++-- .../client/solrj/impl/HttpSolrClient.java | 112 +++++++-- .../client/solrj/impl/LBHttpSolrClient.java | 126 ++++++++-- .../solr/client/solrj/io/SolrClientCache.java | 9 +- .../solrj/io/sql/DatabaseMetaDataImpl.java | 4 +- .../solrj/io/stream/CloudSolrStream.java | 7 +- .../client/solrj/io/stream/FacetStream.java | 8 +- .../client/solrj/io/stream/SolrStream.java | 4 +- .../client/solrj/io/stream/StatsStream.java | 7 +- .../client/solrj/io/stream/TopicStream.java | 8 +- .../client/solrj/io/stream/UpdateStream.java | 5 +- .../client/solrj/SolrExampleBinaryTest.java | 2 +- .../solr/client/solrj/SolrExampleTests.java | 2 +- .../solr/client/solrj/SolrExampleXMLTest.java | 2 +- .../solr/client/solrj/SolrExceptionTest.java | 4 +- .../solrj/SolrSchemalessExampleTest.java | 2 +- .../client/solrj/TestLBHttpSolrClient.java | 30 +-- .../solrj/embedded/SolrExampleJettyTest.java | 2 +- .../embedded/SolrExampleStreamingTest.java | 23 +- .../solrj/impl/BasicHttpSolrClientTest.java | 37 +-- .../impl/CloudSolrClientBuilderTest.java | 90 +++++++ .../solrj/impl/CloudSolrClientTest.java | 21 +- ...ConcurrentUpdateSolrClientBuilderTest.java | 33 +++ .../impl/ConcurrentUpdateSolrClientTest.java | 37 ++- .../solrj/impl/HttpSolrClientBuilderTest.java | 76 ++++++ .../impl/LBHttpSolrClientBuilderTest.java | 65 +++++ .../solr/BaseDistributedSearchTestCase.java | 2 +- .../org/apache/solr/SolrJettyTestBase.java | 2 +- .../java/org/apache/solr/SolrTestCaseHS.java | 40 ++- .../java/org/apache/solr/SolrTestCaseJ4.java | 142 +++++++++++ .../cloud/AbstractFullDistribZkTestBase.java | 18 +- .../solr/cloud/MiniSolrCloudCluster.java | 5 +- 116 files changed, 1442 insertions(+), 503 deletions(-) create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b2f9a84f6564..a1a1fe0f84f8 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -106,6 +106,9 @@ Other Changes * SOLR-8892: Allow SolrInfoMBeans to return different statistics for /jmx vs web ui calls. (Gregory Chanan, Mark Miller) +* SOLR-8097: Implement builder pattern design for constructing SolrJ clients and also deprecate direct construction + of client objects. (Jason Gerlowski, Shawn Heisey, Anshum Gupta) + ================== 6.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index cece0cc9a4c7..5e62731879e6 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -22,6 +22,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -111,11 +112,16 @@ protected void firstInit(Context context) { // (wt="javabin|xml") default is javabin if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) { // TODO: it doesn't matter for this impl when passing a client currently, but we should close this! - solrClient = new HttpSolrClient(url.toExternalForm(), client, new XMLResponseParser()); + solrClient = new Builder(url.toExternalForm()) + .withHttpClient(client) + .withResponseParser(new XMLResponseParser()) + .build(); LOG.info("using XMLResponseParser"); } else { // TODO: it doesn't matter for this impl when passing a client currently, but we should close this! - solrClient = new HttpSolrClient(url.toExternalForm(), client); + solrClient = new Builder(url.toExternalForm()) + .withHttpClient(client) + .build(); LOG.info("using BinaryResponseParser"); } } catch (MalformedURLException e) { diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java index 453f5713709d..95af5f9494b7 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java @@ -69,7 +69,7 @@ public void testSimple() throws Exception { params.set("command", "full-import"); params.set("clean", "false"); req.setParams(params); - try (HttpSolrClient solrClient = new HttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) { + try (HttpSolrClient solrClient = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) { solrClient.request(req); ModifiableSolrParams qparams = new ModifiableSolrParams(); qparams.add("q", "*:*"); @@ -89,7 +89,7 @@ public void testCommitWithin() throws Exception { "clean", "false", UpdateParams.COMMIT, "false", UpdateParams.COMMIT_WITHIN, "1000"); req.setParams(params); - try (HttpSolrClient solrServer = new HttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) { + try (HttpSolrClient solrServer = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) { solrServer.request(req); Thread.sleep(100); ModifiableSolrParams queryAll = params("q", "*"); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java index abefc493f6b1..8ef94c02c76a 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java @@ -282,7 +282,7 @@ private void addDocumentsToSolr(List> docs) throws SolrServer sidl.add(sd); } - try (HttpSolrClient solrServer = new HttpSolrClient(getSourceUrl())) { + try (HttpSolrClient solrServer = getHttpSolrClient(getSourceUrl())) { solrServer.setConnectionTimeout(15000); solrServer.setSoTimeout(30000); solrServer.add(sidl); diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java index bd48c643623d..5b1c343ed684 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/GoLive.java @@ -90,7 +90,7 @@ public boolean goLive(Options options, FileStatus[] outDirs) { Callable task = () -> { Request req = new Request(); LOG.info("Live merge " + dir.getPath() + " into " + mergeUrl); - try (final HttpSolrClient client = new HttpSolrClient(mergeUrl)) { + try (final HttpSolrClient client = new HttpSolrClient.Builder(mergeUrl).build()) { CoreAdminRequest.MergeIndexes mergeRequest = new CoreAdminRequest.MergeIndexes(); mergeRequest.setCoreName(name); mergeRequest.setIndexDirs(Arrays.asList(dir.getPath().toString() + "/data/index")); @@ -138,7 +138,7 @@ public boolean goLive(Options options, FileStatus[] outDirs) { try { LOG.info("Committing live merge..."); if (options.zkHost != null) { - try (CloudSolrClient server = new CloudSolrClient(options.zkHost)) { + try (CloudSolrClient server = new CloudSolrClient.Builder().withZkHost(options.zkHost).build()) { server.setDefaultCollection(options.collection); server.commit(); } @@ -146,7 +146,7 @@ public boolean goLive(Options options, FileStatus[] outDirs) { for (List urls : options.shardUrls) { for (String url : urls) { // TODO: we should do these concurrently - try (HttpSolrClient server = new HttpSolrClient(url)) { + try (HttpSolrClient server = new HttpSolrClient.Builder(url).build()) { server.commit(); } } diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java index 95ed9b2b17d1..ce3d8c48d071 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java @@ -380,7 +380,7 @@ public void test() throws Exception { String[] args = new String[]{}; List argList = new ArrayList<>(); - try (HttpSolrClient server = new HttpSolrClient(cloudJettys.get(0).url)) { + try (HttpSolrClient server = getHttpSolrClient(cloudJettys.get(0).url)) { args = new String[]{ "--solr-home-dir=" + MINIMR_CONF_DIR.getAbsolutePath(), @@ -706,7 +706,7 @@ private void checkConsistency(String replicatedCollection) Collection replicas = slice.getReplicas(); long found = -1; for (Replica replica : replicas) { - try (HttpSolrClient client = new HttpSolrClient(new ZkCoreNodeProps(replica).getCoreUrl())) { + try (HttpSolrClient client = getHttpSolrClient(new ZkCoreNodeProps(replica).getCoreUrl())) { SolrQuery query = new SolrQuery("*:*"); query.set("distrib", false); QueryResponse replicaResults = client.query(query); diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java index 1d632f1dc168..a5fb929c92d1 100644 --- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java +++ b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SafeConcurrentUpdateSolrClient.java @@ -39,7 +39,7 @@ public SafeConcurrentUpdateSolrClient(String solrServerUrl, int queueSize, int t } public SafeConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, int threadCount) { - super(solrServerUrl, client, queueSize, threadCount); + super(solrServerUrl, client, queueSize, threadCount, null, false); } @Override diff --git a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java index 4acb9ac5c484..39cee54e69a2 100644 --- a/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java +++ b/solr/contrib/morphlines-core/src/java/org/apache/solr/morphlines/solr/SolrLocator.java @@ -30,6 +30,7 @@ import com.typesafe.config.ConfigUtil; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; @@ -92,7 +93,9 @@ public DocumentLoader getLoader() { if (collectionName == null || collectionName.length() == 0) { throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config); } - CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost); + CloudSolrClient cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); cloudSolrClient.setDefaultCollection(collectionName); cloudSolrClient.connect(); return new SolrClientDocumentLoader(cloudSolrClient, batchSize); diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java index 9c79c07d2699..9e65c68a1f9e 100644 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java +++ b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java @@ -16,6 +16,22 @@ */ package org.apache.solr.morphlines.solr; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; + import com.codahale.metrics.MetricRegistry; import com.google.common.base.Joiner; import com.google.common.io.Files; @@ -46,22 +62,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicInteger; - public class AbstractSolrMorphlineTestBase extends SolrTestCaseJ4 { private static Locale savedLocale; protected Collector collector; @@ -123,7 +123,7 @@ public void setUp() throws Exception { if (EXTERNAL_SOLR_SERVER_URL != null) { //solrServer = new ConcurrentUpdateSolrServer(EXTERNAL_SOLR_SERVER_URL, 2, 2); //solrServer = new SafeConcurrentUpdateSolrServer(EXTERNAL_SOLR_SERVER_URL, 2, 2); - solrClient = new HttpSolrClient(EXTERNAL_SOLR_SERVER_URL); + solrClient = getHttpSolrClient(EXTERNAL_SOLR_SERVER_URL); ((HttpSolrClient) solrClient).setParser(new XMLResponseParser()); } else { if (TEST_WITH_EMBEDDED_SOLR_SERVER) { diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java index 589ed83e8330..4389d9e2d73f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java +++ b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java @@ -201,7 +201,7 @@ protected void sendRecoveryCommandWithRetry() throws Exception { log.info("Asking core={} coreNodeName={} on " + recoveryUrl + " to recover", coreNeedingRecovery, replicaCoreNodeName); } - try (HttpSolrClient client = new HttpSolrClient(recoveryUrl)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(recoveryUrl).build()) { client.setSoTimeout(60000); client.setConnectionTimeout(15000); try { diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java b/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java index a9f8bdd0997c..f94ffcc73ba0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerAutoReplicaFailoverThread.java @@ -16,9 +16,22 @@ */ package org.apache.solr.cloud; +import java.io.Closeable; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; - import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; import org.apache.solr.common.SolrException; @@ -34,21 +47,6 @@ import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.io.Closeable; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - // TODO: how to tmp exclude nodes? @@ -434,7 +432,7 @@ private boolean createSolrCore(final String collection, final String createUrl, final String dataDir, final String ulogDir, final String coreNodeName, final String coreName) { - try (HttpSolrClient client = new HttpSolrClient(createUrl)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(createUrl).build()) { log.debug("create url={}", createUrl); client.setConnectionTimeout(30000); client.setSoTimeout(60000); diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index 78db47332f0a..503ff291653e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -1315,7 +1315,7 @@ private void commit(NamedList results, String slice, Replica parentShardLeader) static UpdateResponse softCommit(String url) throws SolrServerException, IOException { - try (HttpSolrClient client = new HttpSolrClient(url)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(url).build()) { client.setConnectionTimeout(30000); client.setSoTimeout(120000); UpdateRequest ureq = new UpdateRequest(); diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 5daa6af22253..61b3f90b8390 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud; import java.io.Closeable; - import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -195,7 +194,7 @@ private void replicate(String nodeName, SolrCore core, ZkNodeProps leaderprops) private void commitOnLeader(String leaderUrl) throws SolrServerException, IOException { - try (HttpSolrClient client = new HttpSolrClient(leaderUrl)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderUrl).build()) { client.setConnectionTimeout(30000); UpdateRequest ureq = new UpdateRequest(); ureq.setParams(new ModifiableSolrParams()); @@ -575,7 +574,7 @@ public boolean isClosed() { private void sendPrepRecoveryCmd(String leaderBaseUrl, String leaderCoreName, Slice slice) throws SolrServerException, IOException, InterruptedException, ExecutionException { - try (HttpSolrClient client = new HttpSolrClient(leaderBaseUrl)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderBaseUrl).build()) { client.setConnectionTimeout(30000); WaitForState prepCmd = new WaitForState(); prepCmd.setCoreName(leaderCoreName); diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java index cedcff9ece73..b1d69be97020 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java @@ -292,7 +292,7 @@ public void run() { recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY); recoverRequestCmd.setCoreName(coreName); - try (HttpSolrClient client = new HttpSolrClient(baseUrl, SyncStrategy.this.client)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl).withHttpClient(SyncStrategy.this.client).build()) { client.setConnectionTimeout(30000); client.setSoTimeout(120000); client.request(recoverRequestCmd); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 81897b717e92..e288f5e926da 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -35,6 +35,7 @@ import com.google.common.base.Strings; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.cloud.overseer.SliceMutator; @@ -1574,7 +1575,7 @@ private ZkCoreNodeProps waitForLeaderToSeeDownState( log.info("Replica " + myCoreNodeName + " NOT in leader-initiated recovery, need to wait for leader to see down state."); - try (HttpSolrClient client = new HttpSolrClient(leaderBaseUrl)) { + try (HttpSolrClient client = new Builder(leaderBaseUrl).build()) { client.setConnectionTimeout(15000); client.setSoTimeout(120000); WaitForState prepCmd = new WaitForState(); diff --git a/solr/core/src/java/org/apache/solr/cloud/rule/SnitchContext.java b/solr/core/src/java/org/apache/solr/cloud/rule/SnitchContext.java index b8993da55089..28089068d7e4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/rule/SnitchContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/rule/SnitchContext.java @@ -105,7 +105,8 @@ public void invokeRemote(String node, ModifiableSolrParams params, String klas, public SimpleSolrResponse invoke(UpdateShardHandler shardHandler, final String url, String path, SolrParams params) throws IOException, SolrServerException { GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params); - try (HttpSolrClient client = new HttpSolrClient(url, shardHandler.getHttpClient(), new BinaryResponseParser())) { + try (HttpSolrClient client = new HttpSolrClient.Builder(url).withHttpClient(shardHandler.getHttpClient()) + .withResponseParser(new BinaryResponseParser()).build()) { NamedList rsp = client.request(request); request.response.nl = rsp; return request.response; diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java index 2822e609ae22..af0161ed3c59 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java @@ -16,9 +16,16 @@ */ package org.apache.solr.handler; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -30,12 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { private List replicatorStates; @@ -64,7 +65,10 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { String zkHost = params.get(CdcrParams.ZK_HOST_PARAM); String targetCollection = params.get(CdcrParams.TARGET_COLLECTION_PARAM); - CloudSolrClient client = new CloudSolrClient(zkHost, true); + CloudSolrClient client = new Builder() + .withZkHost(zkHost) + .sendUpdatesOnlyToShardLeaders() + .build(); client.setDefaultCollection(targetCollection); replicatorStates.add(new CdcrReplicatorState(targetCollection, zkHost, client)); } diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java index 23e4abac304b..f60e5621bfb6 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java @@ -595,7 +595,7 @@ private static final class SliceCheckpointCallable implements Callable { @Override public Long call() throws Exception { - try (HttpSolrClient server = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient server = new HttpSolrClient.Builder(baseUrl).build()) { server.setConnectionTimeout(15000); server.setSoTimeout(60000); diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrUpdateLogSynchronizer.java b/solr/core/src/java/org/apache/solr/handler/CdcrUpdateLogSynchronizer.java index 4955992a9384..7071908e70f4 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrUpdateLogSynchronizer.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrUpdateLogSynchronizer.java @@ -129,7 +129,7 @@ public void run() { return; } - HttpSolrClient server = new HttpSolrClient(leaderUrl); + HttpSolrClient server = new HttpSolrClient.Builder(leaderUrl).build(); server.setConnectionTimeout(15000); server.setSoTimeout(60000); diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 39c4158425d9..0686f5ba6965 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -218,7 +218,7 @@ NamedList getLatestVersion() throws IOException { QueryRequest req = new QueryRequest(params); // TODO modify to use shardhandler - try (HttpSolrClient client = new HttpSolrClient(masterUrl, myHttpClient)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { client.setSoTimeout(60000); client.setConnectionTimeout(15000); @@ -240,7 +240,7 @@ private void fetchFileList(long gen) throws IOException { QueryRequest req = new QueryRequest(params); // TODO modify to use shardhandler - try (HttpSolrClient client = new HttpSolrClient(masterUrl, myHttpClient)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { client.setSoTimeout(60000); client.setConnectionTimeout(15000); NamedList response = client.request(req); @@ -1606,7 +1606,11 @@ private FastInputStream getStream() throws IOException { InputStream is = null; // TODO use shardhandler - try (HttpSolrClient client = new HttpSolrClient(masterUrl, myHttpClient, null)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl) + .withHttpClient(myHttpClient) + .withResponseParser(null) + .build() + ) { client.setSoTimeout(60000); client.setConnectionTimeout(15000); QueryRequest req = new QueryRequest(params); @@ -1715,7 +1719,7 @@ NamedList getDetails() throws IOException, SolrServerException { params.set(CommonParams.QT, "/replication"); // TODO use shardhandler - try (HttpSolrClient client = new HttpSolrClient(masterUrl, myHttpClient)) { + try (HttpSolrClient client = new HttpSolrClient.Builder(masterUrl).withHttpClient(myHttpClient).build()) { client.setSoTimeout(60000); client.setConnectionTimeout(15000); QueryRequest request = new QueryRequest(params); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index eac9b11684ee..972859705c92 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -69,7 +69,6 @@ import org.apache.solr.schema.SchemaManager; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; -import org.apache.solr.security.PermissionNameProvider.Name; import org.apache.solr.util.CommandOperation; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.RTimer; @@ -785,7 +784,7 @@ public SolrParams getParams() { public Boolean call() throws Exception { final RTimer timer = new RTimer(); int attempts = 0; - try (HttpSolrClient solr = new HttpSolrClient(coreUrl)) { + try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) { // eventually, this loop will get killed by the ExecutorService's timeout while (true) { try { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index f0a332ebbf4a..64b10ab0c0f9 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard; import org.apache.solr.client.solrj.response.RequestStatusState; @@ -427,7 +428,7 @@ Map call(SolrQueryRequest req, SolrQueryResponse rsp, Collection ZkNodeProps leaderProps = clusterState.getLeader(collection, shard); ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(leaderProps); - try (HttpSolrClient client = new HttpSolrClient(nodeProps.getBaseUrl())) { + try (HttpSolrClient client = new Builder(nodeProps.getBaseUrl()).build()) { client.setConnectionTimeout(15000); client.setSoTimeout(60000); RequestSyncShard reqSyncShard = new CoreAdminRequest.RequestSyncShard(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 4ae013c56811..609e43376675 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -35,7 +35,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.LBHttpSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.util.ClientUtils; @@ -54,7 +54,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -192,7 +191,7 @@ public void submit(final ShardRequest sreq, final String shard, final Modifiable if (urls.size() <= 1) { String url = urls.get(0); srsp.setShardAddress(url); - try (SolrClient client = new HttpSolrClient(url, httpClient)) { + try (SolrClient client = new Builder(url).withHttpClient(httpClient).build()) { ssr.nl = client.request(req); } } else { diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index d2800d75ce00..47e688c281b0 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -23,6 +23,7 @@ import org.apache.solr.client.solrj.impl.HttpClientConfigurer; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.LBHttpSolrClient; +import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ExecutorUtil; @@ -202,7 +203,10 @@ protected ThreadPoolExecutor getThreadPoolExecutor(){ } protected LBHttpSolrClient createLoadbalancer(HttpClient httpClient){ - return new LBHttpSolrClient(httpClient); + LBHttpSolrClient client = new Builder() + .withHttpClient(httpClient) + .build(); + return client; } protected T getParameter(NamedList initArgs, String configKey, T defaultValue, StringBuilder sb) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java b/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java index a8f6ca96e8f2..34f6ed32bac6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java +++ b/solr/core/src/java/org/apache/solr/handler/component/IterativeMergeStrategy.java @@ -29,6 +29,7 @@ import org.apache.solr.client.solrj.impl.HttpClientConfigurer; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; @@ -84,7 +85,9 @@ public static class CallBack implements Callable { public CallBack(ShardResponse originalShardResponse, QueryRequest req) { - this.solrClient = new HttpSolrClient(originalShardResponse.getShardAddress(), getHttpClient()); + this.solrClient = new Builder(originalShardResponse.getShardAddress()) + .withHttpClient(getHttpClient()) + .build(); this.req = req; this.originalShardResponse = originalShardResponse; req.setMethod(SolrRequest.METHOD.POST); @@ -132,4 +135,4 @@ static synchronized HttpClient getHttpClient() { return httpClient; } } -} \ No newline at end of file +} diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java index 953a635297aa..b22feabba2ce 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java @@ -326,7 +326,7 @@ public SolrParams getParams() { @Override public Integer call() throws Exception { int remoteVersion = -1; - try (HttpSolrClient solr = new HttpSolrClient(coreUrl)) { + try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) { // eventually, this loop will get killed by the ExecutorService's timeout while (remoteVersion == -1 || remoteVersion < expectedZkVersion) { try { diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index 9a196b7ca407..c161b82dd00e 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -44,7 +44,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; @@ -260,7 +259,7 @@ private void submit(final Req req, boolean isCommit) { if (req.synchronous) { blockAndDoRetries(); - try (HttpSolrClient client = new HttpSolrClient(req.node.getUrl(), clients.getHttpClient())) { + try (HttpSolrClient client = new HttpSolrClient.Builder(req.node.getUrl()).withHttpClient(clients.getHttpClient()).build()) { client.request(req.uReq); } catch (Exception e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Failed synchronous update on shard " + req.node + " update: " + req.uReq , e); diff --git a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java index ecc2a2bb29e4..fc50be21c003 100644 --- a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java +++ b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java @@ -16,6 +16,16 @@ */ package org.apache.solr.update; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; + import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrClient; @@ -29,16 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutorService; - public class StreamingSolrClients { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -72,24 +72,7 @@ public synchronized SolrClient getSolrClient(final SolrCmdDistributor.Req req) { // NOTE: increasing to more than 1 threadCount for the client could cause updates to be reordered // on a greater scale since the current behavior is to only increase the number of connections/Runners when // the queue is more than half full. - client = new ConcurrentUpdateSolrClient(url, httpClient, 100, runnerCount, updateExecutor, true) { - @Override - public void handleError(Throwable ex) { - req.trackRequestResult(null, false); - log.error("error", ex); - Error error = new Error(); - error.e = (Exception) ex; - if (ex instanceof SolrException) { - error.statusCode = ((SolrException) ex).code(); - } - error.req = req; - errors.add(error); - } - @Override - public void onSuccess(HttpResponse resp) { - req.trackRequestResult(resp, true); - } - }; + client = new ErrorReportingConcurrentUpdateSolrClient(url, httpClient, 100, runnerCount, updateExecutor, true, req); client.setParser(new BinaryResponseParser()); client.setRequestWriter(new BinaryRequestWriter()); client.setPollQueueTime(req.pollQueueTime); @@ -132,4 +115,31 @@ public HttpClient getHttpClient() { public ExecutorService getUpdateExecutor() { return updateExecutor; } + + class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient { + private final SolrCmdDistributor.Req req; + + public ErrorReportingConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, + int threadCount, ExecutorService es, boolean streamDeletes, SolrCmdDistributor.Req req) { + super(solrServerUrl, client, queueSize, threadCount, es, streamDeletes); + this.req = req; + } + + @Override + public void handleError(Throwable ex) { + req.trackRequestResult(null, false); + log.error("error", ex); + Error error = new Error(); + error.e = (Exception) ex; + if (ex instanceof SolrException) { + error.statusCode = ((SolrException) ex).code(); + } + error.req = req; + errors.add(error); + } + @Override + public void onSuccess(HttpResponse resp) { + req.trackRequestResult(resp, true); + } + } } diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index 19aa52ac1843..75a39b2a27cb 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -82,6 +82,7 @@ import org.apache.solr.client.solrj.impl.HttpClientConfigurer; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; @@ -210,7 +211,7 @@ protected void runImpl(CommandLine cli) throws Exception { String zkHost = cli.getOptionValue("zkHost", ZK_HOST); log.debug("Connecting to Solr cluster: " + zkHost); - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { String collection = cli.getOptionValue("collection"); if (collection != null) @@ -1163,7 +1164,7 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th q = new SolrQuery("*:*"); q.setRows(0); q.set("distrib", "false"); - try (HttpSolrClient solr = new HttpSolrClient(coreUrl)) { + try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) { String solrUrl = solr.getBaseURL(); @@ -1286,7 +1287,7 @@ public static String resolveSolrUrl(CommandLine cli) throws Exception { if (zkHost == null) throw new IllegalStateException("Must provide either the '-solrUrl' or '-zkHost' parameters!"); - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { cloudSolrClient.connect(); Set liveNodes = cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes(); if (liveNodes.isEmpty()) @@ -1399,7 +1400,7 @@ protected void runImpl(CommandLine cli) throws Exception { "create_collection can only be used when running in SolrCloud mode.\n"); } - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { echo("\nConnecting to ZooKeeper at " + zkHost+" ..."); cloudSolrClient.connect(); runCloudTool(cloudSolrClient, cli); @@ -1708,7 +1709,7 @@ protected void runImpl(CommandLine cli) throws Exception { " is running in standalone server mode, upconfig can only be used when running in SolrCloud mode.\n"); } - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { echo("\nConnecting to ZooKeeper at " + zkHost + " ..."); cloudSolrClient.connect(); upconfig(cloudSolrClient, cli, cli.getOptionValue("confname"), cli.getOptionValue("confdir")); @@ -1763,7 +1764,7 @@ protected void runImpl(CommandLine cli) throws Exception { } - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { echo("\nConnecting to ZooKeeper at " + zkHost + " ..."); cloudSolrClient.connect(); downconfig(cloudSolrClient, cli.getOptionValue("confname"), cli.getOptionValue("confdir")); @@ -1854,7 +1855,7 @@ protected void runImpl(CommandLine cli) throws Exception { protected void deleteCollection(CommandLine cli) throws Exception { String zkHost = getZkHost(cli); - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { echo("Connecting to ZooKeeper at " + zkHost); cloudSolrClient.connect(); deleteCollection(cloudSolrClient, cli); @@ -2031,7 +2032,7 @@ protected void runImpl(CommandLine cli) throws Exception { echo("\nPOSTing request to Config API: " + solrUrl + updatePath); echo(jsonBody); - try (SolrClient solrClient = new HttpSolrClient(solrUrl)) { + try (SolrClient solrClient = new Builder(solrUrl).build()) { NamedList result = postJsonToSolr(solrClient, updatePath, jsonBody); Integer statusCode = (Integer)((NamedList)result.get("responseHeader")).get("status"); if (statusCode == 0) { @@ -2426,7 +2427,9 @@ protected void setCollectionConfigProperty(String solrUrl, String collectionName protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) { CloudSolrClient cloudClient = null; try { - cloudClient = new CloudSolrClient(zkHost); + cloudClient = new CloudSolrClient.Builder() + .withZkHost(zkHost) + .build(); cloudClient.connect(); Set liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java index 1d7dd2be66c4..cb485d0ca817 100644 --- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java +++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java @@ -59,8 +59,8 @@ public static void createThings() throws Exception { solrHome = createSolrHome(); createJetty(solrHome.getAbsolutePath()); String url = jetty.getBaseUrl().toString(); - collection1 = new HttpSolrClient(url + "/collection1"); - collection2 = new HttpSolrClient(url + "/collection2"); + collection1 = getHttpSolrClient(url + "/collection1"); + collection2 = getHttpSolrClient(url + "/collection2"); String urlCollection1 = jetty.getBaseUrl().toString() + "/" + "collection1"; String urlCollection2 = jetty.getBaseUrl().toString() + "/" + "collection2"; @@ -68,7 +68,7 @@ public static void createThings() throws Exception { shard2 = urlCollection2.replaceAll("https?://", ""); //create second core - try (HttpSolrClient nodeClient = new HttpSolrClient(url)) { + try (HttpSolrClient nodeClient = getHttpSolrClient(url)) { CoreAdminRequest.Create req = new CoreAdminRequest.Create(); req.setCoreName("collection2"); req.setConfigSet("collection1"); diff --git a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java b/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java index 0ec9876b61f3..05f1157c5246 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java @@ -80,11 +80,11 @@ public void test() throws Exception { HttpClient httpClient = HttpClientUtil.createClient(null); int rndClient = random().nextInt(3); if (rndClient == 0) { - client = new ConcurrentUpdateSolrClient(url.toString(), httpClient, 6, 1); // currently only testing with 1 thread + client = getConcurrentUpdateSolrClient(url.toString(), httpClient, 6, 1); // currently only testing with 1 thread } else if (rndClient == 1) { - client = new HttpSolrClient(url.toString(), httpClient); + client = getHttpSolrClient(url.toString(), httpClient); } else if (rndClient == 2) { - client = new CloudSolrClient(zkServer.getZkAddress(), random().nextBoolean(), httpClient); + client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean(), httpClient); ((CloudSolrClient) client).setParallelUpdates(random().nextBoolean()); ((CloudSolrClient) client).setDefaultCollection(DEFAULT_COLLECTION); ((CloudSolrClient) client).getLbClient().setConnectionTimeout(30000); diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestJettySolrRunner.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestJettySolrRunner.java index 17facec72418..688996c23e9c 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestJettySolrRunner.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestJettySolrRunner.java @@ -19,7 +19,6 @@ import com.google.common.base.Charsets; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.junit.Test; @@ -53,7 +52,7 @@ public void testPassSolrHomeToRunner() throws Exception { try { runner.start(); - SolrClient client = new HttpSolrClient(runner.getBaseUrl().toString()); + SolrClient client = getHttpSolrClient(runner.getBaseUrl().toString()); CoreAdminRequest.Create createReq = new CoreAdminRequest.Create(); createReq.setCoreName("newcore"); diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java index a270a108141f..a2fcbd4bb7bc 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java @@ -16,6 +16,11 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; + import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; @@ -32,16 +37,9 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.ModifiableSolrParams; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.junit.Test; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; /** * Test sync phase that occurs when Leader goes down and a new Leader is @@ -119,7 +117,7 @@ public void test() throws Exception { query.set("collection", "testalias"); JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size())); int port = jetty.getLocalPort(); - try (HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { res = client.query(query); assertEquals(3, res.getResults().getNumFound()); } @@ -128,7 +126,7 @@ public void test() throws Exception { query = new SolrQuery("*:*"); jetty = jettys.get(random().nextInt(jettys.size())); port = jetty.getLocalPort(); - try (HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { res = client.query(query); assertEquals(3, res.getResults().getNumFound()); } @@ -136,7 +134,7 @@ public void test() throws Exception { createAlias("testalias", "collection2,collection1"); // search with new cloud client - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkServer.getZkAddress(), random().nextBoolean())) { + try (CloudSolrClient cloudSolrClient = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean())) { cloudSolrClient.setParallelUpdates(random().nextBoolean()); query = new SolrQuery("*:*"); query.set("collection", "testalias"); @@ -155,7 +153,7 @@ public void test() throws Exception { query.set("collection", "testalias"); jetty = jettys.get(random().nextInt(jettys.size())); port = jetty.getLocalPort(); - try (HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { res = client.query(query); assertEquals(5, res.getResults().getNumFound()); } @@ -163,7 +161,7 @@ public void test() throws Exception { query = new SolrQuery("*:*"); jetty = jettys.get(random().nextInt(jettys.size())); port = jetty.getLocalPort(); - try (HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { res = client.query(query); assertEquals(5, res.getResults().getNumFound()); } @@ -190,7 +188,7 @@ public void test() throws Exception { // try a std client // search 1 and 2, but have no collections param query = new SolrQuery("*:*"); - try (HttpSolrClient client = new HttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { res = client.query(query); assertEquals(5, res.getResults().getNumFound()); } @@ -200,7 +198,7 @@ public void test() throws Exception { // a second alias createAlias("testalias2", "collection2"); - try (HttpSolrClient client = new HttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { SolrInputDocument doc8 = getDoc(id, 11, i1, -600, tlong, 600, t1, "humpty dumpy4 sat on a walls"); client.add(doc8); diff --git a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java index b9e6d3a090d2..f6e4b338c611 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.cloud; +import java.io.IOException; + import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -27,8 +29,6 @@ import org.apache.solr.common.util.NamedList; import org.junit.Test; -import java.io.IOException; - public class AsyncMigrateRouteKeyTest extends MigrateRouteKeyTest { public AsyncMigrateRouteKeyTest() { @@ -113,7 +113,7 @@ protected NamedList sendRequest(ModifiableSolrParams params) throws SolrServerEx String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient).getBaseURL(); baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - try (HttpSolrClient baseServer = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient baseServer = getHttpSolrClient(baseUrl)) { baseServer.setConnectionTimeout(15000); return baseServer.request(request); } diff --git a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java index fe94309bba26..fcd9abb52379 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java @@ -49,7 +49,6 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; @@ -172,7 +171,7 @@ public void baseAfter() throws Exception { } protected CloudSolrClient createCloudClient(String defaultCollection) { - CloudSolrClient server = new CloudSolrClient(zkServer.getZkAddress(), random().nextBoolean()); + CloudSolrClient server = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean()); server.setParallelUpdates(random().nextBoolean()); if (defaultCollection != null) server.setDefaultCollection(defaultCollection); server.getLbClient().getHttpClient().getParams() @@ -745,7 +744,7 @@ public String toString() { protected static SolrClient createNewSolrServer(String baseUrl) { try { // setup the server... - HttpSolrClient s = new HttpSolrClient(baseUrl); + HttpSolrClient s = getHttpSolrClient(baseUrl); s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); s.setDefaultMaxConnectionsPerHost(100); s.setMaxTotalConnections(100); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index a04250146315..61b9901f9469 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -147,7 +147,7 @@ public void test() throws Exception { private void testNodeWithoutCollectionForwarding() throws Exception { final String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0)); - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl)) { client.setConnectionTimeout(30000); Create createCmd = new Create(); createCmd.setRoles("none"); @@ -180,7 +180,7 @@ private void addAndQueryDocs(final String baseUrl, int docs) SolrQuery query = new SolrQuery("*:*"); - try (HttpSolrClient qclient = new HttpSolrClient(baseUrl + "/onenodecollection" + "core")) { + try (HttpSolrClient qclient = getHttpSolrClient(baseUrl + "/onenodecollection" + "core")) { // it might take a moment for the proxy node to see us in their cloud state waitForNon403or404or503(qclient); @@ -196,7 +196,7 @@ private void addAndQueryDocs(final String baseUrl, int docs) assertEquals(docs - 1, results.getResults().getNumFound()); } - try (HttpSolrClient qclient = new HttpSolrClient(baseUrl + "/onenodecollection")) { + try (HttpSolrClient qclient = getHttpSolrClient(baseUrl + "/onenodecollection")) { QueryResponse results = qclient.query(query); assertEquals(docs - 1, results.getResults().getNumFound()); diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 4a6b2d7af6b6..e9b8ff757ed5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -404,7 +404,7 @@ private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerExcepti for (Slice slice : dColl.getActiveSlices()) { long sliceDocCount = -1; for (Replica rep : slice.getReplicas()) { - HttpSolrClient one = new HttpSolrClient(rep.getCoreUrl()); + HttpSolrClient one = getHttpSolrClient(rep.getCoreUrl()); SolrQuery query = new SolrQuery("*:*"); query.setDistrib(false); QueryResponse resp = one.query(query); @@ -528,7 +528,7 @@ private void testShardParamVariations() throws Exception { private void testStopAndStartCoresInOneInstance() throws Exception { SolrClient client = clients.get(0); String url3 = getBaseUrl(client); - try (final HttpSolrClient httpSolrClient = new HttpSolrClient(url3)) { + try (final HttpSolrClient httpSolrClient = getHttpSolrClient(url3)) { httpSolrClient.setConnectionTimeout(15000); httpSolrClient.setSoTimeout(60000); ThreadPoolExecutor executor = null; @@ -749,7 +749,7 @@ private void testNumberOfCommitsWithCommitAfterAdd() private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException { - try (HttpSolrClient client = new HttpSolrClient(sourceClient.getBaseURL())) { + try (HttpSolrClient client = getHttpSolrClient(sourceClient.getBaseURL())) { client.setConnectionTimeout(15000); client.setSoTimeout(60000); ModifiableSolrParams params = new ModifiableSolrParams(); @@ -840,7 +840,7 @@ private void testANewCollectionInOneInstanceWithManualShardAssignement() throws ZkCoreNodeProps props = new ZkCoreNodeProps(getCommonCloudSolrClient().getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1")); // now test that unloading a core gets us a new leader - try (HttpSolrClient unloadClient = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient unloadClient = getHttpSolrClient(baseUrl)) { unloadClient.setConnectionTimeout(15000); unloadClient.setSoTimeout(60000); Unload unloadCmd = new Unload(true); @@ -960,7 +960,7 @@ private void createSolrCore(final String collection, List collectionClients, final String baseUrl, final int num, final String shardId) { Callable call = () -> { - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl)) { client.setConnectionTimeout(15000); Create createCmd = new Create(); createCmd.setRoles("none"); @@ -1089,7 +1089,7 @@ private void createNewCollection(final String collection) throws InterruptedExce final int frozeUnique = unique; Callable call = () -> { - try (HttpSolrClient client1 = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client1 = getHttpSolrClient(baseUrl)) { client1.setConnectionTimeout(15000); client1.setSoTimeout(60000); Create createCmd = new Create(); @@ -1117,10 +1117,11 @@ private void createNewCollection(final String collection) throws InterruptedExce protected SolrClient createNewSolrClient(String collection, String baseUrl) { try { // setup the server... - HttpSolrClient client = new HttpSolrClient(baseUrl + "/" + collection); + HttpSolrClient client = getHttpSolrClient(baseUrl + "/" + collection); client.setSoTimeout(120000); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); + return client; } catch (Exception ex) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 7dceada16687..09e3adcb267c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -16,7 +16,15 @@ */ package org.apache.solr.cloud; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import java.lang.invoke.MethodHandles; +import java.net.ConnectException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.http.client.HttpClient; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4.SuppressObjectReleaseTracker; @@ -36,13 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.net.ConnectException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; @Slow @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") @@ -295,14 +297,10 @@ public FullThrottleStoppableIndexingThread(List clients, this.clients = clients; HttpClientUtil.setConnectionTimeout(httpClient, clientConnectionTimeout); HttpClientUtil.setSoTimeout(httpClient, clientSoTimeout); - cusc = new ConcurrentUpdateSolrClient( - ((HttpSolrClient) clients.get(0)).getBaseURL(), httpClient, 8, - 2) { - @Override - public void handleError(Throwable ex) { - log.warn("cusc error", ex); - } - }; + + cusc = new ErrorLoggingConcurrentUpdateSolrClient(((HttpSolrClient) clients.get(0)).getBaseURL(), httpClient, 8, 2); + cusc.setConnectionTimeout(10000); + cusc.setSoTimeout(clientSoTimeout); } @Override @@ -359,14 +357,8 @@ private void changeUrlOnError(Exception e) { clientIndex = 0; } cusc.shutdownNow(); - cusc = new ConcurrentUpdateSolrClient( - ((HttpSolrClient) clients.get(clientIndex)).getBaseURL(), - httpClient, 30, 3) { - @Override - public void handleError(Throwable ex) { - log.warn("cusc error", ex); - } - }; + cusc = new ErrorLoggingConcurrentUpdateSolrClient(((HttpSolrClient) clients.get(clientIndex)).getBaseURL(), + httpClient, 30, 3); } } @@ -403,4 +395,13 @@ protected void indexr(Object... fields) throws Exception { indexDoc(doc); } + class ErrorLoggingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient { + public ErrorLoggingConcurrentUpdateSolrClient(String serverUrl, HttpClient httpClient, int queueSize, int threadCount) { + super(serverUrl, httpClient, queueSize, threadCount, null, false); + } + @Override + public void handleError(Throwable ex) { + log.warn("cusc error", ex); + } + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java index 65ff78bf06a1..cf8a65013c25 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java @@ -131,7 +131,7 @@ protected boolean reloadCollection(Replica replica, String testCollectionName) t ZkCoreNodeProps coreProps = new ZkCoreNodeProps(replica); String coreName = coreProps.getCoreName(); boolean reloadedOk = false; - try (HttpSolrClient client = new HttpSolrClient(coreProps.getBaseUrl())) { + try (HttpSolrClient client = getHttpSolrClient(coreProps.getBaseUrl())) { CoreAdminResponse statusResp = CoreAdminRequest.getStatus(coreName, client); long leaderCoreStartTime = statusResp.getStartTime(coreName).getTime(); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java index 641dadfc236f..702dedb155aa 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java @@ -722,7 +722,7 @@ private void testCollectionsAPI() throws Exception { String url = getUrlFromZk(collection); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } @@ -742,7 +742,7 @@ private void testCollectionsAPI() throws Exception { String url = getUrlFromZk(collection); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } @@ -789,7 +789,7 @@ private void testCollectionsAPI() throws Exception { String url = getUrlFromZk(collectionName); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // lets try and use the solrj client to index a couple documents SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1, @@ -886,7 +886,7 @@ private void testCollectionsAPI() throws Exception { url = getUrlFromZk(collectionName); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } @@ -1061,7 +1061,7 @@ private void collectStartTimes(String collectionName, Entry shardEntry = shardIt.next(); ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue()); CoreAdminResponse mcr; - try (HttpSolrClient server = new HttpSolrClient(coreProps.getBaseUrl())) { + try (HttpSolrClient server = getHttpSolrClient(coreProps.getBaseUrl())) { mcr = CoreAdminRequest.getStatus(coreProps.getCoreName(), server); } long before = mcr.getStartTime(coreProps.getCoreName()).getTime(); @@ -1196,7 +1196,7 @@ private void addReplicaTest() throws Exception { null, client, props); assertNotNull(newReplica); - HttpSolrClient coreclient = new HttpSolrClient(newReplica.getStr(ZkStateReader.BASE_URL_PROP)); + HttpSolrClient coreclient = getHttpSolrClient(newReplica.getStr(ZkStateReader.BASE_URL_PROP)); CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient); NamedList coreStatus = status.getCoreStatus(newReplica.getStr("core")); String instanceDirStr = (String) coreStatus.get("instanceDir"); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index 6b0db0f151f9..52ae96f0487f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -257,7 +257,7 @@ private void testCreateCollectionWithPropertyParam() throws Exception { Replica replica1 = testCollection.getReplica("core_node1"); - try (HttpSolrClient client = new HttpSolrClient(replica1.getStr("base_url"))) { + try (HttpSolrClient client = getHttpSolrClient(replica1.getStr("base_url"))) { CoreAdminResponse status = CoreAdminRequest.getStatus(replica1.getStr("core"), client); NamedList coreStatus = status.getCoreStatus(replica1.getStr("core")); String dataDirStr = (String) coreStatus.get("dataDir"); diff --git a/solr/core/src/test/org/apache/solr/cloud/ConcurrentDeleteAndCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/ConcurrentDeleteAndCreateCollectionTest.java index 82bf9d4a4aa2..c8e30eb69173 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ConcurrentDeleteAndCreateCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ConcurrentDeleteAndCreateCollectionTest.java @@ -60,7 +60,8 @@ public void testConcurrentCreateAndDeleteDoesNotFail() { for (int i = 0; i < threads.length; i++) { final String collectionName = "collection" + i; uploadConfig(configDir, collectionName); - final SolrClient solrClient = new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); threads[i] = new CreateDeleteSearchCollectionThread("create-delete-search-" + i, collectionName, collectionName, timeToRunSec, solrClient, failure); } @@ -75,7 +76,8 @@ public void testConcurrentCreateAndDeleteOverTheSameConfig() { final String configName = "testconfig"; final File configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf").toFile(); uploadConfig(configDir, configName); // upload config once, to be used by all collections - final SolrClient solrClient = new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); final AtomicReference failure = new AtomicReference<>(); final int timeToRunSec = 30; final Thread[] threads = new Thread[2]; diff --git a/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java index 0951b5d6fc02..0873fa3b7836 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java @@ -167,7 +167,7 @@ private void testCustomCollectionsAPI() throws Exception { String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collection); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } @@ -195,7 +195,7 @@ private void testCustomCollectionsAPI() throws Exception { String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName); String shard_fld = "shard_s"; - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // lets try and use the solrj client to index a couple documents @@ -292,12 +292,12 @@ private void testCustomCollectionsAPI() throws Exception { url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName); } - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // lets try and use the solrj client to index a couple documents collectionClient.add(getDoc(id, 6, i1, -600, tlong, 600, t1, @@ -343,13 +343,13 @@ private void testRouteFieldForHashRouter()throws Exception{ String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // poll for a second - it can take a moment before we are ready to serve waitForNon403or404or503(collectionClient); } - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { // lets try and use the solrj client to index a couple documents collectionClient.add(getDoc(id, 6, i1, -600, tlong, 600, t1, diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java index 959d3b216809..18a5f9cf6bbf 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java @@ -128,7 +128,7 @@ public void deleteInactiveReplicaTest() throws Exception { Map m = Utils.makeMap("qt", "/admin/cores", "action", "status"); - try (SolrClient queryClient = new HttpSolrClient(replica1.getStr(ZkStateReader.BASE_URL_PROP))) { + try (SolrClient queryClient = getHttpSolrClient(replica1.getStr(ZkStateReader.BASE_URL_PROP))) { NamedList resp = queryClient.request(new QueryRequest(new MapSolrParams(m))); assertNull("The core is up and running again", ((NamedList) resp.get("status")).get(replica1.getStr("core"))); diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java index f0d648402787..bb8d3beb155f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java @@ -97,7 +97,7 @@ public void deleteLiveReplicaTest() throws Exception { if (replica1 == null) fail("no active replicas found"); String dataDir = null; - try (HttpSolrClient replica1Client = new HttpSolrClient(replica1.getStr("base_url"))) { + try (HttpSolrClient replica1Client = getHttpSolrClient(replica1.getStr("base_url"))) { CoreAdminResponse status = CoreAdminRequest.getStatus(replica1.getStr("core"), replica1Client); NamedList coreStatus = status.getCoreStatus(replica1.getStr("core")); dataDir = (String) coreStatus.get("dataDir"); @@ -186,7 +186,7 @@ public void deleteReplicaAndVerifyDirectoryCleanup() throws IOException, SolrSer String instanceDir; String dataDir; - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl)) { CoreAdminResponse statusResp = CoreAdminRequest.getStatus(core, client); NamedList r = statusResp.getCoreStatus().get(core); instanceDir = (String) r.findRecursive("instanceDir"); diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java index 812fbe932187..90973636c2fc 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java @@ -121,7 +121,7 @@ protected void deleteShard(String shard) throws SolrServerException, IOException .getBaseURL(); baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - try (HttpSolrClient baseServer = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient baseServer = getHttpSolrClient(baseUrl)) { baseServer.setConnectionTimeout(15000); baseServer.setSoTimeout(60000); baseServer.request(request); @@ -173,7 +173,7 @@ public void testDirectoryCleanupAfterDeleteShard() throws InterruptedException, String instanceDir; String dataDir; - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl)) { CoreAdminResponse statusResp = CoreAdminRequest.getStatus(core, client); NamedList r = statusResp.getCoreStatus().get(core); instanceDir = (String) r.findRecursive("instanceDir"); @@ -199,7 +199,7 @@ public void testDirectoryCleanupAfterDeleteShard() throws InterruptedException, baseUrl = (String) leader.get("base_url"); core = (String) leader.get("core"); - try (HttpSolrClient client = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl)) { CoreAdminResponse statusResp = CoreAdminRequest.getStatus(core, client); NamedList r = statusResp.getCoreStatus().get(core); instanceDir = (String) r.findRecursive("instanceDir"); diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java index 898f38a1874d..812815b01738 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java @@ -105,7 +105,7 @@ protected void testReplicaVersionHandling() throws Exception { assertEquals("leader and replica should have same max version: " + maxOnLeader, maxOnLeader, maxOnReplica); // send the same doc but with a lower version than the max in the index - try (SolrClient client = new HttpSolrClient(replica.getCoreUrl())) { + try (SolrClient client = getHttpSolrClient(replica.getCoreUrl())) { String docId = String.valueOf(1); SolrInputDocument doc = new SolrInputDocument(); doc.setField(id, docId); @@ -276,7 +276,7 @@ protected long getVersionFromIndex(Replica replica, String docId) throws IOExcep query.addSort(new SolrQuery.SortClause("_version_", SolrQuery.ORDER.desc)); query.setParam("distrib", false); - try (SolrClient client = new HttpSolrClient(replica.getCoreUrl())) { + try (SolrClient client = getHttpSolrClient(replica.getCoreUrl())) { QueryResponse qr = client.query(query); SolrDocumentList hits = qr.getResults(); if (hits.isEmpty()) @@ -343,7 +343,7 @@ protected void assertDocsExistInAllReplicas(List notLeaders, } protected HttpSolrClient getHttpSolrClient(Replica replica) throws Exception { - return new HttpSolrClient(replica.getCoreUrl()); + return getHttpSolrClient(replica.getCoreUrl()); } protected void sendDoc(int docId) throws Exception { @@ -378,7 +378,7 @@ protected boolean reloadCollection(Replica replica, String testCollectionName) t ZkCoreNodeProps coreProps = new ZkCoreNodeProps(replica); String coreName = coreProps.getCoreName(); boolean reloadedOk = false; - try (HttpSolrClient client = new HttpSolrClient(coreProps.getBaseUrl())) { + try (HttpSolrClient client = getHttpSolrClient(coreProps.getBaseUrl())) { CoreAdminResponse statusResp = CoreAdminRequest.getStatus(coreName, client); long leaderCoreStartTime = statusResp.getStartTime(coreName).getTime(); diff --git a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java index a71c3e614131..e9e8907affaf 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java @@ -301,7 +301,7 @@ protected Replica.State getLastPublishedState(String collection, String slice, R SolrRequest req = new GenericSolrRequest(METHOD.GET, "/admin/cores", params); NamedList resp = null; - try (HttpSolrClient hsc = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient hsc = getHttpSolrClient(baseUrl)) { resp = hsc.request(req); } diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java index 26099142d85b..e15ab0d4cbab 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java @@ -698,7 +698,7 @@ private long testConcurrentIndexing(long docId) throws Exception { QueryResponse results = query(cloudClient); long beforeCount = results.getResults().getNumFound(); int cnt = TEST_NIGHTLY ? 2933 : 313; - try (ConcurrentUpdateSolrClient concurrentClient = new ConcurrentUpdateSolrClient( + try (ConcurrentUpdateSolrClient concurrentClient = getConcurrentUpdateSolrClient( ((HttpSolrClient) clients.get(0)).getBaseURL(), 10, 2)) { concurrentClient.setConnectionTimeout(120000); for (int i = 0; i < cnt; i++) { diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java index f1960aa952ea..7e2c17d91a3b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java @@ -588,7 +588,7 @@ protected void assertDocsExistInAllReplicas(List notLeaders, protected HttpSolrClient getHttpSolrClient(Replica replica, String coll) throws Exception { ZkCoreNodeProps zkProps = new ZkCoreNodeProps(replica); String url = zkProps.getBaseUrl() + "/" + coll; - return new HttpSolrClient(url); + return getHttpSolrClient(url); } protected int sendDoc(int docId) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java index 7d6c633f4827..fd122ad6a086 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.java @@ -163,7 +163,7 @@ protected void sendCommitWithRetry(Replica replica) throws Exception { String replicaCoreUrl = replica.getCoreUrl(); log.info("Sending commit request to: "+replicaCoreUrl); final RTimer timer = new RTimer(); - try (HttpSolrClient client = new HttpSolrClient(replicaCoreUrl)) { + try (HttpSolrClient client = getHttpSolrClient(replicaCoreUrl)) { try { client.commit(); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnShardRestartTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnShardRestartTest.java index d130f3bbcd79..928f4d618388 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnShardRestartTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnShardRestartTest.java @@ -80,7 +80,7 @@ public void testRestartWithAllInLIR() throws Exception { QueryRequest request = new QueryRequest(params); request.setPath("/admin/collections"); String baseUrl = ((HttpSolrClient) clients.get(0)).getBaseURL(); - HttpSolrClient delClient = new HttpSolrClient(baseUrl.substring(0, baseUrl.lastIndexOf("/"))); + HttpSolrClient delClient = getHttpSolrClient(baseUrl.substring(0, baseUrl.lastIndexOf("/"))); delClient.request(request); delClient.close(); diff --git a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java index c09e0d1dc6a0..fadb880b8b6f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java @@ -106,7 +106,7 @@ protected void invoke(ModifiableSolrParams params) throws SolrServerException, I .getBaseURL(); baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - try (HttpSolrClient baseClient = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient baseClient = getHttpSolrClient(baseUrl)) { baseClient.setConnectionTimeout(15000); baseClient.setSoTimeout(60000 * 5); baseClient.request(request); @@ -161,7 +161,7 @@ protected void multipleShardMigrateTest() throws Exception { String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), targetCollection); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { SolrQuery solrQuery = new SolrQuery("*:*"); assertEquals("DocCount on target collection does not match", 0, collectionClient.query(solrQuery).getResults().getNumFound()); diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java index 894f26d70f44..a4fbba186c3e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java @@ -179,7 +179,7 @@ protected void sendNonDirectUpdateRequestReplica(Replica replica, UpdateRequest ZkCoreNodeProps zkProps = new ZkCoreNodeProps(replica); String url = zkProps.getBaseUrl() + "/" + collection; - try (HttpSolrClient solrServer = new HttpSolrClient(url)) { + try (HttpSolrClient solrServer = getHttpSolrClient(url)) { NamedList resp = solrServer.request(up); NamedList hdr = (NamedList) resp.get("responseHeader"); Integer batchRf = (Integer)hdr.get(UpdateRequest.REPFACT); diff --git a/solr/core/src/test/org/apache/solr/cloud/SSLMigrationTest.java b/solr/core/src/test/org/apache/solr/cloud/SSLMigrationTest.java index a0bb08f822af..8a72adb7effe 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SSLMigrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SSLMigrationTest.java @@ -130,7 +130,7 @@ private void setUrlScheme(String value) throws Exception { urls.add(replica.getStr(ZkStateReader.BASE_URL_PROP)); } //Create new SolrServer to configure new HttpClient w/ SSL config - new LBHttpSolrClient(urls.toArray(new String[]{})).request(request); + getLBHttpSolrClient(urls.toArray(new String[]{})).request(request); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java index 6d4b9cc3b8f9..21dc2577066a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java @@ -16,6 +16,16 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + import org.apache.http.params.CoreConnectionPNames; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrClient; @@ -44,16 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; - import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; @@ -274,7 +274,7 @@ public void splitByRouteFieldTest() throws Exception { String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); final DocRouter router = clusterState.getCollection(collectionName).getRouter(); @@ -348,7 +348,7 @@ private void splitByRouteKeyTest() throws Exception { String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName); - try (HttpSolrClient collectionClient = new HttpSolrClient(url)) { + try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { String splitKey = "b!"; @@ -445,7 +445,7 @@ protected void checkDocCountsAndShardStates(int[] docCounts, int numReplicas) th ZkCoreNodeProps shard1_0 = getLeaderUrlFromZk(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_0); QueryResponse response; - try (HttpSolrClient shard1_0Client = new HttpSolrClient(shard1_0.getCoreUrl())) { + try (HttpSolrClient shard1_0Client = getHttpSolrClient(shard1_0.getCoreUrl())) { response = shard1_0Client.query(query); } long shard10Count = response.getResults().getNumFound(); @@ -453,7 +453,7 @@ protected void checkDocCountsAndShardStates(int[] docCounts, int numReplicas) th ZkCoreNodeProps shard1_1 = getLeaderUrlFromZk( AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_1); QueryResponse response2; - try (HttpSolrClient shard1_1Client = new HttpSolrClient(shard1_1.getCoreUrl())) { + try (HttpSolrClient shard1_1Client = getHttpSolrClient(shard1_1.getCoreUrl())) { response2 = shard1_1Client.query(query); } long shard11Count = response2.getResults().getNumFound(); @@ -475,7 +475,7 @@ protected void checkSubShardConsistency(String shard) throws SolrServerException for (Replica replica : slice.getReplicas()) { String coreUrl = new ZkCoreNodeProps(replica).getCoreUrl(); QueryResponse response; - try (HttpSolrClient client = new HttpSolrClient(coreUrl)) { + try (HttpSolrClient client = getHttpSolrClient(coreUrl)) { response = client.query(query); } numFound[c++] = response.getResults().getNumFound(); @@ -514,7 +514,7 @@ protected void splitShard(String collection, String shardId, List ol private void verifyCreate(String baseConfigSetName, String configSetName, Map oldProps, Map newProps) throws Exception { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); setupBaseConfigSet(baseConfigSetName, oldProps); SolrZkClient zkClient = new SolrZkClient(solrCluster.getZkServer().getZkAddress(), @@ -235,8 +235,8 @@ private void verifyProperties(String configSetName, Map oldProps @Test public void testDeleteErrors() throws Exception { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); final File configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf").toFile(); final File tmpConfigDir = createTempDir().toFile(); tmpConfigDir.deleteOnExit(); @@ -274,8 +274,8 @@ private void verifyException(SolrClient solrClient, ConfigSetAdminRequest reques @Test public void testDelete() throws Exception { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); final File configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf").toFile(); final String configSet = "configSet"; solrCluster.uploadConfigDir(configDir, configSet); @@ -300,8 +300,8 @@ public void testDelete() throws Exception { @Test public void testList() throws Exception { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); SolrZkClient zkClient = new SolrZkClient(solrCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT, null); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java index 2d2aa58853de..c5a0bc574a01 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java @@ -121,8 +121,8 @@ public ConfigSetsAPIThread(MiniSolrCloudCluster solrCluster, int trials) { public abstract ConfigSetAdminRequest createRequest(); public void run() { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); ConfigSetAdminRequest request = createRequest(); for (int i = 0; i < trials; ++i) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java index 9e5989adaf19..458d68a1fa94 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java @@ -99,8 +99,8 @@ public void tearDown() throws Exception { @Test public void testCreateZkFailure() throws Exception { - final SolrClient solrClient = - new HttpSolrClient(solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); final Map oldProps = ImmutableMap.of("immutable", "true"); setupBaseConfigSet(BASE_CONFIGSET_NAME, oldProps); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java index cdbc170b8edf..a8e206376dbd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java @@ -119,7 +119,7 @@ public void test() throws Exception { String baseURL = randomClient.getBaseURL(); baseURL = baseURL.substring(0, baseURL.lastIndexOf('/')); - TestBlobHandler.createSystemCollection(new HttpSolrClient(baseURL, randomClient.getHttpClient())); + TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient())); waitForRecoveriesToFinish(".system", true); ByteBuffer jar = TestDynamicLoading.getFileContent("runtimecode/runtimelibs.jar.bin"); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java index bc4352f731f4..b82ba743e581 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java @@ -74,7 +74,7 @@ public void testMiniSolrCloudClusterSSL() throws Exception { private void sendRequestToEachServer() throws Exception { List jettys = miniCluster.getJettySolrRunners(); for (JettySolrRunner jetty : jettys) { - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString())) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString())) { CoreAdminRequest req = new CoreAdminRequest(); req.setAction( CoreAdminAction.STATUS ); client.request(req); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java index 256774d08c39..f13094418ee0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java @@ -96,7 +96,7 @@ private void testRequestTracking() throws Exception { assertEquals(1, replicas.size()); String baseUrl = replicas.iterator().next().getStr(ZkStateReader.BASE_URL_PROP); if (!baseUrl.endsWith("/")) baseUrl += "/"; - HttpSolrClient client = new HttpSolrClient(baseUrl + "a1x2"); + HttpSolrClient client = getHttpSolrClient(baseUrl + "a1x2"); client.setSoTimeout(5000); client.setConnectionTimeout(2000); @@ -173,7 +173,7 @@ private void testQueryAgainstDownReplica() throws Exception { if (!baseUrl.endsWith("/")) baseUrl += "/"; String path = baseUrl + "football"; log.info("Firing queries against path=" + path); - HttpSolrClient client = new HttpSolrClient(path); + HttpSolrClient client = getHttpSolrClient(path); client.setSoTimeout(5000); client.setConnectionTimeout(2000); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRequestStatusCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestRequestStatusCollectionAPI.java index 485558fff29a..d8b66e9cd87f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRequestStatusCollectionAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRequestStatusCollectionAPI.java @@ -189,7 +189,7 @@ protected NamedList sendRequest(ModifiableSolrParams params) throws SolrServerEx String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient).getBaseURL(); baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - try (HttpSolrClient baseServer = new HttpSolrClient(baseUrl)) { + try (HttpSolrClient baseServer = getHttpSolrClient(baseUrl)) { baseServer.setConnectionTimeout(15000); return baseServer.request(request); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java index 054c0745fd46..9e5e68b96cc0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java @@ -162,17 +162,17 @@ private static void createMiniSolrCloudCluster() throws Exception { assertNotNull("could not find URL for " + shardName + " replica", passiveUrl); if (shardName.equals("shard1")) { - S_ONE_LEADER_CLIENT = new HttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); - S_ONE_NON_LEADER_CLIENT = new HttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); + S_ONE_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); + S_ONE_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); } else if (shardName.equals("shard2")) { - S_TWO_LEADER_CLIENT = new HttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); - S_TWO_NON_LEADER_CLIENT = new HttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); + S_TWO_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/"); + S_TWO_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/"); } else { fail("unexpected shard: " + shardName); } } assertEquals("Should be exactly one server left (nost hosting either shard)", 1, urlMap.size()); - NO_COLLECTION_CLIENT = new HttpSolrClient(urlMap.values().iterator().next() + + NO_COLLECTION_CLIENT = getHttpSolrClient(urlMap.values().iterator().next() + "/" + COLLECTION_NAME + "/"); assertNotNull(S_ONE_LEADER_CLIENT); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java index b3f0423c9999..08ab4c020444 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java @@ -127,7 +127,7 @@ private static void createMiniSolrCloudCluster() throws Exception { for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { URL jettyURL = jetty.getBaseUrl(); - NODE_CLIENTS.add(new HttpSolrClient(jettyURL.toString() + "/" + COLLECTION_NAME + "/")); + NODE_CLIENTS.add(getHttpSolrClient(jettyURL.toString() + "/" + COLLECTION_NAME + "/")); } assertEquals(numServers, NODE_CLIENTS.size()); diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 7d53feebf73c..616cd374d637 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -123,7 +123,7 @@ private void testUnloadShardAndCollection() throws Exception{ SolrClient client = clients.get(0); String url1 = getBaseUrl(client); - try (HttpSolrClient adminClient = new HttpSolrClient(url1)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url1)) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(60000); adminClient.request(createCmd); @@ -174,7 +174,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // create a new collection collection SolrClient client = clients.get(0); String url1 = getBaseUrl(client); - try (HttpSolrClient adminClient = new HttpSolrClient(url1)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url1)) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(60000); @@ -194,7 +194,7 @@ private void testCoreUnloadAndLeaders() throws Exception { client = clients.get(1); String url2 = getBaseUrl(client); - try (HttpSolrClient adminClient = new HttpSolrClient(url2)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url2)) { Create createCmd = new Create(); createCmd.setCoreName("unloadcollection2"); @@ -213,7 +213,7 @@ private void testCoreUnloadAndLeaders() throws Exception { Random random = random(); if (random.nextBoolean()) { - try (HttpSolrClient collectionClient = new HttpSolrClient(leaderProps.getCoreUrl())) { + try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getCoreUrl())) { // lets try and use the solrj client to index and retrieve a couple // documents SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1, @@ -232,7 +232,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // create another replica for our collection client = clients.get(2); String url3 = getBaseUrl(client); - try (HttpSolrClient adminClient = new HttpSolrClient(url3)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url3)) { Create createCmd = new Create(); createCmd.setCoreName("unloadcollection3"); createCmd.setCollection("unloadcollection"); @@ -246,7 +246,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // so that we start with some versions when we reload... DirectUpdateHandler2.commitOnClose = false; - try (HttpSolrClient addClient = new HttpSolrClient(url3 + "/unloadcollection3")) { + try (HttpSolrClient addClient = getHttpSolrClient(url3 + "/unloadcollection3")) { addClient.setConnectionTimeout(30000); // add a few docs @@ -260,7 +260,7 @@ private void testCoreUnloadAndLeaders() throws Exception { //collectionClient.commit(); // unload the leader - try (HttpSolrClient collectionClient = new HttpSolrClient(leaderProps.getBaseUrl())) { + try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getBaseUrl())) { collectionClient.setConnectionTimeout(15000); collectionClient.setSoTimeout(30000); @@ -284,7 +284,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // ensure there is a leader zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); - try (HttpSolrClient addClient = new HttpSolrClient(url2 + "/unloadcollection2")) { + try (HttpSolrClient addClient = getHttpSolrClient(url2 + "/unloadcollection2")) { addClient.setConnectionTimeout(30000); addClient.setSoTimeout(90000); @@ -299,7 +299,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // create another replica for our collection client = clients.get(3); String url4 = getBaseUrl(client); - try (HttpSolrClient adminClient = new HttpSolrClient(url4)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url4)) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(30000); @@ -314,7 +314,7 @@ private void testCoreUnloadAndLeaders() throws Exception { // unload the leader again leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1"); - try (HttpSolrClient collectionClient = new HttpSolrClient(leaderProps.getBaseUrl())) { + try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getBaseUrl())) { collectionClient.setConnectionTimeout(15000); collectionClient.setSoTimeout(30000); @@ -338,7 +338,7 @@ private void testCoreUnloadAndLeaders() throws Exception { DirectUpdateHandler2.commitOnClose = true; // bring the downed leader back as replica - try (HttpSolrClient adminClient = new HttpSolrClient(leaderProps.getBaseUrl())) { + try (HttpSolrClient adminClient = getHttpSolrClient(leaderProps.getBaseUrl())) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(30000); @@ -352,7 +352,7 @@ private void testCoreUnloadAndLeaders() throws Exception { long found1, found3; - try (HttpSolrClient adminClient = new HttpSolrClient(url2 + "/unloadcollection")) { + try (HttpSolrClient adminClient = getHttpSolrClient(url2 + "/unloadcollection")) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(30000); adminClient.commit(); @@ -360,7 +360,7 @@ private void testCoreUnloadAndLeaders() throws Exception { q.set("distrib", false); found1 = adminClient.query(q).getResults().getNumFound(); } - try (HttpSolrClient adminClient = new HttpSolrClient(url3 + "/unloadcollection")) { + try (HttpSolrClient adminClient = getHttpSolrClient(url3 + "/unloadcollection")) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(30000); adminClient.commit(); @@ -369,7 +369,7 @@ private void testCoreUnloadAndLeaders() throws Exception { found3 = adminClient.query(q).getResults().getNumFound(); } - try (HttpSolrClient adminClient = new HttpSolrClient(url4 + "/unloadcollection")) { + try (HttpSolrClient adminClient = getHttpSolrClient(url4 + "/unloadcollection")) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(30000); adminClient.commit(); @@ -387,7 +387,7 @@ private void testCoreUnloadAndLeaders() throws Exception { private void testUnloadLotsOfCores() throws Exception { SolrClient client = clients.get(2); String url3 = getBaseUrl(client); - try (final HttpSolrClient adminClient = new HttpSolrClient(url3)) { + try (final HttpSolrClient adminClient = getHttpSolrClient(url3)) { adminClient.setConnectionTimeout(15000); adminClient.setSoTimeout(60000); int cnt = atLeast(3); diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java index 5faa065ae89c..ddf6e51c2243 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java @@ -97,7 +97,7 @@ public void test() throws Exception { List cloudClients = new ArrayList<>(); List threads = new ArrayList<>(); for (int i = 0; i < cnt; i++) { - CloudSolrClient client = new CloudSolrClient(zkServer.getZkAddress()); + CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress()); client.setDefaultCollection(ACOLLECTION + i); cloudClients.add(client); StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true); diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java index 601f4fe723ab..aeb4cb7803c4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java @@ -165,7 +165,7 @@ private void createAndDeleteCollection() throws SolrServerException, int i = 0; for (SolrClient client : clients) { - try (HttpSolrClient c = new HttpSolrClient(getBaseUrl(client) + "/" + DELETE_DATA_DIR_COLLECTION)) { + try (HttpSolrClient c = getHttpSolrClient(getBaseUrl(client) + "/" + DELETE_DATA_DIR_COLLECTION)) { int docCnt = random().nextInt(1000) + 1; for (int j = 0; j < docCnt; j++) { c.add(getDoc("id", i++, "txt_t", "just some random text for a doc")); diff --git a/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java b/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java index 7875b7c6c137..1c9cbeedb26e 100644 --- a/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java +++ b/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java @@ -148,14 +148,14 @@ private void buildClients() throws Exception { // Mostly to keep annoying logging messages from being sent out all the time. for (int idx = 0; idx < indexingThreads; ++idx) { - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); client.setDefaultMaxConnectionsPerHost(25); client.setConnectionTimeout(30000); client.setSoTimeout(60000); indexingClients.add(client); } for (int idx = 0; idx < queryThreads; ++idx) { - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); client.setDefaultMaxConnectionsPerHost(25); client.setConnectionTimeout(30000); client.setSoTimeout(30000); diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java index c8b46306fc2b..63afe6c6e7ee 100644 --- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java +++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java @@ -112,7 +112,7 @@ public void testDynamicLoading() throws Exception { assertEquals(TestBlobHandler.getAsString(map), ".system collection not available", map.get("msg")); - TestBlobHandler.createSystemCollection(new HttpSolrClient(baseURL, randomClient.getHttpClient())); + TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient())); waitForRecoveriesToFinish(".system", true); map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client); diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index eab3e87ec499..ec5280c73e7b 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -156,7 +156,8 @@ private static JettySolrRunner createJetty(SolrInstance instance) throws Excepti private static SolrClient createNewSolrClient(int port) { try { // setup the client... - HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/" + DEFAULT_TEST_CORENAME); + final String baseUrl = buildUrl(port) + "/" + DEFAULT_TEST_CORENAME; + HttpSolrClient client = getHttpSolrClient(baseUrl); client.setConnectionTimeout(15000); client.setSoTimeout(60000); client.setDefaultMaxConnectionsPerHost(100); @@ -277,7 +278,7 @@ private NamedList reloadCore(SolrClient s, String core) throws Exception private HttpSolrClient adminClient(SolrClient client) { String adminUrl = ((HttpSolrClient)client).getBaseURL().replace("/collection1", ""); - return new HttpSolrClient(adminUrl); + return getHttpSolrClient(adminUrl); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java index 29c0dd752394..3ef6259babe0 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java @@ -79,7 +79,8 @@ private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance i private static SolrClient createNewSolrClient(int port) { try { // setup the client... - HttpSolrClient client = new HttpSolrClient(buildUrl(port, context) + "/" + DEFAULT_TEST_CORENAME); + final String baseUrl = buildUrl(port, context) + "/" + DEFAULT_TEST_CORENAME; + HttpSolrClient client = getHttpSolrClient(baseUrl); client.setConnectionTimeout(15000); client.setSoTimeout(60000); client.setDefaultMaxConnectionsPerHost(100); diff --git a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java index 1218783d7b23..a39a57e4be6c 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java +++ b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java @@ -72,7 +72,8 @@ private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance i private static SolrClient createNewSolrClient(int port) { try { // setup the client... - HttpSolrClient client = new HttpSolrClient(buildUrl(port, context) + "/" + DEFAULT_TEST_CORENAME); + final String baseUrl = buildUrl(port, context) + "/" + DEFAULT_TEST_CORENAME; + HttpSolrClient client = getHttpSolrClient(baseUrl); client.setConnectionTimeout(15000); client.setSoTimeout(60000); client.setDefaultMaxConnectionsPerHost(100); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index 89f4b2b25885..3d4a51d518f0 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -254,7 +254,7 @@ public void testDeleteInstanceDir() throws Exception { JettySolrRunner runner = new JettySolrRunner(solrHomeDirectory.getAbsolutePath(), buildJettyConfig("/solr")); runner.start(); - try (HttpSolrClient client = new HttpSolrClient(runner.getBaseUrl() + "/corex")) { + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex")) { client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); SolrInputDocument doc = new SolrInputDocument(); @@ -263,7 +263,7 @@ public void testDeleteInstanceDir() throws Exception { client.commit(); } - try (HttpSolrClient client = new HttpSolrClient(runner.getBaseUrl().toString())) { + try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl().toString())) { client.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); client.setSoTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT); CoreAdminRequest.Unload req = new CoreAdminRequest.Unload(false); diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java index 16a3e875058b..b44766898ba3 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java @@ -67,8 +67,8 @@ public static void createThings() throws Exception { createJetty(solrHome.getAbsolutePath()); String url = jetty.getBaseUrl().toString(); - collection1 = new HttpSolrClient(url + "/collection1"); - collection2 = new HttpSolrClient(url + "/collection2"); + collection1 = getHttpSolrClient(url + "/collection1"); + collection2 = getHttpSolrClient(url + "/collection2"); String urlCollection1 = jetty.getBaseUrl().toString() + "/" + "collection1"; String urlCollection2 = jetty.getBaseUrl().toString() + "/" + "collection2"; @@ -76,7 +76,7 @@ public static void createThings() throws Exception { shard2 = urlCollection2.replaceAll("https?://", ""); //create second core - try (HttpSolrClient nodeClient = new HttpSolrClient(url)) { + try (HttpSolrClient nodeClient = getHttpSolrClient(url)) { CoreAdminRequest.Create req = new CoreAdminRequest.Create(); req.setCoreName("collection2"); req.setConfigSet("collection1"); diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java index 0c18b25fc244..960021eaadcf 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryElevationComponentTest.java @@ -109,7 +109,8 @@ public void test() throws Exception { assertEquals(true, document.getFieldValue("[elevated]")); // Force javabin format - HttpSolrClient client = new HttpSolrClient(((HttpSolrClient)clients.get(0)).getBaseURL()); + final String clientUrl = ((HttpSolrClient)clients.get(0)).getBaseURL(); + HttpSolrClient client = getHttpSolrClient(clientUrl); client.setParser(new BinaryResponseParser()); SolrQuery solrQuery = new SolrQuery("XXXX").setParam("qt", "/elevate").setParam("shards.qt", "/elevate").setRows(500).setFields("id,[elevated]") .setParam("enableElevation", "true").setParam("forceElevation", "true").setParam("elevateIds", "6", "wt", "javabin") diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrJ.java b/solr/core/src/test/org/apache/solr/search/TestSolrJ.java index b70205efd47d..1fd485f92920 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrJ.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrJ.java @@ -59,7 +59,7 @@ public static void main(String[] args) throws Exception { ConcurrentUpdateSolrClient concurrentClient = null; // server = concurrentClient = new ConcurrentUpdateSolrServer(addr,32,8); - client = concurrentClient = new ConcurrentUpdateSolrClient(addr,64,nConnections); + client = concurrentClient = getConcurrentUpdateSolrClient(addr,64,nConnections); client.deleteByQuery("*:*"); client.commit(); @@ -164,7 +164,7 @@ public static void indexDocs(int base, int count, int maxSleep) throws IOExcepti public void doCommitPerf() throws Exception { - try (HttpSolrClient client = new HttpSolrClient("http://127.0.0.1:8983/solr")) { + try (HttpSolrClient client = getHttpSolrClient("http://127.0.0.1:8983/solr")) { final RTimer timer = new RTimer(); diff --git a/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java b/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java index 9bb4d21eec23..f1b64bc2cbd4 100644 --- a/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java +++ b/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java @@ -117,8 +117,8 @@ public void testSimpleQuery() throws Exception { //Test against all nodes for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) { - SolrClient solrClient = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString()); - SolrClient solrClient_local = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString()); + SolrClient solrClient = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString()); + SolrClient solrClient_local = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString()); SolrQuery query = new SolrQuery("cat:football"); query.setFields("*,score"); @@ -161,8 +161,8 @@ public void testMultiCollectionQuery() throws Exception { //Test against all nodes for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) { - SolrClient solrClient = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString()); - SolrClient solrClient_local = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString()); + SolrClient solrClient = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString()); + SolrClient solrClient_local = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString()); SolrQuery query = new SolrQuery("cat:football"); query.setFields("*,score").add("collection", "collection1,collection2"); diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 96a8c146eb08..d24ef3e60141 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -59,6 +59,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singletonMap; +import static org.apache.solr.SolrTestCaseJ4.getHttpSolrClient; import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; @@ -162,7 +163,7 @@ protected void doExtraTests(MiniSolrCloudCluster miniCluster, SolrZkClient zkCli CollectionAdminRequest.Reload reload = new CollectionAdminRequest.Reload(); reload.setCollectionName(defaultCollName); - HttpSolrClient solrClient = new HttpSolrClient(baseUrl); + HttpSolrClient solrClient = getHttpSolrClient(baseUrl); try { rsp = solrClient.request(reload); fail("must have failed"); diff --git a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java b/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java index 85f219057168..d3304f45074e 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java @@ -78,7 +78,8 @@ public void test() throws Exception { HttpSolrClient randomClient = (HttpSolrClient) clients.get(random().nextInt(clients.size())); String baseURL = randomClient.getBaseURL(); - TestBlobHandler.createSystemCollection(new HttpSolrClient(baseURL.substring(0, baseURL.lastIndexOf('/')), randomClient.getHttpClient())); + final String solrClientUrl = baseURL.substring(0, baseURL.lastIndexOf('/')); + TestBlobHandler.createSystemCollection(getHttpSolrClient(solrClientUrl, randomClient.getHttpClient())); waitForRecoveriesToFinish(".system", true); TestBlobHandler.postAndCheck(cloudClient, baseURL.substring(0, baseURL.lastIndexOf('/')), blobName, TestDynamicLoading.generateZip(RuntimeUrp.class), 1); diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java index 1a07d3124aa5..28a58ff69e4c 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java +++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java @@ -351,7 +351,7 @@ protected void testExample(String exampleName) throws Exception { exampleSolrHomeDir.isDirectory()); if ("techproducts".equals(exampleName)) { - HttpSolrClient solrClient = new HttpSolrClient("http://localhost:" + bindPort + "/solr/" + exampleName); + HttpSolrClient solrClient = getHttpSolrClient("http://localhost:" + bindPort + "/solr/" + exampleName); SolrQuery query = new SolrQuery("*:*"); QueryResponse qr = solrClient.query(query); long numFound = qr.getResults().getNumFound(); @@ -439,7 +439,7 @@ public void testInteractiveSolrCloudExample() throws Exception { CloudSolrClient cloudClient = null; try { - cloudClient = new CloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress()); + cloudClient = getCloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress()); cloudClient.connect(); cloudClient.setDefaultCollection(collectionName); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 4ad37260a948..bd1dd2ceeb8a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -191,12 +190,17 @@ boolean isExpired(long timeToLiveMs) { * "host1:2181,host2:2181,host3:2181/mysolrchroot" *

    * "zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181" + * + * @deprecated use {@link Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost) { this.zkHost = zkHost; this.clientIsInternal = true; this.myClient = HttpClientUtil.createClient(null); - this.lbClient = new LBHttpSolrClient(myClient); + this.lbClient = new LBHttpSolrClient.Builder() + .withHttpClient(myClient) + .build(); this.lbClient.setRequestWriter(new BinaryRequestWriter()); this.lbClient.setParser(new BinaryResponseParser()); this.updatesToLeaders = true; @@ -226,14 +230,15 @@ public CloudSolrClient(String zkHost) { * @param httpClient * the {@link HttpClient} instance to be used for all requests. The * provided httpClient should use a multi-threaded connection manager. + * + * @deprecated use {@link Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost, HttpClient httpClient) { this.zkHost = zkHost; this.clientIsInternal = httpClient == null; this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); + this.lbClient = createLBHttpSolrClient(myClient); this.updatesToLeaders = true; shutdownLBHttpSolrServer = true; lbClient.addQueryParams(STATE_VERSION); @@ -256,7 +261,9 @@ public CloudSolrClient(String zkHost, HttpClient httpClient) { * @throws IllegalArgumentException * if the chroot value does not start with a forward slash. * @see #CloudSolrClient(String) + * @deprecated use {@link Builder} instead. */ + @Deprecated public CloudSolrClient(Collection zkHosts, String chroot) { this(zkHosts, chroot, null); } @@ -281,47 +288,64 @@ public CloudSolrClient(Collection zkHosts, String chroot) { * @throws IllegalArgumentException * if the chroot value does not start with a forward slash. * @see #CloudSolrClient(String) + * @deprecated use {@link Builder} instead. */ + @Deprecated public CloudSolrClient(Collection zkHosts, String chroot, HttpClient httpClient) { - StringBuilder zkBuilder = new StringBuilder(); - int lastIndexValue = zkHosts.size() - 1; - int i = 0; - for (String zkHost : zkHosts) { - zkBuilder.append(zkHost); - if (i < lastIndexValue) { - zkBuilder.append(","); - } - i++; - } - if (chroot != null) { - if (chroot.startsWith("/")) { - zkBuilder.append(chroot); - } else { - throw new IllegalArgumentException( - "The chroot must start with a forward slash."); - } - } - - /* Log the constructed connection string and then initialize. */ - log.info("Final constructed zkHost string: " + zkBuilder.toString()); - - this.zkHost = zkBuilder.toString(); + this.zkHost = buildZkHostString(zkHosts, chroot); this.clientIsInternal = httpClient == null; this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); - this.lbClient.setRequestWriter(new BinaryRequestWriter()); - this.lbClient.setParser(new BinaryResponseParser()); + this.lbClient = createLBHttpSolrClient(myClient); this.updatesToLeaders = true; shutdownLBHttpSolrServer = true; } + /** + * Create a new client object that connects to Zookeeper and is always aware + * of the SolrCloud state. If there is a fully redundant Zookeeper quorum and + * SolrCloud has enough replicas for every shard in a collection, there is no + * single point of failure. Updates will be sent to shard leaders by default. + * + * @param zkHosts + * A Java Collection (List, Set, etc) of HOST:PORT strings, one for + * each host in the zookeeper ensemble. Note that with certain + * Collection types like HashSet, the order of hosts in the final + * connect string may not be in the same order you added them. + * @param chroot + * A chroot value for zookeeper, starting with a forward slash. If no + * chroot is required, use null. + * @param httpClient + * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a + * multi-threaded connection manager. If null, a default HttpClient will be used. + * @param lbSolrClient + * LBHttpSolrServer instance for requests. If null, a default HttpClient will be used. + * @param updatesToLeaders + * If true, sends updates to shard leaders. + * + * @deprecated use {@link Builder} instead. This will soon be a protected method, and will only + * be available for use in implementing subclasses. + */ + @Deprecated + public CloudSolrClient(Collection zkHosts, String chroot, HttpClient httpClient, LBHttpSolrClient lbSolrClient, boolean updatesToLeaders) { + this.zkHost = buildZkHostString(zkHosts, chroot); + this.updatesToLeaders = updatesToLeaders; + + this.clientIsInternal = httpClient == null; + this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; + + this.shutdownLBHttpSolrServer = lbSolrClient == null; + this.lbClient = lbSolrClient == null ? createLBHttpSolrClient(myClient) : lbSolrClient; + } + /** * @param zkHost * A zookeeper client endpoint. * @param updatesToLeaders * If true, sends updates only to shard leaders. * @see #CloudSolrClient(String) for full description and details on zkHost + * @deprecated use {@link CloudSolrClient.Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost, boolean updatesToLeaders) { this(zkHost, updatesToLeaders, null); } @@ -335,12 +359,16 @@ public CloudSolrClient(String zkHost, boolean updatesToLeaders) { * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a * multi-threaded connection manager. * @see #CloudSolrClient(String) for full description and details on zkHost + * @deprecated use {@link CloudSolrClient.Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost, boolean updatesToLeaders, HttpClient httpClient) { this.zkHost = zkHost; this.clientIsInternal = httpClient == null; this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient; - this.lbClient = new LBHttpSolrClient(myClient); + this.lbClient = new LBHttpSolrClient.Builder() + .withHttpClient(myClient) + .build(); this.lbClient.setRequestWriter(new BinaryRequestWriter()); this.lbClient.setParser(new BinaryResponseParser()); this.updatesToLeaders = updatesToLeaders; @@ -362,7 +390,9 @@ public void setCollectionCacheTTl(int seconds){ * @param lbClient * LBHttpSolrServer instance for requests. * @see #CloudSolrClient(String) for full description and details on zkHost + * @deprecated use {@link CloudSolrClient.Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient) { this(zkHost, lbClient, true); } @@ -375,7 +405,9 @@ public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient) { * @param updatesToLeaders * If true, sends updates only to shard leaders. * @see #CloudSolrClient(String) for full description and details on zkHost + * @deprecated use {@link Builder} instead. */ + @Deprecated public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient, boolean updatesToLeaders) { this.zkHost = zkHost; this.lbClient = lbClient; @@ -1340,4 +1372,137 @@ public Map getShardReplicationFactor(String collection, NamedLis } return results; } + + private static LBHttpSolrClient createLBHttpSolrClient(HttpClient httpClient) { + final LBHttpSolrClient lbClient = new LBHttpSolrClient.Builder() + .withHttpClient(httpClient) + .build(); + lbClient.setRequestWriter(new BinaryRequestWriter()); + lbClient.setParser(new BinaryResponseParser()); + + return lbClient; + } + + private static String buildZkHostString(Collection zkHosts, String chroot) { + if (zkHosts == null || zkHosts.isEmpty()) { + throw new IllegalArgumentException("Cannot create CloudSearchClient without valid ZooKeeper host; none specified!"); + } + + StringBuilder zkBuilder = new StringBuilder(); + int lastIndexValue = zkHosts.size() - 1; + int i = 0; + for (String zkHost : zkHosts) { + zkBuilder.append(zkHost); + if (i < lastIndexValue) { + zkBuilder.append(","); + } + i++; + } + if (chroot != null) { + if (chroot.startsWith("/")) { + zkBuilder.append(chroot); + } else { + throw new IllegalArgumentException( + "The chroot must start with a forward slash."); + } + } + + /* Log the constructed connection string and then initialize. */ + final String zkHostString = zkBuilder.toString(); + log.info("Final constructed zkHost string: " + zkHostString); + return zkHostString; + } + + /** + * Constructs {@link CloudSolrClient} instances from provided configuration. + */ + public static class Builder { + private Collection zkHosts; + private HttpClient httpClient; + private String zkChroot; + private LBHttpSolrClient loadBalancedSolrClient; + private boolean shardLeadersOnly; + + public Builder() { + this.zkHosts = new ArrayList(); + this.shardLeadersOnly = true; + } + + /** + * Provide a ZooKeeper client endpoint to be used when configuring {@link CloudSolrClient} instances. + * + * Method may be called multiple times. All provided values will be used. + * + * @param zkHost + * The client endpoint of the ZooKeeper quorum containing the cloud + * state. + */ + public Builder withZkHost(String zkHost) { + this.zkHosts.add(zkHost); + return this; + } + + /** + * Provides a {@link HttpClient} for the builder to use when creating clients. + */ + public Builder withHttpClient(HttpClient httpClient) { + this.httpClient = httpClient; + return this; + } + + /** + * Provide a series of ZooKeeper client endpoints for the builder to use when creating clients. + * + * Method may be called multiple times. All provided values will be used. + * + * @param zkHosts + * A Java Collection (List, Set, etc) of HOST:PORT strings, one for + * each host in the ZooKeeper ensemble. Note that with certain + * Collection types like HashSet, the order of hosts in the final + * connect string may not be in the same order you added them. + */ + public Builder withZkHost(Collection zkHosts) { + this.zkHosts.addAll(zkHosts); + return this; + } + + /** + * Provides a ZooKeeper chroot for the builder to use when creating clients. + */ + public Builder withZkChroot(String zkChroot) { + this.zkChroot = zkChroot; + return this; + } + + /** + * Provides a {@link LBHttpSolrClient} for the builder to use when creating clients. + */ + public Builder withLBHttpSolrClient(LBHttpSolrClient loadBalancedSolrClient) { + this.loadBalancedSolrClient = loadBalancedSolrClient; + return this; + } + + /** + * Tells {@link Builder} that created clients should send updats only to shard leaders. + */ + public Builder sendUpdatesOnlyToShardLeaders() { + shardLeadersOnly = true; + return this; + } + + /** + * Tells {@link Builder} that created clients should send updates to all replicas for a shard. + */ + public Builder sendUpdatesToAllReplicasInShard() { + shardLeadersOnly = false; + return this; + } + + /** + * Create a {@link CloudSolrClient} based on the provided configuration. + */ + public CloudSolrClient build() { + return new CloudSolrClient(zkHosts, zkChroot, httpClient, loadBalancedSolrClient, shardLeadersOnly); + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java index 5bec96a27f52..c58baadb0703 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java @@ -16,6 +16,21 @@ */ package org.apache.solr.client.solrj.impl; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.Locale; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; @@ -45,21 +60,6 @@ import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.LinkedList; -import java.util.Locale; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - /** * ConcurrentUpdateSolrClient buffers all added documents and writes * them into open HTTP connections. This class is thread safe. @@ -96,7 +96,10 @@ public class ConcurrentUpdateSolrClient extends SolrClient { * The buffer size before the documents are sent to the server * @param threadCount * The number of background threads used to empty the queue + * + * @deprecated use {@link Builder} instead. */ + @Deprecated public ConcurrentUpdateSolrClient(String solrServerUrl, int queueSize, int threadCount) { this(solrServerUrl, null, queueSize, threadCount); @@ -104,6 +107,10 @@ public ConcurrentUpdateSolrClient(String solrServerUrl, int queueSize, internalHttpClient = true; } + /** + * @deprecated use {@link Builder} instead. + */ + @Deprecated public ConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, int threadCount) { this(solrServerUrl, client, queueSize, threadCount, ExecutorUtil.newMDCAwareCachedThreadPool( @@ -113,7 +120,10 @@ public ConcurrentUpdateSolrClient(String solrServerUrl, /** * Uses the supplied HttpClient to send documents to the Solr server. + * + * @deprecated use {@link Builder} instead. */ + @Deprecated public ConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es) { this(solrServerUrl, client, queueSize, threadCount, es, false); @@ -121,16 +131,30 @@ public ConcurrentUpdateSolrClient(String solrServerUrl, /** * Uses the supplied HttpClient to send documents to the Solr server. + * + * @deprecated use {@link Builder} instead. This will soon be a + * protected method, and will only be available for use in implementing subclasses. */ + @Deprecated public ConcurrentUpdateSolrClient(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) { - this.client = new HttpSolrClient(solrServerUrl, client); + this.internalHttpClient = (client == null); + this.client = new HttpSolrClient.Builder(solrServerUrl) + .withHttpClient(client) + .build(); this.client.setFollowRedirects(false); queue = new LinkedBlockingQueue<>(queueSize); this.threadCount = threadCount; runners = new LinkedList<>(); - scheduler = es; this.streamDeletes = streamDeletes; + + if (es != null) { + scheduler = es; + shutdownExecutor = false; + } else { + scheduler = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("concurrentUpdateScheduler")); + shutdownExecutor = true; + } } public Set getQueryParams() { @@ -274,6 +298,7 @@ public void writeTo(OutputStream out) throws IOException { method = new HttpPost(client.getBaseURL() + "/update" + requestParams.toQueryString()); + method.setEntity(template); method.addHeader("User-Agent", HttpSolrClient.AGENT); method.addHeader("Content-Type", contentType); @@ -530,4 +555,91 @@ public void setPollQueueTime(int pollQueueTime) { public void setRequestWriter(RequestWriter requestWriter) { client.setRequestWriter(requestWriter); } + + /** + * Constructs {@link ConcurrentUpdateSolrClient} instances from provided configuration. + */ + public static class Builder { + private String baseSolrUrl; + private HttpClient httpClient; + private int queueSize; + private int threadCount; + private ExecutorService executorService; + private boolean streamDeletes; + + /** + * Create a Builder object, based on the provided Solr URL. + * + * @param baseSolrUrl the base URL of the Solr server that will be targeted by any created clients. + */ + public Builder(String baseSolrUrl) { + this.baseSolrUrl = baseSolrUrl; + } + + /** + * Provides a {@link HttpClient} for the builder to use when creating clients. + */ + public Builder withHttpClient(HttpClient httpClient) { + this.httpClient = httpClient; + return this; + } + + /** + * The number of documents to batch together before sending to Solr. + */ + public Builder withQueueSize(int queueSize) { + if (queueSize <= 0) { + throw new IllegalArgumentException("queueSize must be a positive integer."); + } + this.queueSize = queueSize; + return this; + } + + /** + * The number of threads used to empty {@link ConcurrentUpdateSolrClient}s queue. + */ + public Builder withThreadCount(int threadCount) { + if (threadCount <= 0) { + throw new IllegalArgumentException("threadCount must be a positive integer."); + } + + this.threadCount = threadCount; + return this; + } + + /** + * Provides the {@link ExecutorService} for clients to use when servicing requests. + */ + public Builder withExecutorService(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + + /** + * Configures created clients to always stream delete requests. + */ + public Builder alwaysStreamDeletes() { + this.streamDeletes = true; + return this; + } + + /** + * Configures created clients to not stream delete requests. + */ + public Builder neverStreamDeletes() { + this.streamDeletes = false; + return this; + } + + /** + * Create a {@link ConcurrentUpdateSolrClient} based on the provided configuration options. + */ + public ConcurrentUpdateSolrClient build() { + if (baseSolrUrl == null) { + throw new IllegalArgumentException("Cannot create HttpSolrClient without a valid baseSolrUrl!"); + } + + return new ConcurrentUpdateSolrClient(baseSolrUrl, httpClient, queueSize, threadCount, executorService, streamDeletes); + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index b987afc032f9..538a2b729b33 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -16,6 +16,23 @@ */ package org.apache.solr.client.solrj.impl; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandles; +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -61,24 +78,6 @@ import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.lang.invoke.MethodHandles; -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - /** * A SolrClient implementation that talks directly to a Solr server via HTTP * @@ -157,16 +156,35 @@ public class HttpSolrClient extends SolrClient { * The URL of the Solr server. For example, " * http://localhost:8983/solr/" if you are using the * standard distribution Solr webapp on your local machine. + * @deprecated use {@link Builder} instead. */ + @Deprecated public HttpSolrClient(String baseURL) { this(baseURL, null, new BinaryResponseParser()); } + /** + * @deprecated use {@link Builder} instead. + */ + @Deprecated public HttpSolrClient(String baseURL, HttpClient client) { this(baseURL, client, new BinaryResponseParser()); } + /** + * @deprecated use {@link Builder} instead. + */ + @Deprecated public HttpSolrClient(String baseURL, HttpClient client, ResponseParser parser) { + this(baseURL, client, parser, false); + } + + /** + * @deprecated use {@link Builder} instead. This will soon be a 'protected' + * method, and will only be available for use in implementing subclasses. + */ + @Deprecated + public HttpSolrClient(String baseURL, HttpClient client, ResponseParser parser, boolean allowCompression) { this.baseUrl = baseURL; if (baseUrl.endsWith("/")) { baseUrl = baseUrl.substring(0, baseUrl.length() - 1); @@ -468,7 +486,7 @@ public boolean isRepeatable() { protected NamedList executeMethod(HttpRequestBase method, final ResponseParser processor) throws SolrServerException { method.addHeader("User-Agent", AGENT); - + HttpEntity entity = null; InputStream respBody = null; boolean shouldClose = true; @@ -752,4 +770,60 @@ public RemoteSolrException(String remoteHost, int code, String msg, Throwable th super(code, "Error from server at " + remoteHost + ": " + msg, th); } } + + /** + * Constructs {@link HttpSolrClient} instances from provided configuration. + */ + public static class Builder { + private String baseSolrUrl; + private HttpClient httpClient; + private ResponseParser responseParser; + private boolean compression; + + /** + * Create a Builder object, based on the provided Solr URL. + * + * By default, compression is not enabled in created HttpSolrClient objects. + * + * @param baseSolrUrl the base URL of the Solr server that will be targeted by any created clients. + */ + public Builder(String baseSolrUrl) { + this.baseSolrUrl = baseSolrUrl; + this.responseParser = new BinaryResponseParser(); + } + + /** + * Provides a {@link HttpClient} for the builder to use when creating clients. + */ + public Builder withHttpClient(HttpClient httpClient) { + this.httpClient = httpClient; + return this; + } + + /** + * Provides a {@link ResponseParser} for created clients to use when handling requests. + */ + public Builder withResponseParser(ResponseParser responseParser) { + this.responseParser = responseParser; + return this; + } + + /** + * Chooses whether created {@link HttpSolrClient}s use compression by default. + */ + public Builder allowCompression(boolean compression) { + this.compression = compression; + return this; + } + + /** + * Create a {@link HttpSolrClient} based on provided configuration. + */ + public HttpSolrClient build() { + if (baseSolrUrl == null) { + throw new IllegalArgumentException("Cannot create HttpSolrClient without a valid baseSolrUrl!"); + } + return new HttpSolrClient(baseSolrUrl, httpClient, responseParser, compression); + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java index 6d4711e8faf9..0b83969f9920 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java @@ -16,30 +16,44 @@ */ package org.apache.solr.client.solrj.impl; +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.net.ConnectException; +import java.net.MalformedURLException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.http.client.HttpClient; -import org.apache.solr.client.solrj.*; +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.IsUpdateRequest; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.apache.solr.common.SolrException; import org.slf4j.MDC; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.net.ConnectException; -import java.net.MalformedURLException; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.net.URL; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.*; - /** * LBHttpSolrClient or "LoadBalanced HttpSolrClient" is a load balancing wrapper around * {@link HttpSolrClient}. This is useful when you @@ -206,19 +220,31 @@ public String getServer() { } } + /** + * @deprecated use {@link Builder} instead. + */ + @Deprecated public LBHttpSolrClient(String... solrServerUrls) throws MalformedURLException { this(null, solrServerUrls); } - /** The provided httpClient should use a multi-threaded connection manager */ + /** + * The provided httpClient should use a multi-threaded connection manager + * @deprecated use {@link Builder} instead. + */ + @Deprecated public LBHttpSolrClient(HttpClient httpClient, String... solrServerUrl) { this(httpClient, new BinaryResponseParser(), solrServerUrl); } - /** The provided httpClient should use a multi-threaded connection manager */ + /** + * The provided httpClient should use a multi-threaded connection manager + * @deprecated use {@link Builder} instead. This will soon be a protected + * method and will only be available for use in implementing subclasses. + */ + @Deprecated public LBHttpSolrClient(HttpClient httpClient, ResponseParser parser, String... solrServerUrl) { clientIsInternal = (httpClient == null); - this.parser = parser; if (httpClient == null) { ModifiableSolrParams params = new ModifiableSolrParams(); if (solrServerUrl.length > 1) { @@ -231,6 +257,9 @@ public LBHttpSolrClient(HttpClient httpClient, ResponseParser parser, String... } else { this.httpClient = httpClient; } + + this.parser = parser; + for (String s : solrServerUrl) { ServerWrapper wrapper = new ServerWrapper(makeSolrClient(s)); aliveServers.put(wrapper.getKey(), wrapper); @@ -260,7 +289,10 @@ public static String normalize(String server) { } protected HttpSolrClient makeSolrClient(String server) { - HttpSolrClient client = new HttpSolrClient(server, httpClient, parser); + HttpSolrClient client = new HttpSolrClient.Builder(server) + .withHttpClient(httpClient) + .withResponseParser(parser) + .build(); if (requestWriter != null) { client.setRequestWriter(requestWriter); } @@ -717,4 +749,64 @@ protected void finalize() throws Throwable { private static final int CHECK_INTERVAL = 60 * 1000; //1 minute between checks private static final int NONSTANDARD_PING_LIMIT = 5; // number of times we'll ping dead servers not in the server list + /** + * Constructs {@link LBHttpSolrClient} instances from provided configuration. + */ + public static class Builder { + private final List baseSolrUrls; + private HttpClient httpClient; + private ResponseParser responseParser; + + public Builder() { + this.baseSolrUrls = new ArrayList(); + this.responseParser = new BinaryResponseParser(); + } + + /** + * Provide a Solr endpoint to be used when configuring {@link LBHttpSolrClient} instances. + * + * Method may be called multiple times. All provided values will be used. + */ + public Builder withBaseSolrUrl(String baseSolrUrl) { + this.baseSolrUrls.add(baseSolrUrl); + return this; + } + + /** + * Provide Solr endpoints to be used when configuring {@link LBHttpSolrClient} instances. + * + * Method may be called multiple times. All provided values will be used. + */ + public Builder withBaseSolrUrls(String... solrUrls) { + for (String baseSolrUrl : solrUrls) { + this.baseSolrUrls.add(baseSolrUrl); + } + return this; + } + + + /** + * Provides a {@link HttpClient} for the builder to use when creating clients. + */ + public Builder withHttpClient(HttpClient httpClient) { + this.httpClient = httpClient; + return this; + } + + /** + * Provides a {@link ResponseParser} for created clients to use when handling requests. + */ + public Builder withResponseParser(ResponseParser responseParser) { + this.responseParser = responseParser; + return this; + } + + /** + * Create a {@link HttpSolrClient} based on provided configuration. + */ + public LBHttpSolrClient build() { + final String[] baseUrlArray = new String[baseSolrUrls.size()]; + return new LBHttpSolrClient(httpClient, responseParser, baseSolrUrls.toArray(baseUrlArray)); + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java index e544c104252d..cd992a4215cd 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java @@ -21,7 +21,7 @@ import java.lang.invoke.MethodHandles; import java.util.Map; import java.util.HashMap; -import java.util.Iterator; + import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -44,7 +44,9 @@ public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { if (solrClients.containsKey(zkHost)) { client = (CloudSolrClient) solrClients.get(zkHost); } else { - client = new CloudSolrClient(zkHost); + client = new CloudSolrClient.Builder() + .withZkHost(zkHost) + .build(); client.connect(); solrClients.put(zkHost, client); } @@ -57,7 +59,8 @@ public synchronized HttpSolrClient getHttpSolrClient(String host) { if (solrClients.containsKey(host)) { client = (HttpSolrClient) solrClients.get(host); } else { - client = new HttpSolrClient(host); + client = new HttpSolrClient.Builder(host) + .build(); solrClients.put(host, client); } return client; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java index d316c83df09d..53f5c8dd25e8 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/DatabaseMetaDataImpl.java @@ -29,7 +29,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.util.SimpleOrderedMap; @@ -119,7 +119,7 @@ public String getDatabaseProductVersion() throws SQLException { for (String node : liveNodes) { try { String nodeURL = cloudSolrClient.getZkStateReader().getBaseUrlForNodeName(node); - solrClient = new HttpSolrClient(nodeURL); + solrClient = new Builder(nodeURL).build(); QueryResponse rsp = solrClient.query(sysQuery); return String.valueOf(((SimpleOrderedMap) rsp.getResponse().get("lucene")).get("solr-spec-version")); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java index 07e6cb16200c..c9c7b22e5ec9 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java @@ -34,6 +34,7 @@ import java.util.concurrent.Future; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -236,7 +237,9 @@ public void open() throws IOException { if(this.cache != null) { this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); } else { - this.cloudSolrClient = new CloudSolrClient(zkHost); + this.cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); this.cloudSolrClient.connect(); } constructStreams(); @@ -476,4 +479,4 @@ public TupleWrapper call() throws Exception { } } } -} \ No newline at end of file +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java index e4ae04cba683..3474257efc16 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,6 +26,7 @@ import java.util.Map.Entry; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -281,7 +281,9 @@ public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { - cloudSolrClient = new CloudSolrClient(zkHost); + cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); } FieldComparator[] adjustedSorts = adjustSorts(buckets, bucketSorts); @@ -479,4 +481,4 @@ public StreamComparator getStreamSort() { return bucketSorts[0]; } } -} \ No newline at end of file +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java index 1c149cc40786..7ff8a4e0810e 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java @@ -86,7 +86,7 @@ public void open() throws IOException { if(cache == null) { - client = new HttpSolrClient(baseUrl); + client = new HttpSolrClient.Builder(baseUrl).build(); } else { client = cache.getHttpSolrClient(baseUrl); } @@ -230,4 +230,4 @@ private Map mapFields(Map fields, Map mappings) { return fields; } -} \ No newline at end of file +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java index 478bc56074e3..6ef49eaf04d5 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java @@ -25,6 +25,7 @@ import java.util.Map.Entry; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -162,7 +163,9 @@ public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { - cloudSolrClient = new CloudSolrClient(zkHost); + cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); } ModifiableSolrParams params = getParams(this.props); @@ -290,4 +293,4 @@ private void addStat(Map map, String field, String stat, Object val) { map.put(stat+"("+field+")", val); } } -} \ No newline at end of file +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java index a52e8f9550bc..e3434fc91318 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java @@ -33,7 +33,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -218,7 +218,9 @@ public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { - cloudSolrClient = new CloudSolrClient(zkHost); + cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); this.cloudSolrClient.connect(); } @@ -474,4 +476,4 @@ protected void constructStreams() throws IOException { throw new IOException(e); } } -} \ No newline at end of file +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java index 67e52e630bd8..9ddd38115140 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java @@ -26,6 +26,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -219,7 +220,9 @@ private void setCloudSolrClient() { if(this.cache != null) { this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); } else { - this.cloudSolrClient = new CloudSolrClient(zkHost); + this.cloudSolrClient = new Builder() + .withZkHost(zkHost) + .build(); this.cloudSolrClient.connect(); } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java index 04807fe2e75a..79a56aadf322 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java @@ -40,7 +40,7 @@ public SolrClient createNewSolrClient() try { // setup the server... String url = jetty.getBaseUrl().toString() + "/collection1"; - HttpSolrClient client = new HttpSolrClient( url ); + HttpSolrClient client = getHttpSolrClient(url); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index 6547bc13791b..1dd074edf5d5 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -220,7 +220,7 @@ public void testExampleConfig() throws Exception if (jetty != null) { // check system wide system handler + "/admin/info/system" String url = jetty.getBaseUrl().toString(); - try (HttpSolrClient adminClient = new HttpSolrClient(url)) { + try (HttpSolrClient adminClient = getHttpSolrClient(url)) { SolrQuery q = new SolrQuery(); q.set("qt", "/admin/info/system"); QueryResponse rsp = adminClient.query(q); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java index f1b77e4e38d0..d6eb62a55ccd 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java @@ -37,7 +37,7 @@ public static void beforeTest() throws Exception { public SolrClient createNewSolrClient() { try { String url = jetty.getBaseUrl().toString() + "/collection1"; - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); client.setUseMultiPartPost(random().nextBoolean()); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java index 75a030d8fb75..6af6f731d2f4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.client.solrj; +import static org.apache.solr.SolrTestCaseJ4.getHttpSolrClient; + import org.apache.http.impl.client.CloseableHttpClient; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -39,7 +41,7 @@ public void testSolrException() throws Throwable { // set a 1ms timeout to let the connection fail faster. httpClient = HttpClientUtil.createClient(null); HttpClientUtil.setConnectionTimeout(httpClient, 1); - SolrClient client = new HttpSolrClient("http://[ff01::114]:11235/solr/", httpClient); + SolrClient client = getHttpSolrClient("http://[ff01::114]:11235/solr/", httpClient); SolrQuery query = new SolrQuery("test123"); client.query(query); httpClient.close(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java index dbef180a68e4..a2978345dd7f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrSchemalessExampleTest.java @@ -131,7 +131,7 @@ public SolrClient createNewSolrClient() { try { // setup the server... String url = jetty.getBaseUrl().toString() + "/collection1"; - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrClient.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrClient.java index 5bebd6446176..195f0f83dfce 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrClient.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrClient.java @@ -16,6 +16,17 @@ */ package org.apache.solr.client.solrj; +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import junit.framework.Assert; import org.apache.commons.io.FileUtils; @@ -40,17 +51,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeUnit; - /** * Test for LBHttpSolrClient * @@ -110,7 +110,7 @@ private void addDocs(SolrInstance solrInstance) throws IOException, SolrServerEx docs.add(doc); } SolrResponseBase resp; - try (HttpSolrClient client = new HttpSolrClient(solrInstance.getUrl(), httpClient)) { + try (HttpSolrClient client = getHttpSolrClient(solrInstance.getUrl(), httpClient)) { resp = client.add(docs); assertEquals(0, resp.getStatus()); resp = client.commit(); @@ -134,7 +134,7 @@ public void testSimple() throws Exception { for (int i = 0; i < solr.length; i++) { s[i] = solr[i].getUrl(); } - LBHttpSolrClient client = new LBHttpSolrClient(httpClient, s); + LBHttpSolrClient client = getLBHttpSolrClient(httpClient, s); client.setAliveCheckInterval(500); SolrQuery solrQuery = new SolrQuery("*:*"); Set names = new HashSet<>(); @@ -172,7 +172,7 @@ public void testSimple() throws Exception { } public void testTwoServers() throws Exception { - LBHttpSolrClient client = new LBHttpSolrClient(httpClient, solr[0].getUrl(), solr[1].getUrl()); + LBHttpSolrClient client = getLBHttpSolrClient(httpClient, solr[0].getUrl(), solr[1].getUrl()); client.setAliveCheckInterval(500); SolrQuery solrQuery = new SolrQuery("*:*"); QueryResponse resp = null; @@ -209,7 +209,7 @@ public void testReliability() throws Exception { params.set(HttpClientUtil.PROP_SO_TIMEOUT, 250); CloseableHttpClient myHttpClient = HttpClientUtil.createClient(params); try { - LBHttpSolrClient client = new LBHttpSolrClient(myHttpClient, s); + LBHttpSolrClient client = getLBHttpSolrClient(myHttpClient, s); client.setAliveCheckInterval(500); // Kill a server and test again diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java index 035619698eb8..6022b9acc142 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java @@ -54,7 +54,7 @@ public void testBadSetup() try { // setup the server... String url = "http" + (isSSLMode() ? "s" : "") + "://127.0.0.1/?core=xxx"; - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); Assert.fail("HttpSolrServer should not allow a path with a parameter: " + client.getBaseURL()); } catch( Exception ex ) { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java index c3fa7846e61d..093238d52d87 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java @@ -80,14 +80,8 @@ public SolrClient createNewSolrClient() public void testWaitOptions() throws Exception { // SOLR-3903 final List failures = new ArrayList<>(); - try (ConcurrentUpdateSolrClient concurrentClient = new ConcurrentUpdateSolrClient - (jetty.getBaseUrl().toString() + "/collection1", 2, 2) { - @Override - public void handleError(Throwable ex) { - failures.add(ex); - } - }) { - + final String serverUrl = jetty.getBaseUrl().toString() + "/collection1"; + try (ConcurrentUpdateSolrClient concurrentClient = new FailureRecordingConcurrentUpdateSolrClient(serverUrl, 2, 2)) { int docId = 42; for (UpdateRequest.ACTION action : EnumSet.allOf(UpdateRequest.ACTION.class)) { for (boolean waitSearch : Arrays.asList(true, false)) { @@ -110,5 +104,18 @@ public void handleError(Throwable ex) { null, failures.get(0)); } } + + class FailureRecordingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient { + private final List failures = new ArrayList<>(); + + public FailureRecordingConcurrentUpdateSolrClient(String serverUrl, int queueSize, int numThreads) { + super(serverUrl, null, queueSize, numThreads, null, false); + } + + @Override + public void handleError(Throwable ex) { + failures.add(ex); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrClientTest.java index 2c7ac9d96529..23b73db534b5 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrClientTest.java @@ -208,7 +208,7 @@ public static void beforeTest() throws Exception { public void testTimeout() throws Exception { SolrQuery q = new SolrQuery("*:*"); - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/slow/foo")) { + try(HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/slow/foo")) { client.setSoTimeout(2000); client.query(q, METHOD.GET); fail("No exception thrown."); @@ -228,7 +228,7 @@ public void testSolrExceptionCodeNotFromSolr() throws IOException, SolrServerExc assertEquals(status + " didn't generate an UNKNOWN error code, someone modified the list of valid ErrorCode's w/o changing this test to work a different way", ErrorCode.UNKNOWN, ErrorCode.getErrorCode(status)); - try ( HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { DebugServlet.setErrorCode(status); try { SolrQuery q = new SolrQuery("foo"); @@ -245,7 +245,7 @@ public void testSolrExceptionCodeNotFromSolr() throws IOException, SolrServerExc @Test public void testQuery() throws Exception { DebugServlet.clear(); - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { SolrQuery q = new SolrQuery("foo"); q.setParam("a", "\u1234"); try { @@ -369,7 +369,7 @@ public void testQuery() throws Exception { @Test public void testDelete() throws Exception { DebugServlet.clear(); - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { try { client.deleteById("id"); } catch (ParseException ignored) {} @@ -410,7 +410,7 @@ public void testDelete() throws Exception { @Test public void testGetById() throws Exception { DebugServlet.clear(); - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { Collection ids = Collections.singletonList("a"); try { client.getById("a"); @@ -433,7 +433,7 @@ public void testGetById() throws Exception { @Test public void testUpdate() throws Exception { DebugServlet.clear(); - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { UpdateRequest req = new UpdateRequest(); req.add(new SolrInputDocument()); req.setParam("a", "\u1234"); @@ -497,7 +497,8 @@ public void testUpdate() throws Exception { @Test public void testRedirect() throws Exception { - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/redirect/foo")) { + final String clientUrl = jetty.getBaseUrl().toString() + "/redirect/foo"; + try (HttpSolrClient client = getHttpSolrClient(clientUrl)) { SolrQuery q = new SolrQuery("*:*"); // default = false try { @@ -524,9 +525,9 @@ public void testRedirect() throws Exception { @Test public void testCompression() throws Exception { - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { SolrQuery q = new SolrQuery("*:*"); - + // verify request header gets set DebugServlet.clear(); try { @@ -564,7 +565,7 @@ public void testCompression() throws Exception { } // verify compressed response can be handled - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/collection1")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/collection1")) { client.setAllowCompression(true); SolrQuery q = new SolrQuery("foo"); QueryResponse response = client.query(q); @@ -575,7 +576,7 @@ public void testCompression() throws Exception { @Test public void testCollectionParameters() throws IOException, SolrServerException { - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString())) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString())) { SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", "collection"); client.add("collection1", doc); @@ -584,7 +585,8 @@ public void testCollectionParameters() throws IOException, SolrServerException { assertEquals(1, client.query("collection1", new SolrQuery("id:collection")).getResults().getNumFound()); } - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/collection1")) { + final String collection1Url = jetty.getBaseUrl().toString() + "/collection1"; + try (HttpSolrClient client = getHttpSolrClient(collection1Url)) { assertEquals(1, client.query(new SolrQuery("id:collection")).getResults().getNumFound()); } @@ -594,7 +596,7 @@ public void testCollectionParameters() throws IOException, SolrServerException { public void testSetParametersExternalClient() throws IOException{ try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null); - HttpSolrClient solrClient = new HttpSolrClient(jetty.getBaseUrl().toString(), httpClient)) { + HttpSolrClient solrClient = getHttpSolrClient(jetty.getBaseUrl().toString(), httpClient)) { try { solrClient.setMaxTotalConnections(1); @@ -611,7 +613,7 @@ public void testSetParametersExternalClient() throws IOException{ @Test public void testGetRawStream() throws SolrServerException, IOException{ try (CloseableHttpClient client = HttpClientUtil.createClient(null)) { - HttpSolrClient solrClient = new HttpSolrClient(jetty.getBaseUrl().toString() + "/collection1", + HttpSolrClient solrClient = getHttpSolrClient(jetty.getBaseUrl().toString() + "/collection1", client, null); QueryRequest req = new QueryRequest(); NamedList response = solrClient.request(req); @@ -698,8 +700,8 @@ public void testInterceptors() { HttpClientUtil.addRequestInterceptor(changeRequestInterceptor); HttpClientUtil.addRequestInterceptor(cookieSettingRequestInterceptor); - try(HttpSolrClient server = new HttpSolrClient(jetty.getBaseUrl().toString() + - "/debug/foo")) { + final String clientUrl = jetty.getBaseUrl().toString() + "/debug/foo"; + try(HttpSolrClient server = getHttpSolrClient(clientUrl)) { SolrQuery q = new SolrQuery("foo"); q.setParam("a", "\u1234"); @@ -770,7 +772,8 @@ private void verifyServletState(HttpSolrClient client, SolrRequest request) { @Test public void testQueryString() throws Exception { - try (HttpSolrClient client = new HttpSolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) { + final String clientUrl = jetty.getBaseUrl().toString() + "/debug/foo"; + try(HttpSolrClient client = getHttpSolrClient(clientUrl)) { // test without request query params DebugServlet.clear(); client.setQueryParams(setOf("serverOnly")); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java new file mode 100644 index 000000000000..57692c75e3f6 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java @@ -0,0 +1,90 @@ +package org.apache.solr.client.solrj.impl; + +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class CloudSolrClientBuilderTest extends LuceneTestCase { + private static final String ANY_CHROOT = "/ANY_CHROOT"; + private static final String ANY_ZK_HOST = "ANY_ZK_HOST"; + private static final String ANY_OTHER_ZK_HOST = "ANY_OTHER_ZK_HOST"; + + @Test(expected = IllegalArgumentException.class) + public void testNoZkHostSpecified() { + new Builder() + .withZkChroot(ANY_CHROOT) + .build(); + } + + @Test + public void testSingleZkHostSpecified() throws IOException { + try(CloudSolrClient createdClient = new Builder() + .withZkHost(ANY_ZK_HOST) + .withZkChroot(ANY_CHROOT) + .build()) { + final String clientZkHost = createdClient.getZkHost(); + + assertTrue(clientZkHost.contains(ANY_ZK_HOST)); + } + } + + @Test + public void testSeveralZkHostsSpecifiedSingly() throws IOException { + try (CloudSolrClient createdClient = new Builder() + .withZkHost(ANY_ZK_HOST) + .withZkHost(ANY_OTHER_ZK_HOST) + .withZkChroot(ANY_CHROOT) + .build()) { + final String clientZkHost = createdClient.getZkHost(); + + assertTrue(clientZkHost.contains(ANY_ZK_HOST)); + assertTrue(clientZkHost.contains(ANY_OTHER_ZK_HOST)); + } + } + + @Test + public void testSeveralZkHostsSpecifiedTogether() throws IOException { + final ArrayList zkHosts = new ArrayList(); + zkHosts.add(ANY_ZK_HOST); + zkHosts.add(ANY_OTHER_ZK_HOST); + try(CloudSolrClient createdClient = new Builder() + .withZkHost(zkHosts) + .withZkChroot(ANY_CHROOT) + .build()) { + final String clientZkHost = createdClient.getZkHost(); + + assertTrue(clientZkHost.contains(ANY_ZK_HOST)); + assertTrue(clientZkHost.contains(ANY_OTHER_ZK_HOST)); + } + } + + @Test + public void testByDefaultConfiguresClientToSendUpdatesOnlyToShardLeaders() throws IOException { + try(CloudSolrClient createdClient = new Builder() + .withZkHost(ANY_ZK_HOST) + .withZkChroot(ANY_CHROOT) + .build()) { + assertTrue(createdClient.isUpdatesToLeaders() == true); + } + } +} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java index 77d8d0fff3bd..36aea5817d74 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java @@ -225,7 +225,7 @@ private void allTests() throws Exception { params.add("q", "id:" + id); params.add("distrib", "false"); QueryRequest queryRequest = new QueryRequest(params); - try (HttpSolrClient solrClient = new HttpSolrClient(url)) { + try (HttpSolrClient solrClient = getHttpSolrClient(url)) { QueryResponse queryResponse = queryRequest.process(solrClient); SolrDocumentList docList = queryResponse.getResults(); assertTrue(docList.getNumFound() == 1); @@ -247,7 +247,7 @@ private void allTests() throws Exception { assertTrue(docs.getNumFound() == 0); // Test Multi-Threaded routed updates for UpdateRequest - try (CloudSolrClient threadedClient = new CloudSolrClient(zkServer.getZkAddress())) { + try (CloudSolrClient threadedClient = getCloudSolrClient(zkServer.getZkAddress())) { threadedClient.setParallelUpdates(true); threadedClient.setDefaultCollection(collectionName); response = threadedClient.request(request); @@ -266,7 +266,7 @@ private void allTests() throws Exception { params.add("q", "id:" + id); params.add("distrib", "false"); QueryRequest queryRequest = new QueryRequest(params); - try (HttpSolrClient solrClient = new HttpSolrClient(url)) { + try (HttpSolrClient solrClient = getHttpSolrClient(url)) { QueryResponse queryResponse = queryRequest.process(solrClient); SolrDocumentList docList = queryResponse.getResults(); assertTrue(docList.getNumFound() == 1); @@ -480,7 +480,7 @@ private Long getNumRequests(String baseUrl, String collectionName) throws SolrServerException, IOException { NamedList resp; - try (HttpSolrClient client = new HttpSolrClient(baseUrl + "/"+ collectionName)) { + try (HttpSolrClient client = getHttpSolrClient(baseUrl + "/"+ collectionName)) { client.setConnectionTimeout(15000); client.setSoTimeout(60000); ModifiableSolrParams params = new ModifiableSolrParams(); @@ -552,7 +552,8 @@ private void stateVersionParamTest() throws Exception { SolrQuery q = new SolrQuery().setQuery("*:*"); HttpSolrClient.RemoteSolrException sse = null; - try (HttpSolrClient solrClient = new HttpSolrClient(r.getStr(ZkStateReader.BASE_URL_PROP) + "/"+collectionName)) { + final String url = r.getStr(ZkStateReader.BASE_URL_PROP) + "/" +collectionName; + try (HttpSolrClient solrClient = getHttpSolrClient(url)) { log.info("should work query, result {}", solrClient.query(q)); //no problem @@ -588,7 +589,9 @@ private void stateVersionParamTest() throws Exception { log.info("the node which does not serve this collection{} ",theNode); assertNotNull(theNode); - try (SolrClient solrClient = new HttpSolrClient(theNode + "/"+collectionName)) { + + final String solrClientUrl = theNode + "/" + collectionName; + try (SolrClient solrClient = getHttpSolrClient(solrClientUrl)) { q.setParam(CloudSolrClient.STATE_VERSION, collectionName + ":" + (coll.getZNodeVersion()-1)); try { @@ -605,7 +608,7 @@ private void stateVersionParamTest() throws Exception { } public void testShutdown() throws IOException { - try (CloudSolrClient client = new CloudSolrClient("[ff01::114]:33332")) { + try (CloudSolrClient client = getCloudSolrClient("[ff01::114]:33332")) { client.setZkConnectTimeout(100); client.connect(); fail("Expected exception"); @@ -622,7 +625,7 @@ public void testWrongZkChrootTest() throws IOException { exception.expect(SolrException.class); exception.expectMessage("cluster not found/not ready"); - try (CloudSolrClient client = new CloudSolrClient(zkServer.getZkAddress() + "/xyz/foo")) { + try (CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress() + "/xyz/foo")) { client.setDefaultCollection(DEFAULT_COLLECTION); client.setZkClientTimeout(1000 * 60); client.connect(); @@ -636,7 +639,7 @@ public void customHttpClientTest() throws IOException { params.set(HttpClientUtil.PROP_SO_TIMEOUT, 1000); try (CloseableHttpClient client = HttpClientUtil.createClient(params); - CloudSolrClient solrClient = new CloudSolrClient(zkServer.getZkAddress(), client)) { + CloudSolrClient solrClient = getCloudSolrClient(zkServer.getZkAddress(), client)) { assertTrue(solrClient.getLbClient().getHttpClient() == client); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java new file mode 100644 index 000000000000..3389b7fcb3bb --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java @@ -0,0 +1,33 @@ +package org.apache.solr.client.solrj.impl; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.Builder; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for {@link Builder}. + */ +public class ConcurrentUpdateSolrClientBuilderTest extends LuceneTestCase { + + @Test(expected = IllegalArgumentException.class) + public void testRejectsMissingBaseSolrUrl() { + new Builder(null).build(); + } +} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.java index bd5e6a25be2d..5e2baeb6c841 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.java @@ -147,17 +147,8 @@ public void testConcurrentUpdate() throws Exception { final StringBuilder errors = new StringBuilder(); @SuppressWarnings("serial") - ConcurrentUpdateSolrClient concurrentClient = new ConcurrentUpdateSolrClient(serverUrl, cussQueueSize, cussThreadCount) { - @Override - public void handleError(Throwable ex) { - errorCounter.incrementAndGet(); - errors.append(" "+ex); - } - @Override - public void onSuccess(HttpResponse resp) { - successCounter.incrementAndGet(); - } - }; + ConcurrentUpdateSolrClient concurrentClient = new OutcomeCountingConcurrentUpdateSolrClient(serverUrl, cussQueueSize, + cussThreadCount, successCounter, errorCounter, errors); concurrentClient.setPollQueueTime(0); @@ -224,4 +215,28 @@ public void run() { } } } + + class OutcomeCountingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient { + private final AtomicInteger successCounter; + private final AtomicInteger failureCounter; + private final StringBuilder errors; + public OutcomeCountingConcurrentUpdateSolrClient(String serverUrl, int queueSize, int threadCount, + AtomicInteger successCounter, AtomicInteger failureCounter, StringBuilder errors) { + super(serverUrl, null, queueSize, threadCount, null, false); + + this.successCounter = successCounter; + this.failureCounter = failureCounter; + this.errors = errors; + } + + @Override + public void handleError(Throwable ex) { + failureCounter.incrementAndGet(); + errors.append(" "+ex); + } + @Override + public void onSuccess(HttpResponse resp) { + successCounter.incrementAndGet(); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java new file mode 100644 index 000000000000..da56576034f1 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java @@ -0,0 +1,76 @@ +package org.apache.solr.client.solrj.impl; + +import java.io.IOException; + +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for {@link Builder}. + */ +public class HttpSolrClientBuilderTest extends LuceneTestCase { + private static final String ANY_BASE_SOLR_URL = "ANY_BASE_SOLR_URL"; + private static final HttpClient ANY_HTTP_CLIENT = HttpClientBuilder.create().build(); + private static final ResponseParser ANY_RESPONSE_PARSER = new NoOpResponseParser(); + + @Test(expected = IllegalArgumentException.class) + public void testBaseSolrUrlIsRequired() { + new Builder(null) + .build(); + } + + @Test + public void testProvidesBaseSolrUrlToClient() throws IOException { + try (HttpSolrClient createdClient = new HttpSolrClient.Builder(ANY_BASE_SOLR_URL).build()) { + assertTrue(createdClient.getBaseURL().equals(ANY_BASE_SOLR_URL)); + } + } + + @Test + public void testProvidesHttpClientToClient() throws IOException { + try(HttpSolrClient createdClient = new Builder(ANY_BASE_SOLR_URL) + .withHttpClient(ANY_HTTP_CLIENT) + .build()) { + assertTrue(createdClient.getHttpClient().equals(ANY_HTTP_CLIENT)); + } + } + + @Test + public void testProvidesResponseParserToClient() throws IOException { + try(HttpSolrClient createdClient = new Builder(ANY_BASE_SOLR_URL) + .withResponseParser(ANY_RESPONSE_PARSER) + .build()) { + assertTrue(createdClient.getParser().equals(ANY_RESPONSE_PARSER)); + } + } + + @Test + public void testDefaultsToBinaryResponseParserWhenNoneProvided() throws IOException { + try(HttpSolrClient createdClient = new Builder(ANY_BASE_SOLR_URL) + .build()) { + final ResponseParser usedParser = createdClient.getParser(); + assertTrue(usedParser instanceof BinaryResponseParser); + } + } +} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java new file mode 100644 index 000000000000..6a02da5afbc3 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java @@ -0,0 +1,65 @@ +package org.apache.solr.client.solrj.impl; + +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder; +import org.apache.solr.client.solrj.ResponseParser; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for {@link Builder}. + */ +public class LBHttpSolrClientBuilderTest extends LuceneTestCase { + private static final String ANY_BASE_SOLR_URL = "ANY_BASE_SOLR_URL"; + private static final HttpClient ANY_HTTP_CLIENT = HttpClientBuilder.create().build(); + private static final ResponseParser ANY_RESPONSE_PARSER = new NoOpResponseParser(); + + @Test + public void providesHttpClientToClient() { + try(LBHttpSolrClient createdClient = new Builder() + .withBaseSolrUrl(ANY_BASE_SOLR_URL) + .withHttpClient(ANY_HTTP_CLIENT) + .build()) { + assertTrue(createdClient.getHttpClient().equals(ANY_HTTP_CLIENT)); + } + } + + @Test + public void providesResponseParserToClient() { + try(LBHttpSolrClient createdClient = new Builder() + .withBaseSolrUrl(ANY_BASE_SOLR_URL) + .withResponseParser(ANY_RESPONSE_PARSER) + .build()) { + assertTrue(createdClient.getParser().equals(ANY_RESPONSE_PARSER)); + } + } + + @Test + public void testDefaultsToBinaryResponseParserWhenNoneProvided() { + try(LBHttpSolrClient createdClient = new Builder() + .withBaseSolrUrl(ANY_BASE_SOLR_URL) + .build()) { + final ResponseParser usedParser = createdClient.getParser(); + + assertTrue(usedParser instanceof BinaryResponseParser); + } + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index a751459e5964..00b7577d1de2 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -442,7 +442,7 @@ public SortedMap,String> getExtraRequestFilters() { protected SolrClient createNewSolrClient(int port) { try { // setup the client... - HttpSolrClient client = new HttpSolrClient(buildUrl(port) + "/" + DEFAULT_TEST_CORENAME); + HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/" + DEFAULT_TEST_CORENAME); client.setConnectionTimeout(clientConnectionTimeout); client.setSoTimeout(clientSoTimeout); client.setDefaultMaxConnectionsPerHost(100); diff --git a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java index 0e77a4713810..b5d1b989c2c9 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java @@ -151,7 +151,7 @@ public SolrClient createNewSolrClient() { try { // setup the client... String url = jetty.getBaseUrl().toString() + "/" + "collection1"; - HttpSolrClient client = new HttpSolrClient( url ); + HttpSolrClient client = getHttpSolrClient(url); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java index 4aed1356415e..118c19421b77 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java @@ -16,15 +16,31 @@ */ package org.apache.solr; +import java.io.File; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; + import com.google.common.base.Charsets; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -43,24 +59,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.Set; - @SolrTestCaseJ4.SuppressSSL //@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending","Asserting"}) @@ -418,7 +416,7 @@ public String getShardURL() { public SolrClient getSolrJ() { if (solrj == null) { - solrj = new HttpSolrClient(getCollectionURL()); + solrj = getHttpSolrClient(getCollectionURL()); } return solrj; } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 3ce252f9b4d0..a98c62d34352 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -31,6 +31,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -56,6 +57,7 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.codec.Charsets; import org.apache.commons.io.FileUtils; +import org.apache.http.client.HttpClient; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.index.IndexWriterConfig; @@ -67,7 +69,13 @@ import org.apache.lucene.util.TestUtil; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.impl.HttpClientConfigurer; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; +import org.apache.solr.client.solrj.impl.LBHttpSolrClient; +import org.apache.solr.client.solrj.ResponseParser; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.IpTables; import org.apache.solr.common.SolrDocument; @@ -2040,6 +2048,140 @@ public boolean assertSolrInputFieldEquals(Object expected, Object actual) { public static Object skewed(Object likely, Object unlikely) { return (0 == TestUtil.nextInt(random(), 0, 9)) ? unlikely : likely; } + + public static CloudSolrClient getCloudSolrClient(String zkHost) { + if (random().nextBoolean()) { + return new CloudSolrClient(zkHost); + } + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .build(); + } + + public static CloudSolrClient getCloudSolrClient(String zkHost, HttpClient httpClient) { + if (random().nextBoolean()) { + return new CloudSolrClient(zkHost, httpClient); + } + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .withHttpClient(httpClient) + .build(); + } + + public static CloudSolrClient getCloudSolrClient(String zkHost, boolean shardLeadersOnly) { + if (random().nextBoolean()) { + return new CloudSolrClient(zkHost, shardLeadersOnly); + } + + if (shardLeadersOnly) { + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .sendUpdatesOnlyToShardLeaders() + .build(); + } + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .sendUpdatesToAllReplicasInShard() + .build(); + } + + public static CloudSolrClient getCloudSolrClient(String zkHost, boolean shardLeadersOnly, HttpClient httpClient) { + if (random().nextBoolean()) { + return new CloudSolrClient(zkHost, shardLeadersOnly, httpClient); + } + + if (shardLeadersOnly) { + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .withHttpClient(httpClient) + .sendUpdatesOnlyToShardLeaders() + .build(); + } + return new CloudSolrClient.Builder() + .withZkHost(zkHost) + .withHttpClient(httpClient) + .sendUpdatesToAllReplicasInShard() + .build(); + } + + public static ConcurrentUpdateSolrClient getConcurrentUpdateSolrClient(String baseSolrUrl, int queueSize, int threadCount) { + if (random().nextBoolean()) { + return new ConcurrentUpdateSolrClient(baseSolrUrl, queueSize, threadCount); + } + return new ConcurrentUpdateSolrClient.Builder(baseSolrUrl) + .withQueueSize(queueSize) + .withThreadCount(threadCount) + .build(); + } + + public static ConcurrentUpdateSolrClient getConcurrentUpdateSolrClient(String baseSolrUrl, HttpClient httpClient, int queueSize, int threadCount) { + if (random().nextBoolean()) { + return new ConcurrentUpdateSolrClient(baseSolrUrl, httpClient, queueSize, threadCount); + } + return new ConcurrentUpdateSolrClient.Builder(baseSolrUrl) + .withHttpClient(httpClient) + .withQueueSize(queueSize) + .withThreadCount(threadCount) + .build(); + } + + public static LBHttpSolrClient getLBHttpSolrClient(HttpClient client, String... solrUrls) { + if (random().nextBoolean()) { + return new LBHttpSolrClient(client, solrUrls); + } + + return new LBHttpSolrClient.Builder() + .withHttpClient(client) + .withBaseSolrUrls(solrUrls) + .build(); + } + + public static LBHttpSolrClient getLBHttpSolrClient(String... solrUrls) throws MalformedURLException { + if (random().nextBoolean()) { + return new LBHttpSolrClient(solrUrls); + } + return new LBHttpSolrClient.Builder() + .withBaseSolrUrls(solrUrls) + .build(); + } + + public static HttpSolrClient getHttpSolrClient(String url, HttpClient httpClient, ResponseParser responseParser, boolean compression) { + if(random().nextBoolean()) { + return new HttpSolrClient(url, httpClient, responseParser, compression); + } + return new Builder(url) + .withHttpClient(httpClient) + .withResponseParser(responseParser) + .allowCompression(compression) + .build(); + } + + public static HttpSolrClient getHttpSolrClient(String url, HttpClient httpClient, ResponseParser responseParser) { + if(random().nextBoolean()) { + return new HttpSolrClient(url, httpClient, responseParser); + } + return new Builder(url) + .withHttpClient(httpClient) + .withResponseParser(responseParser) + .build(); + } + + public static HttpSolrClient getHttpSolrClient(String url, HttpClient httpClient) { + if(random().nextBoolean()) { + return new HttpSolrClient(url, httpClient); + } + return new Builder(url) + .withHttpClient(httpClient) + .build(); + } + + public static HttpSolrClient getHttpSolrClient(String url) { + if(random().nextBoolean()) { + return new HttpSolrClient(url); + } + return new Builder(url) + .build(); + } /** * Returns a randomly generated Date in the appropriate Solr external (input) format diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index a584dbd450bc..6028563bc16f 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -16,11 +16,6 @@ */ package org.apache.solr.cloud; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; -import static org.apache.solr.common.util.Utils.makeMap; - import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -90,6 +85,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; +import static org.apache.solr.common.util.Utils.makeMap; + /** * TODO: we should still test this works as a custom update chain as well as * what we test now - the default update chain @@ -272,7 +272,7 @@ protected void initCloud() throws Exception { } protected CloudSolrClient createCloudClient(String defaultCollection) { - CloudSolrClient client = new CloudSolrClient(zkServer.getZkAddress(), random().nextBoolean()); + CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean()); client.setParallelUpdates(random().nextBoolean()); if (defaultCollection != null) client.setDefaultCollection(defaultCollection); client.getLbClient().setConnectionTimeout(30000); @@ -1626,7 +1626,7 @@ protected SolrClient createNewSolrClient(int port) { // setup the server... String baseUrl = buildUrl(port); String url = baseUrl + (baseUrl.endsWith("/") ? "" : "/") + DEFAULT_COLLECTION; - HttpSolrClient client = new HttpSolrClient(url); + HttpSolrClient client = getHttpSolrClient(url); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setSoTimeout(60000); client.setDefaultMaxConnectionsPerHost(100); @@ -1640,7 +1640,7 @@ protected SolrClient createNewSolrClient(int port) { protected SolrClient createNewSolrClient(String collection, String baseUrl) { try { // setup the server... - HttpSolrClient client = new HttpSolrClient(baseUrl + "/" + collection); + HttpSolrClient client = getHttpSolrClient(baseUrl + "/" + collection); client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); @@ -1726,7 +1726,7 @@ protected CloudSolrClient getCommonCloudSolrClient() { if (commonCloudSolrClient == null) { boolean updatesToLeaders = random().nextBoolean(); boolean parallelUpdates = random().nextBoolean(); - commonCloudSolrClient = new CloudSolrClient(zkServer.getZkAddress(), + commonCloudSolrClient = getCloudSolrClient(zkServer.getZkAddress(), updatesToLeaders); commonCloudSolrClient.getLbClient().setConnectionTimeout(5000); commonCloudSolrClient.getLbClient().setSoTimeout(120000); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 118f7bef621e..d23b37cd4b87 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -42,6 +42,7 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.embedded.SSLConfig; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; @@ -445,7 +446,9 @@ public CloudSolrClient getSolrClient() { } protected CloudSolrClient buildSolrClient() { - return new CloudSolrClient(getZkServer().getZkAddress()); + return new Builder() + .withZkHost(getZkServer().getZkAddress()) + .build(); } private static String getHostContextSuitableForServletContext(String ctx) { From a8f9073a3ffae9c8ec89ca7c3367098e6c9b2c5e Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Fri, 8 Apr 2016 20:34:23 -0400 Subject: [PATCH 0316/1113] Stop tests from failing just because of toString test --- .../src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 4e2e02e05fa8..3aa1719f7bcc 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -805,7 +805,7 @@ public void collect(int doc) { public void testToString() { Geo3DPoint point = new Geo3DPoint("point", 44.244272, 7.769736); - assertEquals("Geo3DPoint ", point.toString()); + assertEquals("Geo3DPoint stack = new ArrayList<>(); private List stackToTargetDoc; @@ -1180,7 +1183,9 @@ private static class ExplainingVisitor implements IntersectVisitor { // In the first phase, we always return CROSSES to do a full scan of the BKD tree to see which leaf block the document lives in boolean firstPhase = true; - public ExplainingVisitor(IntersectVisitor in, int targetDocID, int numDims, int bytesPerDim, StringBuilder b) { + public ExplainingVisitor(GeoShape shape, GeoPoint targetDocPoint, IntersectVisitor in, int targetDocID, int numDims, int bytesPerDim, StringBuilder b) { + this.shape = shape; + this.targetDocPoint = targetDocPoint; this.in = in; this.targetDocID = targetDocID; this.numDims = numDims; @@ -1293,7 +1298,29 @@ public String toString() { double yMax = Geo3DUtil.decodeValueCeil(NumericUtils.sortableBytesToInt(maxPackedValue, 1 * Integer.BYTES)); double zMin = Geo3DUtil.decodeValueFloor(NumericUtils.sortableBytesToInt(minPackedValue, 2 * Integer.BYTES)); double zMax = Geo3DUtil.decodeValueCeil(NumericUtils.sortableBytesToInt(maxPackedValue, 2 * Integer.BYTES)); - return "Cell(x=" + xMin + " TO " + xMax + " y=" + yMin + " TO " + yMax + " z=" + zMin + " TO " + zMax + ")"; + final XYZSolid xyzSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, xMin, xMax, yMin, yMax, zMin, zMax); + final int relationship = xyzSolid.getRelationship(shape); + final boolean pointWithinShape = shape.isWithin(targetDocPoint); + final boolean pointWithinCell = xyzSolid.isWithin(targetDocPoint); + final String relationshipString; + switch (relationship) { + case GeoArea.CONTAINS: + relationshipString = "CONTAINS"; + break; + case GeoArea.WITHIN: + relationshipString = "WITHIN"; + break; + case GeoArea.OVERLAPS: + relationshipString = "OVERLAPS"; + break; + case GeoArea.DISJOINT: + relationshipString = "DISJOINT"; + break; + default: + relationshipString = "UNKNOWN"; + break; + } + return "Cell(x=" + xMin + " TO " + xMax + " y=" + yMin + " TO " + yMax + " z=" + zMin + " TO " + zMax + "); Shape relationship = "+relationshipString+"; Point within cell = "+pointWithinCell+"; Point within shape = "+pointWithinShape; } @Override @@ -1313,7 +1340,7 @@ public int hashCode() { } } - public static String explain(String fieldName, GeoShape shape, IndexReader reader, int docID) throws Exception { + public static String explain(String fieldName, GeoShape shape, GeoPoint targetDocPoint, IndexReader reader, int docID) throws Exception { // First find the leaf reader that owns this doc: int subIndex = ReaderUtil.subIndex(docID, reader.leaves()); @@ -1323,7 +1350,7 @@ public static String explain(String fieldName, GeoShape shape, IndexReader reade b.append("target is in leaf " + leafReader + " of full reader " + reader + "\n"); DocIdSetBuilder hits = new DocIdSetBuilder(leafReader.maxDoc()); - ExplainingVisitor visitor = new ExplainingVisitor(new PointInShapeIntersectVisitor(hits, shape), docID - reader.leaves().get(subIndex).docBase, 3, Integer.BYTES, b); + ExplainingVisitor visitor = new ExplainingVisitor(shape, targetDocPoint, new PointInShapeIntersectVisitor(hits, shape), docID - reader.leaves().get(subIndex).docBase, 3, Integer.BYTES, b); // Do first phase, where we just figure out the "path" that leads to the target docID: leafReader.getPointValues().intersect(fieldName, visitor); From 65a69b9757a6cd0a8e8be0ed08797643054634b1 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Sat, 9 Apr 2016 16:12:39 -0400 Subject: [PATCH 0318/1113] LUCENE-7197: Fix two test failures and add more forensics that helped resolve the issue. --- .../spatial3d/geom/GeoConcavePolygon.java | 10 ++++---- .../spatial3d/geom/GeoConvexPolygon.java | 1 + .../lucene/spatial3d/geom/GeoPoint.java | 2 +- .../spatial3d/geom/GeoPolygonFactory.java | 12 +++++++-- .../lucene/spatial3d/geom/PlanetModel.java | 25 +++++++++++++++++++ .../spatial3d/geom/StandardXYZSolid.java | 18 ++++++++++++- .../lucene/spatial3d/TestGeo3DPoint.java | 25 ++++++++++++++----- 7 files changed, 78 insertions(+), 15 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 05e58ee5734e..8c6f75713806 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -202,11 +202,9 @@ public void done(final boolean isInternalReturnEdge) { throw new IllegalArgumentException("Polygon points are all coplanar"); } final GeoPoint check = points.get(endPointIndex); - // Here note the flip of the sense of the sided plane!! - final SidedPlane sp = new SidedPlane(check, false, start, end); //System.out.println("Created edge "+sp+" using start="+start+" end="+end+" check="+check); - edges[i] = sp; - invertedEdges[i] = new SidedPlane(sp); + edges[i] = new SidedPlane(check, false, start, end); + invertedEdges[i] = new SidedPlane(edges[i]); notableEdgePoints[i] = new GeoPoint[]{start, end}; } // In order to naively confirm that the polygon is concave, I would need to @@ -277,9 +275,11 @@ public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final M // cannot use them as bounds. They are independent hemispheres. for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { final SidedPlane edge = edges[edgeIndex]; + final SidedPlane invertedEdge = invertedEdges[edgeIndex]; final GeoPoint[] points = this.notableEdgePoints[edgeIndex]; if (!isInternalEdges.get(edgeIndex)) { - if (edge.intersects(planetModel, p, notablePoints, points, bounds, eitherBounds.get(edge))) { + //System.err.println("Checking concave edge "+edge+" for intersection against plane "+p); + if (invertedEdge.intersects(planetModel, p, notablePoints, points, bounds, eitherBounds.get(edge))) { //System.err.println(" intersects!"); return true; } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index d1e009178f38..b631b55a0595 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -265,6 +265,7 @@ public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final M final SidedPlane edge = edges[edgeIndex]; final GeoPoint[] points = this.notableEdgePoints[edgeIndex]; if (!isInternalEdges.get(edgeIndex)) { + //System.err.println("Checking convex edge "+edge+" for intersection against plane "+p); if (edge.intersects(planetModel, p, notablePoints, points, bounds, eitherBounds.get(edge))) { //System.err.println(" intersects!"); return true; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java index 31ab0aa2f0af..662a0565ce8b 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPoint.java @@ -188,6 +188,6 @@ public String toString() { if (this.longitude == Double.NEGATIVE_INFINITY) { return super.toString(); } - return "[lat="+getLatitude()+", lon="+getLongitude()+"]"; + return "[lat="+getLatitude()+", lon="+getLongitude()+"("+super.toString()+")]"; } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index a68f908dc225..7fc22dd558bd 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -395,6 +395,7 @@ public static boolean buildPolygonShape( if (confirmEdge == checkEdge) { continue; } + // Look for a point that is on the wrong side of the check edge. This means that we can't build the polygon. final GeoPoint thePoint; if (checkEdge.startPoint != confirmEdge.startPoint && checkEdge.endPoint != confirmEdge.startPoint && !flippedPlane.isWithin(confirmEdge.startPoint)) { thePoint = confirmEdge.startPoint; @@ -404,7 +405,11 @@ public static boolean buildPolygonShape( thePoint = null; } if (thePoint != null) { - // Found a split!! + // thePoint is on the wrong side of the complementary plane. That means we cannot build a concave polygon, because the complement would not + // be a legal convex polygon. + // But we can take advantage of the fact that the distance between the edge and thePoint is less than 180 degrees, and so we can split the + // would-be concave polygon into three segments. The first segment includes the edge and thePoint, and uses the sense of the edge to determine the sense + // of the polygon. // This should be the only problematic part of the polygon. // We know that thePoint is on the "wrong" side of the edge -- that is, it's on the side that the @@ -431,6 +436,8 @@ public static boolean buildPolygonShape( } //System.out.println("...done convex part."); + // ??? check if we get the sense right + // The part preceding the bad edge, back to thePoint, needs to be recursively // processed. So, assemble what we need, which is basically a list of edges. Edge loopEdge = edgeBuffer.getPrevious(checkEdge); @@ -459,6 +466,7 @@ public static boolean buildPolygonShape( return false; } //System.out.println("...done first part."); + final List secondPartPoints = new ArrayList<>(); final BitSet secondPartInternal = new BitSet(); loopEdge = edgeBuffer.getNext(checkEdge); @@ -479,7 +487,7 @@ public static boolean buildPolygonShape( secondPartInternal, secondPartPoints.size()-1, 0, - new SidedPlane(checkEdge.endPoint, true, checkEdge.startPoint, thePoint), + new SidedPlane(checkEdge.startPoint, false, checkEdge.endPoint, thePoint), holes, testPoint) == false) { return false; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java index c7d45a826c4f..f5ab8d86daae 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/PlanetModel.java @@ -189,6 +189,31 @@ public boolean pointOutside(final double x, final double y, final double z) { return (x * x + y * y) * inverseAb * inverseAb + z * z * inverseC * inverseC - 1.0 > Vector.MINIMUM_RESOLUTION; } + /** Compute a GeoPoint that's scaled to actually be on the planet surface. + * @param vector is the vector. + * @return the scaled point. + */ + public GeoPoint createSurfacePoint(final Vector vector) { + return createSurfacePoint(vector.x, vector.y, vector.z); + } + + /** Compute a GeoPoint that's based on (x,y,z) values, but is scaled to actually be on the planet surface. + * @param x is the x value. + * @param y is the y value. + * @param z is the z value. + * @return the scaled point. + */ + public GeoPoint createSurfacePoint(final double x, final double y, final double z) { + // The equation of the surface is: + // (x^2 / a^2 + y^2 / b^2 + z^2 / c^2) = 1 + // We will need to scale the passed-in x, y, z values: + // ((tx)^2 / a^2 + (ty)^2 / b^2 + (tz)^2 / c^2) = 1 + // t^2 * (x^2 / a^2 + y^2 / b^2 + z^2 / c^2) = 1 + // t = sqrt ( 1 / (x^2 / a^2 + y^2 / b^2 + z^2 / c^2)) + final double t = Math.sqrt(1.0 / (x*x*inverseAbSquared + y*y*inverseAbSquared + z*z*inverseCSquared)); + return new GeoPoint(t*x, t*y, t*z); + } + /** Compute a GeoPoint that's a bisection between two other GeoPoints. * @param pt1 is the first point. * @param pt2 is the second point. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java index ec2e26ce870c..9d94c519b90e 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardXYZSolid.java @@ -16,6 +16,8 @@ */ package org.apache.lucene.spatial3d.geom; +import java.util.Arrays; + /** * 3D rectangle, bounded on six sides by X,Y,Z limits * @@ -152,6 +154,11 @@ public StandardXYZSolid(final PlanetModel planetModel, notableMinZPoints = glueTogether(minXminZ, maxXminZ, minYminZ, maxYminZ); notableMaxZPoints = glueTogether(minXmaxZ, maxXmaxZ, minYmaxZ, maxYmaxZ); + //System.err.println( + // " notableMinXPoints="+Arrays.asList(notableMinXPoints)+" notableMaxXPoints="+Arrays.asList(notableMaxXPoints)+ + // " notableMinYPoints="+Arrays.asList(notableMinYPoints)+" notableMaxYPoints="+Arrays.asList(notableMaxYPoints)+ + // " notableMinZPoints="+Arrays.asList(notableMinZPoints)+" notableMaxZPoints="+Arrays.asList(notableMaxZPoints)); + // Now, compute the edge points. // This is the trickiest part of setting up an XYZSolid. We've computed intersections already, so // we'll start there. @@ -173,7 +180,11 @@ public StandardXYZSolid(final PlanetModel planetModel, final boolean maxXminYmaxZ = planetModel.pointOutside(maxX, minY, maxZ); final boolean maxXmaxYminZ = planetModel.pointOutside(maxX, maxY, minZ); final boolean maxXmaxYmaxZ = planetModel.pointOutside(maxX, maxY, maxZ); - + + //System.err.println("Outside world: minXminYminZ="+minXminYminZ+" minXminYmaxZ="+minXminYmaxZ+" minXmaxYminZ="+minXmaxYminZ+ + // " minXmaxYmaxZ="+minXmaxYmaxZ+" maxXminYminZ="+maxXminYminZ+" maxXminYmaxZ="+maxXminYmaxZ+" maxXmaxYminZ="+maxXmaxYminZ+ + // " maxXmaxYmaxZ="+maxXmaxYmaxZ); + // Look at single-plane/world intersections. // We detect these by looking at the world model and noting its x, y, and z bounds. @@ -286,6 +297,11 @@ public StandardXYZSolid(final PlanetModel planetModel, maxZEdges = EMPTY_POINTS; } + //System.err.println( + // " minXEdges="+Arrays.asList(minXEdges)+" maxXEdges="+Arrays.asList(maxXEdges)+ + // " minYEdges="+Arrays.asList(minYEdges)+" maxYEdges="+Arrays.asList(maxYEdges)+ + // " minZEdges="+Arrays.asList(minZEdges)+" maxZEdges="+Arrays.asList(maxZEdges)); + // Glue everything together. This is not a minimal set of edgepoints, as of now, but it does completely describe all shapes on the // planet. this.edgePoints = glueTogether(minXminY, minXmaxY, minXminZ, minXmaxZ, diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 4edfd2d91205..5c876ceebbbb 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -782,6 +782,7 @@ public void collect(int doc) { if (point != null) { boolean expected = ((deleted.contains(id) == false) && ((PointInGeo3DShapeQuery)query).getShape().isWithin(point)); if (hits.get(docID) != expected) { + GeoPoint scaledPoint = PlanetModel.WGS84.createSurfacePoint(point); StringBuilder b = new StringBuilder(); if (expected) { b.append("FAIL: id=" + id + " should have matched but did not\n"); @@ -789,10 +790,14 @@ public void collect(int doc) { b.append("FAIL: id=" + id + " should not have matched but did\n"); } b.append(" shape=" + ((PointInGeo3DShapeQuery)query).getShape() + "\n"); + b.append(" world bounds=(" + + " minX=" + PlanetModel.WGS84.getMinimumXValue() + " maxX=" + PlanetModel.WGS84.getMaximumXValue() + + " minY=" + PlanetModel.WGS84.getMinimumYValue() + " maxY=" + PlanetModel.WGS84.getMaximumYValue() + + " minZ=" + PlanetModel.WGS84.getMinimumZValue() + " maxZ=" + PlanetModel.WGS84.getMaximumZValue() + "\n"); b.append(" point=" + point + "\n"); b.append(" docID=" + docID + " deleted?=" + deleted.contains(id) + "\n"); b.append(" query=" + query + "\n"); - b.append(" explanation:\n " + explain("point", ((PointInGeo3DShapeQuery)query).getShape(), point, r, docID).replace("\n", "\n ")); + b.append(" explanation:\n " + explain("point", ((PointInGeo3DShapeQuery)query).getShape(), point, scaledPoint, r, docID).replace("\n", "\n ")); fail(b.toString()); } } else { @@ -810,7 +815,7 @@ public void testToString() { } public void testShapeQueryToString() { - assertEquals("PointInGeo3DShapeQuery: field=point: Shape: GeoStandardCircle: {planetmodel=PlanetModel.WGS84, center=[lat=0.7722082215479366, lon=0.13560747521073413], radius=0.1(5.729577951308232)}", + assertEquals("PointInGeo3DShapeQuery: field=point: Shape: GeoStandardCircle: {planetmodel=PlanetModel.WGS84, center=[lat=0.7722082215479366, lon=0.13560747521073413([X=0.7094263130137863, Y=0.09679758930862137, Z=0.6973564619248455])], radius=0.1(5.729577951308232)}", Geo3DPoint.newShapeQuery("point", GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, toRadians(44.244272), toRadians(7.769736), 0.1)).toString()); } @@ -1171,6 +1176,7 @@ private static class ExplainingVisitor implements IntersectVisitor { final GeoShape shape; final GeoPoint targetDocPoint; + final GeoPoint scaledDocPoint; final IntersectVisitor in; final List stack = new ArrayList<>(); private List stackToTargetDoc; @@ -1183,9 +1189,10 @@ private static class ExplainingVisitor implements IntersectVisitor { // In the first phase, we always return CROSSES to do a full scan of the BKD tree to see which leaf block the document lives in boolean firstPhase = true; - public ExplainingVisitor(GeoShape shape, GeoPoint targetDocPoint, IntersectVisitor in, int targetDocID, int numDims, int bytesPerDim, StringBuilder b) { + public ExplainingVisitor(GeoShape shape, GeoPoint targetDocPoint, GeoPoint scaledDocPoint, IntersectVisitor in, int targetDocID, int numDims, int bytesPerDim, StringBuilder b) { this.shape = shape; this.targetDocPoint = targetDocPoint; + this.scaledDocPoint = scaledDocPoint; this.in = in; this.targetDocID = targetDocID; this.numDims = numDims; @@ -1302,6 +1309,9 @@ public String toString() { final int relationship = xyzSolid.getRelationship(shape); final boolean pointWithinShape = shape.isWithin(targetDocPoint); final boolean pointWithinCell = xyzSolid.isWithin(targetDocPoint); + final boolean scaledWithinShape = shape.isWithin(scaledDocPoint); + final boolean scaledWithinCell = xyzSolid.isWithin(scaledDocPoint); + final String relationshipString; switch (relationship) { case GeoArea.CONTAINS: @@ -1320,7 +1330,10 @@ public String toString() { relationshipString = "UNKNOWN"; break; } - return "Cell(x=" + xMin + " TO " + xMax + " y=" + yMin + " TO " + yMax + " z=" + zMin + " TO " + zMax + "); Shape relationship = "+relationshipString+"; Point within cell = "+pointWithinCell+"; Point within shape = "+pointWithinShape; + return "Cell(x=" + xMin + " TO " + xMax + " y=" + yMin + " TO " + yMax + " z=" + zMin + " TO " + zMax + + "); Shape relationship = "+relationshipString+ + "; Point within cell = "+pointWithinCell+"; Point within shape = "+pointWithinShape+ + "; Scaled point within cell = "+scaledWithinCell+"; Scaled point within shape = "+scaledWithinShape; } @Override @@ -1340,7 +1353,7 @@ public int hashCode() { } } - public static String explain(String fieldName, GeoShape shape, GeoPoint targetDocPoint, IndexReader reader, int docID) throws Exception { + public static String explain(String fieldName, GeoShape shape, GeoPoint targetDocPoint, GeoPoint scaledDocPoint, IndexReader reader, int docID) throws Exception { // First find the leaf reader that owns this doc: int subIndex = ReaderUtil.subIndex(docID, reader.leaves()); @@ -1350,7 +1363,7 @@ public static String explain(String fieldName, GeoShape shape, GeoPoint targetDo b.append("target is in leaf " + leafReader + " of full reader " + reader + "\n"); DocIdSetBuilder hits = new DocIdSetBuilder(leafReader.maxDoc()); - ExplainingVisitor visitor = new ExplainingVisitor(shape, targetDocPoint, new PointInShapeIntersectVisitor(hits, shape), docID - reader.leaves().get(subIndex).docBase, 3, Integer.BYTES, b); + ExplainingVisitor visitor = new ExplainingVisitor(shape, targetDocPoint, scaledDocPoint, new PointInShapeIntersectVisitor(hits, shape), docID - reader.leaves().get(subIndex).docBase, 3, Integer.BYTES, b); // Do first phase, where we just figure out the "path" that leads to the target docID: leafReader.getPointValues().intersect(fieldName, visitor); From 301e178681d72a142dac4bc44416b93f42f33c01 Mon Sep 17 00:00:00 2001 From: yonik Date: Sat, 9 Apr 2016 19:10:02 -0400 Subject: [PATCH 0319/1113] SOLR-8922: optimize DocSetCollector to produce less garbage --- solr/CHANGES.txt | 5 ++ .../apache/solr/search/DocSetCollector.java | 84 +++++++++++++++++-- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a1a1fe0f84f8..975581c643d5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -87,6 +87,11 @@ Optimizations * SOLR-8856: Do not cache merge or 'read once' contexts in the hdfs block cache. (Mark Miller, Mike Drob) +* SOLR-8922: Optimize filter creation (DocSetCollector) to minimize the amount of garbage + produced. This resulted in up to 3x throughput when small filter creation was the bottleneck, + as well as orders of magnitude less garbage. (Jeff Wartes, yonik) + + Other Changes ---------------------- * SOLR-7516: Improve javadocs for JavaBinCodec, ObjectResolver and enforce the single-usage policy. diff --git a/solr/core/src/java/org/apache/solr/search/DocSetCollector.java b/solr/core/src/java/org/apache/solr/search/DocSetCollector.java index 8e529d925d46..25b12c54e619 100644 --- a/solr/core/src/java/org/apache/solr/search/DocSetCollector.java +++ b/solr/core/src/java/org/apache/solr/search/DocSetCollector.java @@ -17,6 +17,7 @@ package org.apache.solr.search; import java.io.IOException; +import java.util.ArrayList; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; @@ -37,7 +38,7 @@ public class DocSetCollector extends SimpleCollector { // in case there aren't that many hits, we may not want a very sparse // bit array. Optimistically collect the first few docs in an array // in case there are only a few. - final int[] scratch; + final ExpandingIntArray scratch; public DocSetCollector(int maxDoc) { this(DocSetUtil.smallSetSize(maxDoc), maxDoc); @@ -46,7 +47,7 @@ public DocSetCollector(int maxDoc) { public DocSetCollector(int smallSetSize, int maxDoc) { this.smallSetSize = smallSetSize; this.maxDoc = maxDoc; - this.scratch = new int[smallSetSize]; + this.scratch = new ExpandingIntArray(smallSetSize); } @Override @@ -59,8 +60,8 @@ public void collect(int doc) throws IOException { // than scanning through a potentially huge bit vector. // FUTURE: when search methods all start returning docs in order, maybe // we could have a ListDocSet() and use the collected array directly. - if (pos < scratch.length) { - scratch[pos]=doc; + if (pos < smallSetSize) { + scratch.add(pos, doc); } else { // this conditional could be removed if BitSet was preallocated, but that // would take up more memory, and add more GC time... @@ -72,12 +73,12 @@ public void collect(int doc) throws IOException { } public DocSet getDocSet() { - if (pos<=scratch.length) { + if (pos<=scratch.size()) { // assumes docs were collected in sorted order! - return new SortedIntDocSet(scratch, pos); + return new SortedIntDocSet(scratch.toArray(), pos); } else { // set the bits for ids that were collected in the array - for (int i=0; i arrays; + + public ExpandingIntArray(int smallSetSize) { + this.smallSetSize = smallSetSize; + this.currentAddArray = EMPTY; + } + + private void addNewArray() { + int arrSize = Math.max(10, currentAddArray.length << 1); + arrSize = Math.min(arrSize, smallSetSize - size); // max out at the smallSetSize + this.currentAddArray = new int[arrSize]; + if (arrays == null) { + arrays = new ArrayList<>(); + } + arrays.add(this.currentAddArray); + indexForNextAddInCurrentAddArray = 0; + // System.out.println("### ALLOCATED " + this + " " + arrSize + " smallSetSize="+smallSetSize + " left=" + (smallSetSize-size)); + } + + public void add(int index, int value) { + // assert index == size; // only appending is supported + if (indexForNextAddInCurrentAddArray >= currentAddArray.length) { + addNewArray(); + } + currentAddArray[indexForNextAddInCurrentAddArray++] = value; + size++; + } + + public void copyTo(FixedBitSet bits) { + if (size > 0) { + int resultPos = 0; + for (int i = 0; i < arrays.size(); i++) { + int[] srcArray = arrays.get(i); + int intsToCopy = (i < (arrays.size() - 1)) ? srcArray.length : indexForNextAddInCurrentAddArray; + for (int j = 0; j < intsToCopy; j++) { + bits.set(srcArray[j]); + } + resultPos += intsToCopy; + } + assert resultPos == size; + } + } + + public int[] toArray() { + int[] result = new int[size]; + if (size > 0) { + int resultPos = 0; + for (int i = 0; i < arrays.size(); i++) { + int[] srcArray = arrays.get(i); + int intsToCopy = (i < (arrays.size() - 1)) ? srcArray.length : indexForNextAddInCurrentAddArray; + System.arraycopy(srcArray, 0, result, resultPos, intsToCopy); + resultPos += intsToCopy; + } + assert resultPos == size; + } + return result; + } + + public int size() { + return size; + } + } } From 59e8c7b73c4c9225d1acf88d5011b18c128a0b5c Mon Sep 17 00:00:00 2001 From: yonik Date: Sat, 9 Apr 2016 19:21:27 -0400 Subject: [PATCH 0320/1113] tests: better DocSetCollector tests, including better smallSetSize boundary testing --- .../org/apache/solr/search/TestDocSet.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java index 3d93099cbc04..9c46d5baa57f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java @@ -55,7 +55,29 @@ public void setUp() throws Exception { super.setUp(); rand = random(); } - + + // test the DocSetCollector + public void collect(DocSet set, int maxDoc) { + int smallSetSize = maxDoc >> 64 + 3; + if (set.size() > 1) { + if (random().nextBoolean()) { + smallSetSize = set.size() + random().nextInt(3) - 1; // test the bounds around smallSetSize + } + } + DocSetCollector collector = new DocSetCollector(smallSetSize, maxDoc); + + for(DocIterator i1 = set.iterator(); i1.hasNext();) { + try { + collector.collect( i1.nextDoc() ); + } catch (IOException e) { + throw new RuntimeException(e); // should be impossible + } + } + + DocSet result = collector.getDocSet(); + iter(set, result); // check that they are equal + } + public FixedBitSet getRandomSet(int sz, int bitsToSet) { FixedBitSet bs = new FixedBitSet(sz); if (sz==0) return bs; @@ -165,6 +187,9 @@ protected void doSingle(int maxSize) { iter(a1,b1); iter(a2,b2); + collect(a1, maxSize); + collect(a2, maxSize); + FixedBitSet a_and = bs1.clone(); a_and.and(bs2); FixedBitSet a_or = bs1.clone(); a_or.or(bs2); // FixedBitSet a_xor = bs1.clone(); a_xor.xor(bs2); From f6be813308133de06e08717309750e1f47dd73d1 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Sun, 10 Apr 2016 05:11:12 -0400 Subject: [PATCH 0321/1113] LUCENE-7194: Roll our own toRadians() method, and also make it less likely we'll need to restaple the toString() tests. --- .../apache/lucene/spatial3d/Geo3DPoint.java | 10 ++++++---- .../lucene/spatial3d/TestGeo3DPoint.java | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java index 1e07aa715073..1a36c3e1c4bb 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/Geo3DPoint.java @@ -51,7 +51,9 @@ public final class Geo3DPoint extends Field { /** How many radians are in one earth surface meter */ - protected final static double RADIANS_PER_METER = 1.0 / PlanetModel.WGS84_MEAN; + public final static double RADIANS_PER_METER = 1.0 / PlanetModel.WGS84_MEAN; + /** How many radians are in one degree */ + public final static double RADIANS_PER_DEGREE = Math.PI / 180.0; /** Indexing {@link FieldType}. */ public static final FieldType TYPE = new FieldType(); @@ -75,12 +77,12 @@ public Geo3DPoint(String name, double latitude, double longitude) { } /** Converts degress to radians */ - protected static double fromDegrees(final double degrees) { - return Math.toRadians(degrees); + private static double fromDegrees(final double degrees) { + return degrees * RADIANS_PER_DEGREE; } /** Converts earth-surface meters to radians */ - protected static double fromMeters(final double meters) { + private static double fromMeters(final double meters) { return meters * RADIANS_PER_METER; } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 5c876ceebbbb..c80f3bbee972 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -132,7 +132,7 @@ public void testBasic() throws Exception { } private static double toRadians(double degrees) { - return Math.toRadians(degrees); + return degrees * Geo3DPoint.RADIANS_PER_DEGREE; } private static class Cell { @@ -810,13 +810,17 @@ public void collect(int doc) { } public void testToString() { + // Don't compare entire strings because Java 9 and Java 8 have slightly different values Geo3DPoint point = new Geo3DPoint("point", 44.244272, 7.769736); - assertEquals("Geo3DPoint polyPoints = new ArrayList<>(lats.length-1); for (int i = 0; i < lats.length - 1; i++) { - final GeoPoint newPoint = new GeoPoint(pm, Math.toRadians(lats[i]), Math.toRadians(lons[i])); + final GeoPoint newPoint = new GeoPoint(pm, toRadians(lats[i]), toRadians(lons[i])); if (!outsidePolygon.isWithin(newPoint)) { return false; } From 70ab80b341399651bb130176d87a9b25b4617a1e Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Sun, 10 Apr 2016 08:58:19 -0400 Subject: [PATCH 0322/1113] LUCENE-7199: Use a more refined algorithm for picking the random pole used in clockwise/counterclockwise determination. --- .../spatial3d/geom/GeoPolygonFactory.java | 62 ++++++++++++++++++- .../lucene/spatial3d/TestGeo3DPoint.java | 11 +--- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java index 7fc22dd558bd..4b7f4f405e9e 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPolygonFactory.java @@ -107,15 +107,16 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, // Create a random number generator. Effectively this furnishes us with a repeatable sequence // of points to use for poles. final Random generator = new Random(1234); + //int counter = 0; while (true) { + //counter++; // Pick the next random pole - final double poleLat = generator.nextDouble() * Math.PI - Math.PI * 0.5; - final double poleLon = generator.nextDouble() * Math.PI * 2.0 - Math.PI; - final GeoPoint pole = new GeoPoint(planetModel, poleLat, poleLon); + final GeoPoint pole = pickPole(generator, planetModel, pointList); // Is it inside or outside? final Boolean isPoleInside = isInsidePolygon(pole, pointList); if (isPoleInside != null) { // Legal pole + //System.out.println("Took "+counter+" iterations to find pole"); //System.out.println("Pole = "+pole+"; isInside="+isPoleInside+"; pointList = "+pointList); return makeGeoPolygon(planetModel, pointList, holes, pole, isPoleInside); } @@ -169,6 +170,61 @@ public static GeoPolygon makeGeoPolygon(final PlanetModel planetModel, } } + /** The maximum distance from the close point to the trial pole: 2 degrees */ + private final static double MAX_POLE_DISTANCE = Math.PI * 2.0 / 180.0; + + /** Pick a random pole that has a good chance of being inside the polygon described by the points. + * @param generator is the random number generator to use. + * @param planetModel is the planet model to use. + * @param points is the list of points available. + * @return the randomly-determined pole selection. + */ + private static GeoPoint pickPole(final Random generator, final PlanetModel planetModel, final List points) { + final int pointIndex = generator.nextInt(points.size()); + final GeoPoint closePoint = points.get(pointIndex); + // We pick a random angle and random arc distance, then generate a point based on closePoint + final double angle = generator.nextDouble() * Math.PI * 2.0 - Math.PI; + final double arcDistance = MAX_POLE_DISTANCE - generator.nextDouble() * MAX_POLE_DISTANCE; + // We come up with a unit circle (x,y,z) coordinate given the random angle and arc distance. The point is centered around the positive x axis. + final double x = Math.cos(arcDistance); + final double sinArcDistance = Math.sin(arcDistance); + final double y = Math.cos(angle) * sinArcDistance; + final double z = Math.sin(angle) * sinArcDistance; + // Now, use closePoint for a rotation pole + final double sinLatitude = Math.sin(closePoint.getLatitude()); + final double cosLatitude = Math.cos(closePoint.getLatitude()); + final double sinLongitude = Math.sin(closePoint.getLongitude()); + final double cosLongitude = Math.cos(closePoint.getLongitude()); + // This transformation should take the point (1,0,0) and transform it to the closepoint's actual (x,y,z) coordinates. + // Coordinate rotation formula: + // x1 = x0 cos T - y0 sin T + // y1 = x0 sin T + y0 cos T + // We're in essence undoing the following transformation (from GeoPolygonFactory): + // x1 = x0 cos az + y0 sin az + // y1 = - x0 sin az + y0 cos az + // z1 = z0 + // x2 = x1 cos al + z1 sin al + // y2 = y1 + // z2 = - x1 sin al + z1 cos al + // So, we reverse the order of the transformations, AND we transform backwards. + // Transforming backwards means using these identities: sin(-angle) = -sin(angle), cos(-angle) = cos(angle) + // So: + // x1 = x0 cos al - z0 sin al + // y1 = y0 + // z1 = x0 sin al + z0 cos al + // x2 = x1 cos az - y1 sin az + // y2 = x1 sin az + y1 cos az + // z2 = z1 + final double x1 = x * cosLatitude - z * sinLatitude; + final double y1 = y; + final double z1 = x * sinLatitude + z * cosLatitude; + final double x2 = x1 * cosLongitude - y1 * sinLongitude; + final double y2 = x1 * sinLongitude + y1 * cosLongitude; + final double z2 = z1; + // Finally, scale to put the point on the surface + return planetModel.createSurfacePoint(x2, y2, z2); + } + /** For a specified point and a list of poly points, determine based on point order whether the * point should be considered in or out of the polygon. * @param point is the point to check. diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index c80f3bbee972..9ec0e30661f4 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -1023,15 +1023,8 @@ protected static GeoPoint createPoint(final PlanetModel pm, final double x2 = x1 * cosLongitude - y1 * sinLongitude; final double y2 = x1 * sinLongitude + y1 * cosLongitude; final double z2 = z1; - - // Scale final (x,y,z) to land on planet surface - // Equation of ellipsoid: x^2 / a^2 + y^2 / b^2 + z^2 / c^2 - 1 = 0 - // Use a parameterization, e.g. x = t * x2, y = t * y2, z = t * z2, and find t. - // t^2 ( x2^2 / a^2 + y2^2 / b^2 + z2^2 / c^2 ) = 1 - // t = +/- sqrt( 1 / ( x2^2 / a^2 + y2^2 / b^2 + z2^2 / c^2 ) ) - // We want the + variant because we're scaling in the same direction as the original vector. - final double t = Math.sqrt( 1.0 / (x2 * x2 * pm.inverseAbSquared + y2 * y2 * pm.inverseAbSquared + z2 * z2 * pm.inverseCSquared)); - return new GeoPoint(x2 * t, y2 * t, z2 * t); + // Scale to put the point on the surface + return pm.createSurfacePoint(x2, y2, z2); } protected static boolean verifyPolygon(final PlanetModel pm, final Polygon polygon, final GeoPolygon outsidePolygon) { From f18cb90104a0e55f91ce75d726cb489eb5cbf258 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 11 Apr 2016 09:20:39 +0200 Subject: [PATCH 0323/1113] LUCENE-7187: Block join queries' Weight#extractTerms(...) implementations should delegate to the wrapped weight. --- lucene/CHANGES.txt | 3 ++ .../search/join/ToChildBlockJoinQuery.java | 4 ++- .../search/join/ToParentBlockJoinQuery.java | 6 ++-- .../lucene/search/join/TestBlockJoin.java | 32 ++++++++++++++++++- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index f28254dc460c..63a322542975 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -66,6 +66,9 @@ Bug Fixes * LUCENE-7168: Switch to stable encode for geo3d, remove quantization test leniency, remove dead code (Mike McCandless) +* LUCENE-7187: Block join queries' Weight#extractTerms(...) implementations + should delegate to the wrapped weight. (Martijn van Groningen) + Other * LUCENE-7174: Upgrade randomizedtesting to 2.3.4. (Uwe Schindler, Dawid Weiss) diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java index d381caeaa6ee..237b78696d2f 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java @@ -103,7 +103,9 @@ public ToChildBlockJoinWeight(Query joinQuery, Weight parentWeight, BitSetProduc } @Override - public void extractTerms(Set terms) {} + public void extractTerms(Set terms) { + parentWeight.extractTerms(terms); + } @Override public float getValueForNormalization() throws IOException { diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java index f004fde97356..2a0ebaf2ba78 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java @@ -125,21 +125,21 @@ public Query getChildQuery() { } private static class BlockJoinWeight extends Weight { - private final Query joinQuery; private final Weight childWeight; private final BitSetProducer parentsFilter; private final ScoreMode scoreMode; public BlockJoinWeight(Query joinQuery, Weight childWeight, BitSetProducer parentsFilter, ScoreMode scoreMode) { super(joinQuery); - this.joinQuery = joinQuery; this.childWeight = childWeight; this.parentsFilter = parentsFilter; this.scoreMode = scoreMode; } @Override - public void extractTerms(Set terms) {} + public void extractTerms(Set terms) { + childWeight.extractTerms(terms); + } @Override public float getValueForNormalization() throws IOException { diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index 01a81351511f..a67b0ec28bdf 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -20,8 +20,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -101,7 +103,35 @@ private Document makeQualification(String qualification, int year) { job.add(new IntPoint("year", year)); return job; } - + + public void testExtractTerms() throws Exception { + TermQuery termQuery = new TermQuery(new Term("field", "value")); + QueryBitSetProducer bitSetProducer = new QueryBitSetProducer(new MatchNoDocsQuery()); + ToParentBlockJoinQuery toParentBlockJoinQuery = new ToParentBlockJoinQuery(termQuery, bitSetProducer, ScoreMode.None); + ToChildBlockJoinQuery toChildBlockJoinQuery = new ToChildBlockJoinQuery(toParentBlockJoinQuery, bitSetProducer); + + Directory directory = newDirectory(); + final IndexWriter w = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random()))); + w.close(); + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + + Weight weight = toParentBlockJoinQuery.createWeight(indexSearcher, false); + Set terms = new HashSet<>(); + weight.extractTerms(terms); + Term[] termArr =terms.toArray(new Term[0]); + assertEquals(1, termArr.length); + + weight = toChildBlockJoinQuery.createWeight(indexSearcher, false); + terms = new HashSet<>(); + weight.extractTerms(terms); + termArr =terms.toArray(new Term[0]); + assertEquals(1, termArr.length); + + indexReader.close(); + directory.close(); + } + public void testEmptyChildFilter() throws Exception { final Directory dir = newDirectory(); final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random())); From ff3661abdaa5b4ae33496dc378ae917f7a7a723b Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 8 Apr 2016 16:59:41 +0200 Subject: [PATCH 0324/1113] LUCENE-7196 - guaranteed class coverage in split indexes through grouping by class (cherry picked from commit 112078e) --- lucene/classification/build.xml | 6 +- .../classification/utils/DatasetSplitter.java | 126 ++++++++++++------ .../utils/DataSplitterTest.java | 33 ++--- 3 files changed, 106 insertions(+), 59 deletions(-) diff --git a/lucene/classification/build.xml b/lucene/classification/build.xml index 930b1fa80bdb..fd15239fd180 100644 --- a/lucene/classification/build.xml +++ b/lucene/classification/build.xml @@ -27,6 +27,7 @@ + @@ -35,15 +36,16 @@ - + - + diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java index 0b03b94f56d3..fce786bf1e99 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java @@ -18,6 +18,7 @@ import java.io.IOException; +import java.util.HashMap; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; @@ -28,11 +29,16 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.grouping.GroupDocs; +import org.apache.lucene.search.grouping.GroupingSearch; +import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.store.Directory; +import org.apache.lucene.uninverting.UninvertingReader; /** * Utility class for creating training / test / cross validation indexes from the original index. @@ -61,67 +67,78 @@ public DatasetSplitter(double testRatio, double crossValidationRatio) { * @param testIndex a {@link Directory} used to write the test index * @param crossValidationIndex a {@link Directory} used to write the cross validation index * @param analyzer {@link Analyzer} used to create the new docs + * @param termVectors {@code true} if term vectors should be kept + * @param classFieldName names of the field used as the label for classification * @param fieldNames names of fields that need to be put in the new indexes or null if all should be used * @throws IOException if any writing operation fails on any of the indexes */ public void split(LeafReader originalIndex, Directory trainingIndex, Directory testIndex, Directory crossValidationIndex, - Analyzer analyzer, String... fieldNames) throws IOException { + Analyzer analyzer, boolean termVectors, String classFieldName, String... fieldNames) throws IOException { // create IWs for train / test / cv IDXs IndexWriter testWriter = new IndexWriter(testIndex, new IndexWriterConfig(analyzer)); IndexWriter cvWriter = new IndexWriter(crossValidationIndex, new IndexWriterConfig(analyzer)); IndexWriter trainingWriter = new IndexWriter(trainingIndex, new IndexWriterConfig(analyzer)); + // try to get the exact no. of existing classes + Terms terms = originalIndex.terms(classFieldName); + long noOfClasses = -1; + if (terms != null) { + noOfClasses = terms.size(); + + } + if (noOfClasses == -1) { + noOfClasses = 10000; // fallback + } + + HashMap mapping = new HashMap<>(); + mapping.put(classFieldName, UninvertingReader.Type.SORTED); + UninvertingReader uninvertingReader = new UninvertingReader(originalIndex, mapping); + try { - int size = originalIndex.maxDoc(); - IndexSearcher indexSearcher = new IndexSearcher(originalIndex); - TopDocs topDocs = indexSearcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); + IndexSearcher indexSearcher = new IndexSearcher(uninvertingReader); + GroupingSearch gs = new GroupingSearch(classFieldName); + gs.setGroupSort(Sort.INDEXORDER); + gs.setSortWithinGroup(Sort.INDEXORDER); + gs.setAllGroups(true); + gs.setGroupDocsLimit(originalIndex.maxDoc()); + TopGroups topGroups = gs.search(indexSearcher, new MatchAllDocsQuery(), 0, (int) noOfClasses); // set the type to be indexed, stored, with term vectors FieldType ft = new FieldType(TextField.TYPE_STORED); - ft.setStoreTermVectors(true); - ft.setStoreTermVectorOffsets(true); - ft.setStoreTermVectorPositions(true); + if (termVectors) { + ft.setStoreTermVectors(true); + ft.setStoreTermVectorOffsets(true); + ft.setStoreTermVectorPositions(true); + } int b = 0; // iterate over existing documents - for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - - // create a new document for indexing - Document doc = new Document(); - Document document = originalIndex.document(scoreDoc.doc); - if (fieldNames != null && fieldNames.length > 0) { - for (String fieldName : fieldNames) { - IndexableField field = document.getField(fieldName); - if (field != null) { - doc.add(new Field(fieldName, field.stringValue(), ft)); - } + for (GroupDocs group : topGroups.groups) { + int totalHits = group.totalHits; + double testSize = totalHits * testRatio; + int tc = 0; + double cvSize = totalHits * crossValidationRatio; + int cvc = 0; + for (ScoreDoc scoreDoc : group.scoreDocs) { + + // create a new document for indexing + Document doc = createNewDoc(originalIndex, ft, scoreDoc, fieldNames); + + // add it to one of the IDXs + if (b % 2 == 0 && tc < testSize) { + testWriter.addDocument(doc); + tc++; + } else if (cvc < cvSize) { + cvWriter.addDocument(doc); + cvc++; + } else { + trainingWriter.addDocument(doc); } - } else { - for (IndexableField field : document.getFields()) { - if (field.readerValue() != null) { - doc.add(new Field(field.name(), field.readerValue(), ft)); - } else if (field.binaryValue() != null) { - doc.add(new Field(field.name(), field.binaryValue(), ft)); - } else if (field.stringValue() != null) { - doc.add(new Field(field.name(), field.stringValue(), ft)); - } else if (field.numericValue() != null) { - doc.add(new Field(field.name(), field.numericValue().toString(), ft)); - } - } - } - - // add it to one of the IDXs - if (b % 2 == 0 && testWriter.maxDoc() < size * testRatio) { - testWriter.addDocument(doc); - } else if (cvWriter.maxDoc() < size * crossValidationRatio) { - cvWriter.addDocument(doc); - } else { - trainingWriter.addDocument(doc); + b++; } - b++; } // commit testWriter.commit(); @@ -139,7 +156,34 @@ public void split(LeafReader originalIndex, Directory trainingIndex, Directory t testWriter.close(); cvWriter.close(); trainingWriter.close(); + uninvertingReader.close(); + } + } + + private Document createNewDoc(LeafReader originalIndex, FieldType ft, ScoreDoc scoreDoc, String[] fieldNames) throws IOException { + Document doc = new Document(); + Document document = originalIndex.document(scoreDoc.doc); + if (fieldNames != null && fieldNames.length > 0) { + for (String fieldName : fieldNames) { + IndexableField field = document.getField(fieldName); + if (field != null) { + doc.add(new Field(fieldName, field.stringValue(), ft)); + } + } + } else { + for (IndexableField field : document.getFields()) { + if (field.readerValue() != null) { + doc.add(new Field(field.name(), field.readerValue(), ft)); + } else if (field.binaryValue() != null) { + doc.add(new Field(field.name(), field.binaryValue(), ft)); + } else if (field.stringValue() != null) { + doc.add(new Field(field.name(), field.stringValue(), ft)); + } else if (field.numericValue() != null) { + doc.add(new Field(field.name(), field.numericValue().toString(), ft)); + } + } } + return doc; } } diff --git a/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java b/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java index 2984bb55d450..0b6f077cdfd6 100644 --- a/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java +++ b/lucene/classification/src/test/org/apache/lucene/classification/utils/DataSplitterTest.java @@ -17,27 +17,28 @@ package org.apache.lucene.classification.utils; -import org.apache.lucene.analysis.Analyzer; +import java.io.IOException; +import java.util.Random; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.Random; - /** * Testcase for {@link org.apache.lucene.classification.utils.DatasetSplitter} */ @@ -47,9 +48,9 @@ public class DataSplitterTest extends LuceneTestCase { private RandomIndexWriter indexWriter; private Directory dir; - private String textFieldName = "text"; - private String classFieldName = "class"; - private String idFieldName = "id"; + private static final String textFieldName = "text"; + private static final String classFieldName = "class"; + private static final String idFieldName = "id"; @Override @Before @@ -65,11 +66,11 @@ public void setUp() throws Exception { Document doc; Random rnd = random(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 1000; i++) { doc = new Document(); - doc.add(new Field(idFieldName, Integer.toString(i), ft)); + doc.add(new Field(idFieldName, "id" + Integer.toString(i), ft)); doc.add(new Field(textFieldName, TestUtil.randomUnicodeString(rnd, 1024), ft)); - doc.add(new Field(classFieldName, TestUtil.randomUnicodeString(rnd, 10), ft)); + doc.add(new Field(classFieldName, Integer.toString(rnd.nextInt(10)), ft)); indexWriter.addDocument(doc); } @@ -108,18 +109,18 @@ public static void assertSplit(LeafReader originalIndex, double testRatio, doubl try { DatasetSplitter datasetSplitter = new DatasetSplitter(testRatio, crossValidationRatio); - datasetSplitter.split(originalIndex, trainingIndex, testIndex, crossValidationIndex, new MockAnalyzer(random()), fieldNames); + datasetSplitter.split(originalIndex, trainingIndex, testIndex, crossValidationIndex, new MockAnalyzer(random()), true, classFieldName, fieldNames); assertNotNull(trainingIndex); assertNotNull(testIndex); assertNotNull(crossValidationIndex); DirectoryReader trainingReader = DirectoryReader.open(trainingIndex); - assertTrue((int) (originalIndex.maxDoc() * (1d - testRatio - crossValidationRatio)) == trainingReader.maxDoc()); + assertEquals((int) (originalIndex.maxDoc() * (1d - testRatio - crossValidationRatio)), trainingReader.maxDoc(), 20); DirectoryReader testReader = DirectoryReader.open(testIndex); - assertTrue((int) (originalIndex.maxDoc() * testRatio) == testReader.maxDoc()); + assertEquals((int) (originalIndex.maxDoc() * testRatio), testReader.maxDoc(), 20); DirectoryReader cvReader = DirectoryReader.open(crossValidationIndex); - assertTrue((int) (originalIndex.maxDoc() * crossValidationRatio) == cvReader.maxDoc()); + assertEquals((int) (originalIndex.maxDoc() * crossValidationRatio), cvReader.maxDoc(), 20); trainingReader.close(); testReader.close(); From 4b20575cfed510f776df023d319655816904d597 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 11 Apr 2016 10:03:37 +0200 Subject: [PATCH 0325/1113] LUCENE-7196 - added missing misc dependency (cherry picked from commit 273d586) --- lucene/classification/build.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lucene/classification/build.xml b/lucene/classification/build.xml index fd15239fd180..c56d48123fa6 100644 --- a/lucene/classification/build.xml +++ b/lucene/classification/build.xml @@ -28,6 +28,7 @@ + @@ -36,16 +37,17 @@ - + - + From 2bf3d70bf105021ec4a4de45202ab468a3cd934d Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 11 Apr 2016 05:24:34 -0400 Subject: [PATCH 0326/1113] fix rare test bug --- .../lucene/facet/range/TestRangeFacetCounts.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index 626d772d6a9e..18645e75fbbb 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -22,14 +22,14 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.facet.DrillDownQuery; -import org.apache.lucene.facet.DrillSideways; import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult; +import org.apache.lucene.facet.DrillSideways; import org.apache.lucene.facet.FacetField; import org.apache.lucene.facet.FacetResult; import org.apache.lucene.facet.FacetTestCase; @@ -59,6 +59,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.TestUtil; public class TestRangeFacetCounts extends FacetTestCase { @@ -532,7 +533,11 @@ public void testRandomDoubles() throws Exception { boolean minIncl; boolean maxIncl; - if (min == max) { + + long minAsLong = NumericUtils.doubleToSortableLong(min); + long maxAsLong = NumericUtils.doubleToSortableLong(max); + // NOTE: maxAsLong - minAsLong >= 0 is here to handle the common overflow case! + if (maxAsLong - minAsLong >= 0 && maxAsLong - minAsLong < 2) { minIncl = true; maxIncl = true; } else { From 4c984af8b51506597a88a51dff7d3a5dc85a5aa9 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 11 Apr 2016 05:42:59 -0400 Subject: [PATCH 0327/1113] fix silly stack-building test bug in BKD visitor forensics --- .../src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 9ec0e30661f4..20302b408640 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -1248,7 +1248,7 @@ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { if (firstPhase) { // Pop stack: - while (stack.size() > 0 && stack.get(stack.size()-1).contains(cell)) { + while (stack.size() > 0 && stack.get(stack.size()-1).contains(cell) == false) { stack.remove(stack.size()-1); //System.out.println(" pop"); } From 9cbe3e761b6e79acf62b8c6117704ce1656809b2 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Mon, 11 Apr 2016 05:45:42 -0400 Subject: [PATCH 0328/1113] LUCENE-7189: use fewer sampling points for small earth-surface rectangles --- .../src/java/org/apache/lucene/geo/EarthDebugger.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java index 14956f2c1675..6c06c94a084b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java @@ -62,9 +62,14 @@ public void addPolygon(Polygon poly, String color) { } } + private static double MAX_LAT_LON_PER_STEP = 5.0; + // first point is inclusive, last point is exclusive! private void drawSegment(double minLat, double maxLat, double minLon, double maxLon) { - int steps = 20; + int steps = (int) Math.round(Math.max(Math.abs(maxLat-minLat)/MAX_LAT_LON_PER_STEP, Math.abs(maxLon-minLon)/MAX_LAT_LON_PER_STEP)); + if (steps < 1) { + steps = 1; + } for(int i=0;i max lat, min lon\n"); From 168e2d8406c32a65655b039add12902d431fb973 Mon Sep 17 00:00:00 2001 From: Varun Thacker Date: Mon, 11 Apr 2016 17:17:03 +0530 Subject: [PATCH 0329/1113] SOLR-8967: UI should not show the replication tab in the core selector panel in cloud mode --- solr/CHANGES.txt | 3 +++ solr/webapp/web/index.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 975581c643d5..6a0ace4fa1d1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -114,6 +114,9 @@ Other Changes * SOLR-8097: Implement builder pattern design for constructing SolrJ clients and also deprecate direct construction of client objects. (Jason Gerlowski, Shawn Heisey, Anshum Gupta) +* SOLR-8967: In SolrCloud mode, under the 'Core Selector' dropdown in the UI the Replication tab won't be displayed + anymore. The Replication tab is only beneficial to users running Solr in master-slave mode. (Varun Thacker) + ================== 6.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html index 6c4df6afe86c..a8567a584232 100644 --- a/solr/webapp/web/index.html +++ b/solr/webapp/web/index.html @@ -209,7 +209,7 @@

    Connection recovered...

  • Ping ({{pingMS}}ms)
  • Plugins / Stats
  • Query
    • -
  • Replication
    • +
  • Replication
  • Schema
  • Segments info
    • From b21218e79f2b431b3311a0a07dfbcdb7ae48f1ab Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 11 Apr 2016 10:36:10 -0400 Subject: [PATCH 0330/1113] LUCENE-7185: fix random number generation used for spatial tests. Note that GeoPoint tests are still on the old RNG as we haven't get made them happy. --- .../geopoint/search/GeoPointTestUtil.java | 287 ++++++++++++++++++ .../geopoint/search/TestGeoPointQuery.java | 42 +++ .../search/TestLegacyGeoPointQuery.java | 43 ++- .../spatial/util/BaseGeoPointTestCase.java | 79 +++-- .../org/apache/lucene/geo/GeoTestUtil.java | 46 ++- 5 files changed, 469 insertions(+), 28 deletions(-) create mode 100644 lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/GeoPointTestUtil.java diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/GeoPointTestUtil.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/GeoPointTestUtil.java new file mode 100644 index 000000000000..a8c4271e42f8 --- /dev/null +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/GeoPointTestUtil.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.spatial.geopoint.search; + +import java.util.ArrayList; +import java.util.Random; + +import org.apache.lucene.geo.GeoUtils; +import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; + +import com.carrotsearch.randomizedtesting.RandomizedContext; + +// HACK: this is a wimpier version of GeoTestUtil used for now until we can +// get all tests passing with new random number generator! + +final class GeoPointTestUtil { + + /** returns next pseudorandom latitude (anywhere) */ + public static double nextLatitude() { + return -90 + 180.0 * random().nextDouble(); + } + + /** returns next pseudorandom longitude (anywhere) */ + public static double nextLongitude() { + return -180 + 360.0 * random().nextDouble(); + } + + /** returns next pseudorandom latitude, kinda close to {@code otherLatitude} */ + public static double nextLatitudeNear(double otherLatitude) { + GeoUtils.checkLatitude(otherLatitude); + return normalizeLatitude(otherLatitude + random().nextDouble() - 0.5); + } + + /** returns next pseudorandom longitude, kinda close to {@code otherLongitude} */ + public static double nextLongitudeNear(double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + return normalizeLongitude(otherLongitude + random().nextDouble() - 0.5); + } + + /** + * returns next pseudorandom latitude, kinda close to {@code minLatitude/maxLatitude} + * NOTE:minLatitude/maxLatitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLatitudeAround(double minLatitude, double maxLatitude) { + GeoUtils.checkLatitude(minLatitude); + GeoUtils.checkLatitude(maxLatitude); + return normalizeLatitude(randomRangeMaybeSlightlyOutside(minLatitude, maxLatitude)); + } + + /** + * returns next pseudorandom longitude, kinda close to {@code minLongitude/maxLongitude} + * NOTE:minLongitude/maxLongitude are merely guidelines. the returned value is sometimes + * outside of that range! this is to facilitate edge testing. + */ + public static double nextLongitudeAround(double minLongitude, double maxLongitude) { + GeoUtils.checkLongitude(minLongitude); + GeoUtils.checkLongitude(maxLongitude); + return normalizeLongitude(randomRangeMaybeSlightlyOutside(minLongitude, maxLongitude)); + } + + /** returns next pseudorandom box: can cross the 180th meridian */ + public static Rectangle nextBox() { + return nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), true); + } + + /** returns next pseudorandom box: will not cross the 180th meridian */ + public static Rectangle nextSimpleBox() { + return nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), false); + } + + /** returns next pseudorandom box, can cross the 180th meridian, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static Rectangle nextBoxNear(double otherLatitude, double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + GeoUtils.checkLongitude(otherLongitude); + return nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), true); + } + + /** returns next pseudorandom box, will not cross the 180th meridian, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static Rectangle nextSimpleBoxNear(double otherLatitude, double otherLongitude) { + GeoUtils.checkLongitude(otherLongitude); + GeoUtils.checkLongitude(otherLongitude); + return nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), false); + } + + /** returns next pseudorandom polygon */ + public static Polygon nextPolygon() { + if (random().nextBoolean()) { + return surpriseMePolygon(null, null); + } + + Rectangle box = nextBoxInternal(nextLatitude(), nextLatitude(), nextLongitude(), nextLongitude(), false); + if (random().nextBoolean()) { + // box + return boxPolygon(box); + } else { + // triangle + return trianglePolygon(box); + } + } + + /** returns next pseudorandom polygon, kinda close to {@code otherLatitude} and {@code otherLongitude} */ + public static Polygon nextPolygonNear(double otherLatitude, double otherLongitude) { + if (random().nextBoolean()) { + return surpriseMePolygon(otherLatitude, otherLongitude); + } + + Rectangle box = nextBoxInternal(nextLatitudeNear(otherLatitude), nextLatitudeNear(otherLatitude), + nextLongitudeNear(otherLongitude), nextLongitudeNear(otherLongitude), false); + if (random().nextBoolean()) { + // box + return boxPolygon(box); + } else { + // triangle + return trianglePolygon(box); + } + } + + private static Rectangle nextBoxInternal(double lat0, double lat1, double lon0, double lon1, boolean canCrossDateLine) { + if (lat1 < lat0) { + double x = lat0; + lat0 = lat1; + lat1 = x; + } + + if (canCrossDateLine == false && lon1 < lon0) { + double x = lon0; + lon0 = lon1; + lon1 = x; + } + + return new Rectangle(lat0, lat1, lon0, lon1); + } + + private static Polygon boxPolygon(Rectangle box) { + assert box.crossesDateline() == false; + final double[] polyLats = new double[5]; + final double[] polyLons = new double[5]; + polyLats[0] = box.minLat; + polyLons[0] = box.minLon; + polyLats[1] = box.maxLat; + polyLons[1] = box.minLon; + polyLats[2] = box.maxLat; + polyLons[2] = box.maxLon; + polyLats[3] = box.minLat; + polyLons[3] = box.maxLon; + polyLats[4] = box.minLat; + polyLons[4] = box.minLon; + return new Polygon(polyLats, polyLons); + } + + private static Polygon trianglePolygon(Rectangle box) { + assert box.crossesDateline() == false; + final double[] polyLats = new double[4]; + final double[] polyLons = new double[4]; + polyLats[0] = box.minLat; + polyLons[0] = box.minLon; + polyLats[1] = box.maxLat; + polyLons[1] = box.minLon; + polyLats[2] = box.maxLat; + polyLons[2] = box.maxLon; + polyLats[3] = box.minLat; + polyLons[3] = box.minLon; + return new Polygon(polyLats, polyLons); + } + + private static Polygon surpriseMePolygon(Double otherLatitude, Double otherLongitude) { + // repeat until we get a poly that doesn't cross dateline: + newPoly: + while (true) { + //System.out.println("\nPOLY ITER"); + final double centerLat; + final double centerLon; + if (otherLatitude == null) { + centerLat = nextLatitude(); + centerLon = nextLongitude(); + } else { + GeoUtils.checkLatitude(otherLatitude); + GeoUtils.checkLongitude(otherLongitude); + centerLat = nextLatitudeNear(otherLatitude); + centerLon = nextLongitudeNear(otherLongitude); + } + + double radius = 0.1 + 20 * random().nextDouble(); + double radiusDelta = random().nextDouble(); + + ArrayList lats = new ArrayList<>(); + ArrayList lons = new ArrayList<>(); + double angle = 0.0; + while (true) { + angle += random().nextDouble()*40.0; + //System.out.println(" angle " + angle); + if (angle > 360) { + break; + } + double len = radius * (1.0 - radiusDelta + radiusDelta * random().nextDouble()); + //System.out.println(" len=" + len); + double lat = centerLat + len * Math.cos(Math.toRadians(angle)); + double lon = centerLon + len * Math.sin(Math.toRadians(angle)); + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + if (lat > 90) { + // cross the north pole + lat = 180 - lat; + lon = 180 - lon; + } else if (lat < -90) { + // cross the south pole + lat = -180 - lat; + lon = 180 - lon; + } + if (lon <= GeoUtils.MIN_LON_INCL || lon >= GeoUtils.MAX_LON_INCL) { + // cannot cross dateline: try again! + continue newPoly; + } + lats.add(lat); + lons.add(lon); + + //System.out.println(" lat=" + lats.get(lats.size()-1) + " lon=" + lons.get(lons.size()-1)); + } + + // close it + lats.add(lats.get(0)); + lons.add(lons.get(0)); + + double[] latsArray = new double[lats.size()]; + double[] lonsArray = new double[lons.size()]; + for(int i=0;i= -90 && latitude <= 90) { + return latitude; //common case, and avoids slight double precision shifting + } + double off = Math.abs((latitude + 90) % 360); + return (off <= 180 ? off : 360-off) - 90; + } + + /** Puts longitude in range of -180 to +180. */ + private static double normalizeLongitude(double longitude) { + if (longitude >= -180 && longitude <= 180) { + return longitude; //common case, and avoids slight double precision shifting + } + double off = (longitude + 180) % 360; + if (off < 0) { + return 180 + off; + } else if (off == 0 && longitude > 0) { + return 180; + } else { + return -180 + off; + } + } + + /** Keep it simple, we don't need to take arbitrary Random for geo tests */ + private static Random random() { + return RandomizedContext.current().getRandom(); + } +} diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java index ff8baca95c86..7e64430be413 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestGeoPointQuery.java @@ -20,6 +20,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; @@ -61,4 +62,45 @@ protected Query newPolygonQuery(String field, Polygon... polygons) { return new GeoPointInPolygonQuery(field, TermEncoding.PREFIX, polygons); } + // TODO: remove these once we get tests passing! + + @Override + protected double nextLongitude() { + return GeoPointTestUtil.nextLongitude(); + } + + @Override + protected double nextLongitudeNear(double other) { + return GeoPointTestUtil.nextLongitudeNear(other); + } + + @Override + protected double nextLatitude() { + return GeoPointTestUtil.nextLatitude(); + } + + @Override + protected double nextLatitudeNear(double other) { + return GeoPointTestUtil.nextLatitudeNear(other); + } + + @Override + protected Rectangle nextBox() { + return GeoPointTestUtil.nextBox(); + } + + @Override + protected Rectangle nextBoxNear(double latitude, double longitude) { + return GeoPointTestUtil.nextBoxNear(latitude, longitude); + } + + @Override + protected Polygon nextPolygon() { + return GeoPointTestUtil.nextPolygon(); + } + + @Override + protected Polygon nextPolygonNear(double latitude, double longitude) { + return GeoPointTestUtil.nextPolygonNear(latitude, longitude); + } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java index 4c8e3e32de3f..75cc377d0e5a 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/geopoint/search/TestLegacyGeoPointQuery.java @@ -20,6 +20,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.geo.Polygon; +import org.apache.lucene.geo.Rectangle; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.document.GeoPointField.TermEncoding; import org.apache.lucene.spatial.util.BaseGeoPointTestCase; @@ -77,5 +78,45 @@ public void testSamePointManyTimes() throws Exception { assumeTrue("legacy encoding goes OOM on this test", false); } - + // TODO: remove these once we get tests passing! + + @Override + protected double nextLongitude() { + return GeoPointTestUtil.nextLongitude(); + } + + @Override + protected double nextLongitudeNear(double other) { + return GeoPointTestUtil.nextLongitudeNear(other); + } + + @Override + protected double nextLatitude() { + return GeoPointTestUtil.nextLatitude(); + } + + @Override + protected double nextLatitudeNear(double other) { + return GeoPointTestUtil.nextLatitudeNear(other); + } + + @Override + protected Rectangle nextBox() { + return GeoPointTestUtil.nextBox(); + } + + @Override + protected Rectangle nextBoxNear(double latitude, double longitude) { + return GeoPointTestUtil.nextBoxNear(latitude, longitude); + } + + @Override + protected Polygon nextPolygon() { + return GeoPointTestUtil.nextPolygon(); + } + + @Override + protected Polygon nextPolygonNear(double latitude, double longitude) { + return GeoPointTestUtil.nextPolygonNear(latitude, longitude); + } } diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java index adca4666fafa..c2536bd47a3f 100644 --- a/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java +++ b/lucene/spatial/src/test/org/apache/lucene/spatial/util/BaseGeoPointTestCase.java @@ -38,7 +38,6 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.geo.Rectangle; -import org.apache.lucene.geo.GeoTestUtil; import org.apache.lucene.geo.GeoUtils; import org.apache.lucene.geo.Polygon; import org.apache.lucene.index.DirectoryReader; @@ -68,7 +67,6 @@ import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.bkd.BKDWriter; -import org.junit.BeforeClass; /** * Abstract class to do basic tests for a geospatial impl (high level @@ -85,13 +83,48 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { protected static final String FIELD_NAME = "point"; - private static double originLat; - private static double originLon; + private double originLat; + private double originLon; - @BeforeClass - public static void beforeClassBase() throws Exception { - originLon = GeoTestUtil.nextLongitude(); - originLat = GeoTestUtil.nextLatitude(); + @Override + public void setUp() throws Exception { + super.setUp(); + originLon = nextLongitude(); + originLat = nextLatitude(); + } + + // TODO: remove these hooks once all subclasses can pass with new random! + + protected double nextLongitude() { + return org.apache.lucene.geo.GeoTestUtil.nextLongitude(); + } + + protected double nextLongitudeNear(double other) { + return org.apache.lucene.geo.GeoTestUtil.nextLongitudeNear(other); + } + + protected double nextLatitude() { + return org.apache.lucene.geo.GeoTestUtil.nextLatitude(); + } + + protected double nextLatitudeNear(double other) { + return org.apache.lucene.geo.GeoTestUtil.nextLatitudeNear(other); + } + + protected Rectangle nextBox() { + return org.apache.lucene.geo.GeoTestUtil.nextBox(); + } + + protected Rectangle nextBoxNear(double latitude, double longitude) { + return org.apache.lucene.geo.GeoTestUtil.nextBoxNear(latitude, longitude); + } + + protected Polygon nextPolygon() { + return org.apache.lucene.geo.GeoTestUtil.nextPolygon(); + } + + protected Polygon nextPolygonNear(double latitude, double longitude) { + return org.apache.lucene.geo.GeoTestUtil.nextPolygonNear(latitude, longitude); } /** Valid values that should not cause exception */ @@ -691,19 +724,19 @@ private void doTestRandom(int count) throws Exception { verify(small, lats, lons); } - public double randomLat(boolean small) { + public final double randomLat(boolean small) { if (small) { - return GeoTestUtil.nextLatitudeNear(originLat); + return nextLatitudeNear(originLat); } else { - return GeoTestUtil.nextLatitude(); + return nextLatitude(); } } - public double randomLon(boolean small) { + public final double randomLon(boolean small) { if (small) { - return GeoTestUtil.nextLongitudeNear(originLon); + return nextLongitudeNear(originLon); } else { - return GeoTestUtil.nextLongitude(); + return nextLongitude(); } } @@ -719,11 +752,11 @@ protected double quantizeLon(double lon) { return lon; } - protected Rectangle randomRect(boolean small) { + protected final Rectangle randomRect(boolean small) { if (small) { - return GeoTestUtil.nextBoxNear(originLat, originLon); + return nextBoxNear(originLat, originLon); } else { - return GeoTestUtil.nextBox(); + return nextBox(); } } @@ -1102,9 +1135,9 @@ protected void verifyRandomPolygons(boolean small, double[] lats, double[] lons) // Polygon final Polygon polygon; if (small) { - polygon = GeoTestUtil.nextPolygonNear(originLat, originLon); + polygon = nextPolygonNear(originLat, originLon); } else { - polygon = GeoTestUtil.nextPolygon(); + polygon = nextPolygon(); } Query query = newPolygonQuery(FIELD_NAME, polygon); @@ -1291,8 +1324,8 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); for (int i = 0; i < numDocs; i++) { - double latRaw = GeoTestUtil.nextLatitude(); - double lonRaw = GeoTestUtil.nextLongitude(); + double latRaw = nextLatitude(); + double lonRaw = nextLongitude(); // pre-normalize up front, so we can just use quantized value for testing and do simple exact comparisons double lat = quantizeLat(latRaw); double lon = quantizeLon(lonRaw); @@ -1306,8 +1339,8 @@ public PointsReader fieldsReader(SegmentReadState readState) throws IOException IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < numQueries; i++) { - double lat = GeoTestUtil.nextLatitude(); - double lon = GeoTestUtil.nextLongitude(); + double lat = nextLatitude(); + double lon = nextLongitude(); double radius = 50000000D * random().nextDouble(); BitSet expected = new BitSet(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java index 23e34164668d..9e719c09bb79 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/GeoTestUtil.java @@ -17,24 +17,62 @@ package org.apache.lucene.geo; import java.util.ArrayList; -import java.util.List; import java.util.Random; -import org.apache.lucene.util.SloppyMath; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.TestUtil; import com.carrotsearch.randomizedtesting.RandomizedContext; /** static methods for testing geo */ public class GeoTestUtil { + private static final long LATITUDE_MIN_SORTABLE = NumericUtils.doubleToSortableLong(-90); + private static final long LATITUDE_MAX_SORTABLE = NumericUtils.doubleToSortableLong(90); + /** returns next pseudorandom latitude (anywhere) */ public static double nextLatitude() { - return -90 + 180.0 * random().nextDouble(); + int surpriseMe = random().nextInt(17); + if (surpriseMe == 0) { + // random bitpattern in range + return NumericUtils.sortableLongToDouble(TestUtil.nextLong(random(), LATITUDE_MIN_SORTABLE, LATITUDE_MAX_SORTABLE)); + } else if (surpriseMe == 1) { + // edge case + return -90.0; + } else if (surpriseMe == 2) { + // edge case + return 90.0; + } else if (surpriseMe == 3) { + // may trigger divide by zero + return 0.0; + } else { + // distributed ~ evenly + return -90 + 180.0 * random().nextDouble(); + } } + private static final long LONGITUDE_MIN_SORTABLE = NumericUtils.doubleToSortableLong(-180); + private static final long LONGITUDE_MAX_SORTABLE = NumericUtils.doubleToSortableLong(180); + /** returns next pseudorandom longitude (anywhere) */ public static double nextLongitude() { - return -180 + 360.0 * random().nextDouble(); + int surpriseMe = random().nextInt(17); + if (surpriseMe == 0) { + // random bitpattern in range + return NumericUtils.sortableLongToDouble(TestUtil.nextLong(random(), LONGITUDE_MIN_SORTABLE, LONGITUDE_MAX_SORTABLE)); + } else if (surpriseMe == 1) { + // edge case + return -180.0; + } else if (surpriseMe == 2) { + // edge case + return 180.0; + } else if (surpriseMe == 3) { + // may trigger divide by 0 + return 0.0; + } else { + // distributed ~ evenly + return -180 + 360.0 * random().nextDouble(); + } } /** returns next pseudorandom latitude, kinda close to {@code otherLatitude} */ From cf6c9c78bce51bbe20dcf6dcd97321726960192c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 11 Apr 2016 18:46:47 +0200 Subject: [PATCH 0331/1113] Fix javadoc generation. --- lucene/classification/build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/classification/build.xml b/lucene/classification/build.xml index c56d48123fa6..3ddb9bd26e6d 100644 --- a/lucene/classification/build.xml +++ b/lucene/classification/build.xml @@ -46,7 +46,7 @@ - + From a03c0b806db0c6cd54fcf03abfc0ff86ffb944f1 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 11 Apr 2016 12:55:21 -0400 Subject: [PATCH 0332/1113] LUCENE-7201: Handle illegal rectangles during testing. --- .../test/org/apache/lucene/spatial3d/TestGeo3DPoint.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index 20302b408640..deb35878c017 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -560,7 +560,11 @@ private static Query random3DQuery(final String field) { case 2: { // Rectangles final Rectangle r = GeoTestUtil.nextBox(); - return Geo3DPoint.newBoxQuery(field, r.minLat, r.maxLat, r.minLon, r.maxLon); + try { + return Geo3DPoint.newBoxQuery(field, r.minLat, r.maxLat, r.minLon, r.maxLon); + } catch (IllegalArgumentException e) { + continue; + } } case 3: { From 5f4af54f9900f3c05a18fe16584e72976ca1bd3b Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Mon, 11 Apr 2016 12:52:48 -0400 Subject: [PATCH 0333/1113] LUCENE-7201: Handle backside whole-world case for bbox generation. --- .../java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java index de7493e5c50e..3d0b8cb7a406 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java @@ -45,7 +45,8 @@ public static GeoBBox makeGeoBBox(final PlanetModel planetModel, double topLat, leftLon = -Math.PI; if (rightLon > Math.PI) rightLon = Math.PI; - if (Math.abs(leftLon + Math.PI) < Vector.MINIMUM_RESOLUTION && Math.abs(rightLon - Math.PI) < Vector.MINIMUM_RESOLUTION) { + if ((Math.abs(leftLon + Math.PI) < Vector.MINIMUM_RESOLUTION && Math.abs(rightLon - Math.PI) < Vector.MINIMUM_RESOLUTION) || + (Math.abs(rightLon + Math.PI) < Vector.MINIMUM_RESOLUTION && Math.abs(leftLon - Math.PI) < Vector.MINIMUM_RESOLUTION)) { if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_RESOLUTION && Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_RESOLUTION) return new GeoWorld(planetModel); if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_RESOLUTION) { From ace13b5728df97664ae21e617de9e6ae3706657e Mon Sep 17 00:00:00 2001 From: yonik Date: Mon, 11 Apr 2016 23:51:41 -0400 Subject: [PATCH 0334/1113] LUCENE-7188: remove incorrect sanity check in NRTCachingDirectory.listAll() that throws IllegalStateException --- lucene/CHANGES.txt | 4 ++++ .../java/org/apache/lucene/store/NRTCachingDirectory.java | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 63a322542975..19153778c8fe 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -69,6 +69,10 @@ Bug Fixes * LUCENE-7187: Block join queries' Weight#extractTerms(...) implementations should delegate to the wrapped weight. (Martijn van Groningen) +* LUCENE-7188: remove incorrect sanity check in NRTCachingDirectory.listAll() + that led to IllegalStateException being thrown when nothing was wrong. + (David Smiley, yonik) + Other * LUCENE-7174: Upgrade randomizedtesting to 2.3.4. (Uwe Schindler, Dawid Weiss) diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java index 22a9571b76b2..9be0b9e9c911 100644 --- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java @@ -101,10 +101,7 @@ public synchronized String[] listAll() throws IOException { files.add(f); } for(String f : in.listAll()) { - if (!files.add(f)) { - throw new IllegalStateException("file: " + in + " appears both in delegate and in cache: " + - "cache=" + Arrays.toString(cache.listAll()) + ",delegate=" + Arrays.toString(in.listAll())); - } + files.add(f); } String[] result = files.toArray(new String[files.size()]); Arrays.sort(result); From e4b26922ab9c9af8e190a75df0f0a69f15672622 Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Tue, 12 Apr 2016 06:03:04 -0400 Subject: [PATCH 0335/1113] LUCENE-7203: Improve polygon intersection detection using a small amount of upfront work. --- .../spatial3d/geom/GeoConcavePolygon.java | 39 ++++++++++-------- .../spatial3d/geom/GeoConvexPolygon.java | 40 ++++++++++--------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 8c6f75713806..518df33188be 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -224,7 +224,17 @@ public void done(final boolean isInternalReturnEdge) { // For each edge, create a bounds object. eitherBounds = new HashMap<>(edges.length); for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { - eitherBounds.put(edges[edgeIndex], new EitherBound(invertedEdges[edgeIndex])); + final SidedPlane edge = edges[edgeIndex]; + final SidedPlane invertedEdge = invertedEdges[edgeIndex]; + int bound1Index = legalIndex(edgeIndex+1); + while (invertedEdges[legalIndex(bound1Index)].isNumericallyIdentical(invertedEdge)) { + bound1Index++; + } + int bound2Index = legalIndex(edgeIndex-1); + while (invertedEdges[legalIndex(bound2Index)].isNumericallyIdentical(invertedEdge)) { + bound2Index--; + } + eitherBounds.put(edge, new EitherBound(invertedEdges[legalIndex(bound1Index)], invertedEdges[legalIndex(bound2Index)])); } // Pick an edge point arbitrarily @@ -238,6 +248,9 @@ public void done(final boolean isInternalReturnEdge) { protected int legalIndex(int index) { while (index >= points.size()) index -= points.size(); + while (index < 0) { + index += points.size(); + } return index; } @@ -297,37 +310,29 @@ public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final M return false; } - /** A membership implementation representing polygon edges that all must apply. + /** A membership implementation representing polygon edges that must apply. */ protected class EitherBound implements Membership { - protected final SidedPlane exception; + protected final SidedPlane sideBound1; + protected final SidedPlane sideBound2; /** Constructor. * @param exception is the one plane to exclude from the check. */ - public EitherBound(final SidedPlane exception) { - this.exception = exception; + public EitherBound(final SidedPlane sideBound1, final SidedPlane sideBound2) { + this.sideBound1 = sideBound1; + this.sideBound2 = sideBound2; } @Override public boolean isWithin(final Vector v) { - for (final SidedPlane edge : invertedEdges) { - if (edge != exception && !edge.isWithin(v)) { - return false; - } - } - return true; + return sideBound1.isWithin(v) && sideBound2.isWithin(v); } @Override public boolean isWithin(final double x, final double y, final double z) { - for (final SidedPlane edge : invertedEdges) { - if (edge != exception && !edge.isWithin(x, y, z)) { - return false; - } - } - return true; + return sideBound1.isWithin(x,y,z) && sideBound2.isWithin(x,y,z); } } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index b631b55a0595..4cc81773d23c 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -219,8 +219,17 @@ public void done(final boolean isInternalReturnEdge) { // For each edge, create a bounds object. eitherBounds = new HashMap<>(edges.length); - for (final SidedPlane edge : edges) { - eitherBounds.put(edge, new EitherBound(edge)); + for (int edgeIndex = 0; edgeIndex < edges.length; edgeIndex++) { + final SidedPlane edge = edges[edgeIndex]; + int bound1Index = legalIndex(edgeIndex+1); + while (edges[legalIndex(bound1Index)].isNumericallyIdentical(edge)) { + bound1Index++; + } + int bound2Index = legalIndex(edgeIndex-1); + while (edges[legalIndex(bound2Index)].isNumericallyIdentical(edge)) { + bound2Index--; + } + eitherBounds.put(edge, new EitherBound(edges[legalIndex(bound1Index)], edges[legalIndex(bound2Index)])); } // Pick an edge point arbitrarily @@ -234,6 +243,9 @@ public void done(final boolean isInternalReturnEdge) { protected int legalIndex(int index) { while (index >= points.size()) index -= points.size(); + while (index < 0) { + index += points.size(); + } return index; } @@ -284,37 +296,29 @@ public boolean intersects(final Plane p, final GeoPoint[] notablePoints, final M return false; } - /** A membership implementation representing polygon edges that all must apply. + /** A membership implementation representing polygon edges that must apply. */ protected class EitherBound implements Membership { - protected final SidedPlane exception; + protected final SidedPlane sideBound1; + protected final SidedPlane sideBound2; /** Constructor. * @param exception is the one plane to exclude from the check. */ - public EitherBound(final SidedPlane exception) { - this.exception = exception; + public EitherBound(final SidedPlane sideBound1, final SidedPlane sideBound2) { + this.sideBound1 = sideBound1; + this.sideBound2 = sideBound2; } @Override public boolean isWithin(final Vector v) { - for (final SidedPlane edge : edges) { - if (edge != exception && !edge.isWithin(v)) { - return false; - } - } - return true; + return sideBound1.isWithin(v) && sideBound2.isWithin(v); } @Override public boolean isWithin(final double x, final double y, final double z) { - for (final SidedPlane edge : edges) { - if (edge != exception && !edge.isWithin(x, y, z)) { - return false; - } - } - return true; + return sideBound1.isWithin(x,y,z) && sideBound2.isWithin(x,y,z); } } From 520c31f8ed4ed45bc4fc150903cfc7510983779f Mon Sep 17 00:00:00 2001 From: Karl Wright Date: Tue, 12 Apr 2016 06:05:22 -0400 Subject: [PATCH 0336/1113] LUCENE-7203: Fix the javadoc broken in the previous commit. --- .../org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java | 3 ++- .../org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java index 518df33188be..995e1912da35 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConcavePolygon.java @@ -318,7 +318,8 @@ protected class EitherBound implements Membership { protected final SidedPlane sideBound2; /** Constructor. - * @param exception is the one plane to exclude from the check. + * @param sideBound1 is the first side bound. + * @param sideBound2 is the second side bound. */ public EitherBound(final SidedPlane sideBound1, final SidedPlane sideBound2) { this.sideBound1 = sideBound1; diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java index 4cc81773d23c..dd5971fe07df 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoConvexPolygon.java @@ -304,7 +304,8 @@ protected class EitherBound implements Membership { protected final SidedPlane sideBound2; /** Constructor. - * @param exception is the one plane to exclude from the check. + * @param sideBound1 is the first side bound. + * @param sideBound2 is the second side bound. */ public EitherBound(final SidedPlane sideBound1, final SidedPlane sideBound2) { this.sideBound1 = sideBound1; From 39d74d86cddfed5ec7f0dd9b4dcf0e102384d980 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Tue, 12 Apr 2016 06:10:08 -0400 Subject: [PATCH 0337/1113] LUCENE-7189: improve geo debugging technology --- .../org/apache/lucene/geo/EarthDebugger.java | 43 ++++++---- .../lucene/index/PointsStackTracker.java | 79 +++++++++++++++++++ 2 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 lucene/test-framework/src/java/org/apache/lucene/index/PointsStackTracker.java diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java index 6c06c94a084b..56dde88cd58c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/EarthDebugger.java @@ -17,8 +17,6 @@ * limitations under the License. */ -import org.apache.lucene.geo.Polygon; -import org.apache.lucene.geo.Rectangle; import org.apache.lucene.util.SloppyMath; /** Draws shapes on the earth surface and renders using the very cool http://www.webglearth.org. @@ -40,6 +38,16 @@ public EarthDebugger() { b.append(" var earth = new WE.map('earth_div');\n"); } + public EarthDebugger(double centerLat, double centerLon, double altitudeMeters) { + b.append("\n"); + b.append("\n"); + b.append(" \n"); + b.append(" \n"); + b.append("