Merge remote-tracking branch 'remotes/asf/trunk'

Conflicts:
	solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
	solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java

lucene/CHANGES.txt

@@ -109,6 +109,15 @@ New Features
   parent. Added QueryNode.removeFromParent() that allows nodes to be 
   detached from its parent node. (Adriano Crestani)
 
+* LUCENE-5395: Upgrade Spatial4j 0.4: Parses WKT (including ENVELOPE) with
+  extension "BUFFER"; buffering a point results in a Circle. JTS isn't needed
+  for WKT any more but remains required for Polygons. New Shapes:
+  ShapeCollection and BufferedLineString. Various other improvements and bug
+  fixes too. More info:
+  https://github.com/spatial4j/spatial4j/blob/master/CHANGES.md  (David Smiley)
+
+* LUCENE-5415: Add multitermquery (wildcards,prefix,etc) to PostingsHighlighter.
+  (Mike McCandless, Robert Muir)
 
 Build
 
@@ -186,12 +195,34 @@ Bug fixes
   on Lucene 4.6 if any index segments were Lucene 4.0-4.5.
   (Littlestar, Mike McCandless, Shai Erera, Robert Muir)
 
+* SOLR-5661: PriorityQueue now refuses to allocate itself if the
+  incoming maxSize is too large (Raintung Li via Mike McCandless)
+
+* LUCENE-5228: IndexWriter.addIndexes(Directory[]) now acquires a
+  write lock in each Directory, to ensure that no open IndexWriter is
+  changing the incoming indices.  This also means that you cannot pass
+  the same Directory to multiple concurrent addIndexes calls (which is
+  anyways unusual).  (Robert Muir, Mike McCandless)
+
+* LUCENE-5415: SpanMultiTermQueryWrapper didn't handle its boost in
+  hashcode/equals/tostring/rewrite.  (Robert Muir)
+
 API Changes
 
 * LUCENE-5339: The facet module was simplified/reworked to make the
   APIs more approachable to new users.  (Shai Erera, Gilad Barkai, Rob
   Muir, Mike McCandless)
 
+* LUCENE-5395: The SpatialArgsParser now only reads WKT, no more "lat, lon"
+  etc. but it's easy to override the parseShape method if you wish. (David
+  Smiley)
+
+* LUCENE-5414: DocumentExpressionDictionary was renamed to 
+  DocumentValueSourceDictionary and all dependencies to the lucene-expression
+  module were removed from lucene-suggest. DocumentValueSourceDictionary now
+  only accepts a ValueSource instead of a convenience ctor for an expression
+  string. (Simon Willnauer)
+
 Optimizations
 
 * LUCENE-5372: Replace StringBuffer by StringBuilder, where possible.

lucene/benchmark/build.xml

@@ -67,7 +67,7 @@
         <bunzip2 src="temp/enwiki-20070527-pages-articles.xml.bz2" dest="temp"/>
     </target>
 
-    <target name="geonames-files" depends="check-files">
+    <target name="geonames-files" depends="check-files" description="Get data for spatial.alg">
         <mkdir dir="temp"/>
         <antcall target="get-geonames"/>
         <antcall target="expand-geonames"/>

lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/GeonamesLineParser.java

@@ -39,6 +39,8 @@ public void parseLine(DocData docData, String line) {
 
     docData.setID(Integer.parseInt(parts[0]));//note: overwrites ID assigned by LineDocSource
     docData.setName(parts[1]);
-    docData.setBody(parts[4]+","+parts[5]); // latitude , longitude
+    String latitude = parts[4];
+    String longitude = parts[5];
+    docData.setBody("POINT("+longitude+" "+latitude+")");//WKT is x y order
   }
 }

lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialDocMaker.java

@@ -190,7 +190,7 @@ public Document makeDocument() throws Exception {
   public static Shape makeShapeFromString(SpatialStrategy strategy, String name, String shapeStr) {
     if (shapeStr != null && shapeStr.length() > 0) {
       try {
-        return strategy.getSpatialContext().readShape(shapeStr);
+        return strategy.getSpatialContext().readShapeFromWkt(shapeStr);
       } catch (Exception e) {//InvalidShapeException TODO
         System.err.println("Shape "+name+" wasn't parseable: "+e+"  (skipping it)");
         return null;

lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java

@@ -36,7 +36,7 @@
 
 /**
  * Reads spatial data from the body field docs from an internally created {@link LineDocSource}.
- * It's parsed by {@link com.spatial4j.core.context.SpatialContext#readShape(String)} and then
+ * It's parsed by {@link com.spatial4j.core.context.SpatialContext#readShapeFromWkt(String)} (String)} and then
  * further manipulated via a configurable {@link SpatialDocMaker.ShapeConverter}. When using point
  * data, it's likely you'll want to configure the shape converter so that the query shapes actually
  * cover a region. The queries are all created & cached in advance. This query maker works in

lucene/core/src/java/org/apache/lucene/index/IndexWriter.java

@@ -795,11 +795,7 @@ public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
         if (infoStream.isEnabled("IW")) {
           infoStream.message("IW", "init: hit exception on init; releasing write lock");
         }
-        try {
-          writeLock.release();
-        } catch (Throwable t) {
-          // don't mask the original exception
-        }
+        IOUtils.closeWhileHandlingException(writeLock);
         writeLock = null;
       }
     }
@@ -1052,7 +1048,7 @@ private void closeInternal(boolean waitForMerges, boolean doFlush) throws IOExce
       }
 
       if (writeLock != null) {
-        writeLock.release();                          // release write lock
+        writeLock.close();                          // release write lock
         writeLock = null;
       }
       synchronized(this) {
@@ -2362,6 +2358,28 @@ private void noDupDirs(Directory... dirs) {
     }
   }
 
+  /** Acquires write locks on all the directories; be sure
+   *  to match with a call to {@link IOUtils#close} in a
+   *  finally clause. */
+  private List<Lock> acquireWriteLocks(Directory... dirs) throws IOException {
+    List<Lock> locks = new ArrayList<Lock>();
+    for(int i=0;i<dirs.length;i++) {
+      boolean success = false;
+      try {
+        Lock lock = dirs[i].makeLock(WRITE_LOCK_NAME);
+        locks.add(lock);
+        lock.obtain(config.getWriteLockTimeout());
+        success = true;
+      } finally {
+        if (success == false) {
+          // Release all previously acquired locks:
+          IOUtils.closeWhileHandlingException(locks);
+        }
+      }
+    }
+    return locks;
+  }
+
   /**
    * Adds all segments from an array of indexes into this index.
    *
@@ -2372,11 +2390,10 @@ private void noDupDirs(Directory... dirs) {
    * with this method.
    *
    * <p>
-   * <b>NOTE:</b> the index in each {@link Directory} must not be
-   * changed (opened by a writer) while this method is
-   * running.  This method does not acquire a write lock in
-   * each input Directory, so it is up to the caller to
-   * enforce this.
+   * <b>NOTE:</b> this method acquires the write lock in
+   * each directory, to ensure that no {@code IndexWriter}
+   * is currently open or tries to open while this is
+   * running.
    *
    * <p>This method is transactional in how Exceptions are
    * handled: it does not commit a new segments_N file until
@@ -2405,12 +2422,18 @@ private void noDupDirs(Directory... dirs) {
    *
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
+   * @throws LockObtainFailedException if we were unable to
+   *   acquire the write lock in at least one directory
    */
   public void addIndexes(Directory... dirs) throws IOException {
     ensureOpen();
 
     noDupDirs(dirs);
 
+    List<Lock> locks = acquireWriteLocks(dirs);
+
+    boolean successTop = false;
+
     try {
       if (infoStream.isEnabled("IW")) {
         infoStream.message("IW", "flush at addIndexes(Directory...)");
@@ -2481,8 +2504,16 @@ public void addIndexes(Directory... dirs) throws IOException {
         checkpoint();
       }
 
+      successTop = true;
+
     } catch (OutOfMemoryError oom) {
       handleOOM(oom, "addIndexes(Directory...)");
+    } finally {
+      if (successTop) {
+        IOUtils.close(locks);
+      } else {
+        IOUtils.closeWhileHandlingException(locks);
+      }
     }
   }
 
@@ -4415,7 +4446,7 @@ public static boolean isLocked(Directory directory) throws IOException {
    * currently accessing this index.
    */
   public static void unlock(Directory directory) throws IOException {
-    directory.makeLock(IndexWriter.WRITE_LOCK_NAME).release();
+    directory.makeLock(IndexWriter.WRITE_LOCK_NAME).close();
   }
 
   /** If {@link DirectoryReader#open(IndexWriter,boolean)} has

lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java

@@ -128,4 +128,9 @@ public String toString(String field) {
     buffer.append(ToStringUtils.boost(getBoost()));
     return buffer.toString();
   }
+
+  /** Returns the automaton used to create this query */
+  public Automaton getAutomaton() {
+    return automaton;
+  }
 }

lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java

@@ -138,6 +138,14 @@ public int getMaxEdits() {
   public int getPrefixLength() {
     return prefixLength;
   }
+
+  /**
+   * Returns true if transpositions should be treated as a primitive edit operation. 
+   * If this is false, comparisons will implement the classic Levenshtein algorithm.
+   */
+  public boolean getTranspositions() {
+    return transpositions;
+  }
 
   @Override
   protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {

lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java

@@ -100,13 +100,22 @@ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map<Term,Ter
   public String getField() {
     return query.getField();
   }
+
+  /** Returns the wrapped query */
+  public Query getWrappedQuery() {
+    return query;
+  }
 
   @Override
   public String toString(String field) {
     StringBuilder builder = new StringBuilder();
     builder.append("SpanMultiTermQueryWrapper(");
     builder.append(query.toString(field));
     builder.append(")");
+    if (getBoost() != 1F) {
+      builder.append('^');
+      builder.append(getBoost());
+    }
     return builder.toString();
   }
 
@@ -115,22 +124,26 @@ public Query rewrite(IndexReader reader) throws IOException {
     final Query q = query.rewrite(reader);
     if (!(q instanceof SpanQuery))
       throw new UnsupportedOperationException("You can only use SpanMultiTermQueryWrapper with a suitable SpanRewriteMethod.");
+    q.setBoost(q.getBoost() * getBoost()); // multiply boost
     return q;
   }
 
   @Override
   public int hashCode() {
-    return 31 * query.hashCode();
+    final int prime = 31;
+    int result = super.hashCode();
+    result = prime * result + query.hashCode();
+    return result;
   }
 
   @Override
-  @SuppressWarnings({"rawtypes","unchecked"})
   public boolean equals(Object obj) {
     if (this == obj) return true;
-    if (obj == null) return false;
+    if (!super.equals(obj)) return false;
     if (getClass() != obj.getClass()) return false;
-    final SpanMultiTermQueryWrapper other = (SpanMultiTermQueryWrapper) obj;
-    return query.equals(other.query);
+    SpanMultiTermQueryWrapper<?> other = (SpanMultiTermQueryWrapper<?>) obj;
+    if (!query.equals(other.query)) return false;
+    return true;
   }
 
   /** Abstract class that defines how the query is rewritten. */

lucene/core/src/java/org/apache/lucene/store/Lock.java

@@ -17,9 +17,11 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.util.ThreadInterruptedException;
+import java.io.Closeable;
 import java.io.IOException;
 
+import org.apache.lucene.util.ThreadInterruptedException;
+
 /** An interprocess mutex lock.
  * <p>Typical use might look like:<pre class="prettyprint">
  * new Lock.With(directory.makeLock("my.lock")) {
@@ -30,8 +32,10 @@
  * </pre>
  *
  * @see Directory#makeLock(String)
+ *
+ * @lucene.internal
  */
-public abstract class Lock {
+public abstract class Lock implements Closeable {
 
   /** How long {@link #obtain(long)} waits, in milliseconds,
    *  in between attempts to acquire the lock. */
@@ -42,7 +46,8 @@ public abstract class Lock {
   public static final long LOCK_OBTAIN_WAIT_FOREVER = -1;
 
   /** Attempts to obtain exclusive access and immediately return
-   *  upon success or failure.
+   *  upon success or failure.  Use {@link #close} to
+   *  release the lock.
    * @return true iff exclusive access is obtained
    */
   public abstract boolean obtain() throws IOException;
@@ -98,7 +103,7 @@ public boolean obtain(long lockWaitTimeout) throws IOException {
   }
 
   /** Releases exclusive access. */
-  public abstract void release() throws IOException;
+  public abstract void close() throws IOException;
 
   /** Returns true if the resource is currently locked.  Note that one must
    * still call {@link #obtain()} before using the resource. */
@@ -134,8 +139,9 @@ public Object run() throws IOException {
          locked = lock.obtain(lockWaitTimeout);
          return doBody();
       } finally {
-        if (locked)
-          lock.release();
+        if (locked) {
+          lock.close();
+        }
       }
     }
   }

lucene/core/src/java/org/apache/lucene/store/LockReleaseFailedException.java

@@ -22,7 +22,7 @@
 /**
  * This exception is thrown when the <code>write.lock</code>
  * could not be released.
- * @see Lock#release()
+ * @see Lock#close()
  */
 public class LockReleaseFailedException extends IOException {
   public LockReleaseFailedException(String message) {

lucene/core/src/java/org/apache/lucene/store/LockStressTest.java

@@ -102,7 +102,7 @@ public static void main(String[] args) throws Exception {
 
       if (obtained) {
         System.out.print("l");
-        l.release();
+        l.close();
       }
       Thread.sleep(sleepTimeMS);
     }

lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java

@@ -107,7 +107,7 @@ public void clearLock(String lockName) throws IOException {
       // NOTE: makeLock fixes the lock name by prefixing it w/ lockPrefix.
       // Therefore it should be called before the code block next which prefixes
       // the given name.
-      makeLock(lockName).release();
+      makeLock(lockName).close();
 
       if (lockPrefix != null) {
         lockName = lockPrefix + "-" + lockName;
@@ -259,7 +259,7 @@ public synchronized boolean obtain() throws IOException {
   }
 
   @Override
-  public synchronized void release() throws IOException {
+  public synchronized void close() throws IOException {
     if (lockExists()) {
       try {
         lock.release();
@@ -298,7 +298,7 @@ public synchronized void release() throws IOException {
         }
       } finally {
         if (obtained) {
-          release();
+          close();
         }
       }
     }
@@ -317,7 +317,7 @@ public synchronized boolean isLocked() {
     // Try to obtain and release (if was locked) the lock
     try {
       boolean obtained = obtain();
-      if (obtained) release();
+      if (obtained) close();
       return !obtained;
     } catch (IOException ioe) {
       return false;

lucene/core/src/java/org/apache/lucene/store/NoLockFactory.java

@@ -55,7 +55,7 @@ public boolean obtain() throws IOException {
   }
 
   @Override
-  public void release() {
+  public void close() {
   }
 
   @Override