From 6602c01097c66d242046fb0490e0a5dcc1ca36c5 Mon Sep 17 00:00:00 2001 From: Lubos Kosco Date: Fri, 20 Feb 2015 18:19:00 +0100 Subject: [PATCH 1/2] lucene 5.0.0 build (no pkg, mvn yet) --- build.xml | 94 ++++++++++--------- nbproject/project.properties | 2 +- .../nbproject/project.properties | 2 +- .../opengrok/analysis/CompatibleAnalyser.java | 17 ++-- .../opengrok/analysis/FileAnalyzer.java | 13 ++- .../opengrok/analysis/HistoryAnalyzer.java | 10 +- .../opengrok/analysis/JFlexTokenizer.java | 15 ++- .../opengrok/analysis/PathTokenizer.java | 6 +- .../analysis/archive/BZip2Analyzer.java | 9 +- .../analysis/archive/GZIPAnalyzer.java | 9 +- .../opengrok/analysis/c/CScopeParser.lex | 1 - .../opengrok/analysis/c/CSymbolTokenizer.lex | 5 +- .../analysis/c/CxxSymbolTokenizer.lex | 5 +- .../analysis/document/TroffAnalyzer.java | 8 +- .../analysis/document/TroffFullTokenizer.lex | 10 +- .../analysis/java/JavaScopeParser.lex | 2 - .../plain/AbstractSourceCodeAnalyzer.java | 8 +- .../analysis/plain/PlainFullTokenizer.lex | 10 +- .../analysis/plain/PlainSymbolTokenizer.lex | 10 +- .../analysis/uue/UuencodeAnalyzer.java | 8 +- .../analysis/uue/UuencodeFullTokenizer.lex | 10 +- .../opengrok/index/IndexDatabase.java | 19 ++-- .../opengrok/search/SearchEngine.java | 17 ++-- .../opengrok/web/SearchHelper.java | 86 +++++++++-------- .../opengrok/analysis/PathTokenizerTest.java | 3 +- .../analysis/php/PhpSymbolTokenizerTest.java | 10 +- 26 files changed, 199 insertions(+), 190 deletions(-) diff --git a/build.xml b/build.xml index 70da0d7011d..0bc1dc9f5a5 100644 --- a/build.xml +++ b/build.xml @@ -18,7 +18,7 @@ information: Portions Copyright [yyyy] [name of copyright owner] CDDL HEADER END -Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. --> @@ -170,49 +170,55 @@ Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/nbproject/project.properties b/nbproject/project.properties index 4e1e6a1b1a3..c28fdbcf211 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -45,7 +45,7 @@ jnlp.mixed.code=default jnlp.signing= jnlp.signing.alias= jnlp.signing.keystore= -lucene.version=4.10.3 +lucene.version=5.0.0 lucene-core.jar=lucene-core-${lucene.version}.jar lucene-analyzers-common.jar=lucene-analyzers-common-${lucene.version}.jar lucene-queryparser.jar=lucene-queryparser-${lucene.version}.jar diff --git a/opengrok-web-nbproject/nbproject/project.properties b/opengrok-web-nbproject/nbproject/project.properties index c1f2f643227..dfd21c78f9f 100644 --- a/opengrok-web-nbproject/nbproject/project.properties +++ b/opengrok-web-nbproject/nbproject/project.properties @@ -28,7 +28,7 @@ endorsed.classpath= excludes= file.reference.json-simple-1.1.1.jar=../lib/json-simple-1.1.1.jar j2ee.platform.classpath=${j2ee.server.home}/lib/annotations-api.jar:${j2ee.server.home}/lib/catalina-ant.jar:${j2ee.server.home}/lib/catalina-ha.jar:${j2ee.server.home}/lib/catalina-storeconfig.jar:${j2ee.server.home}/lib/catalina-tribes.jar:${j2ee.server.home}/lib/catalina.jar:${j2ee.server.home}/lib/ecj-4.4.jar:${j2ee.server.home}/lib/el-api.jar:${j2ee.server.home}/lib/jasper-el.jar:${j2ee.server.home}/lib/jasper.jar:${j2ee.server.home}/lib/jsp-api.jar:${j2ee.server.home}/lib/servlet-api.jar:${j2ee.server.home}/lib/tomcat-api.jar:${j2ee.server.home}/lib/tomcat-coyote.jar:${j2ee.server.home}/lib/tomcat-dbcp.jar:${j2ee.server.home}/lib/tomcat-i18n-es.jar:${j2ee.server.home}/lib/tomcat-i18n-fr.jar:${j2ee.server.home}/lib/tomcat-i18n-ja.jar:${j2ee.server.home}/lib/tomcat-jdbc.jar:${j2ee.server.home}/lib/tomcat-jni.jar:${j2ee.server.home}/lib/tomcat-spdy.jar:${j2ee.server.home}/lib/tomcat-util-scan.jar:${j2ee.server.home}/lib/tomcat-util.jar:${j2ee.server.home}/lib/tomcat-websocket.jar:${j2ee.server.home}/lib/websocket-api.jar -lucene.version=4.10.3 +lucene.version=5.0.0 lucene-core.jar=lucene-core-${lucene.version}.jar lucene-analyzers-common.jar=lucene-analyzers-common-${lucene.version}.jar lucene-queryparser.jar=lucene-queryparser-${lucene.version}.jar diff --git a/src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java b/src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java index 7308cb8c099..92a8383fba1 100644 --- a/src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java +++ b/src/org/opensolaris/opengrok/analysis/CompatibleAnalyser.java @@ -18,11 +18,10 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.opensolaris.opengrok.analysis.plain.PlainFullTokenizer; import org.opensolaris.opengrok.analysis.plain.PlainSymbolTokenizer; @@ -35,21 +34,21 @@ public CompatibleAnalyser() { } @Override - protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + protected TokenStreamComponents createComponents(String fieldName) { switch (fieldName) { case QueryBuilder.FULL: - return new TokenStreamComponents(new PlainFullTokenizer(reader)); + return new TokenStreamComponents(new PlainFullTokenizer()); case QueryBuilder.REFS: - return new TokenStreamComponents(new PlainSymbolTokenizer(reader)); + return new TokenStreamComponents(new PlainSymbolTokenizer()); case QueryBuilder.DEFS: - return new TokenStreamComponents(new PlainSymbolTokenizer(reader)); + return new TokenStreamComponents(new PlainSymbolTokenizer()); case QueryBuilder.PATH: case QueryBuilder.PROJECT: - return new TokenStreamComponents(new PathTokenizer(reader)); + return new TokenStreamComponents(new PathTokenizer()); case QueryBuilder.HIST: - return new HistoryAnalyzer().createComponents(fieldName, reader); + return new HistoryAnalyzer().createComponents(fieldName); default: - return new TokenStreamComponents(new PlainFullTokenizer(reader)); + return new TokenStreamComponents(new PlainFullTokenizer()); } } } diff --git a/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java b/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java index 9f6c01ac1e8..340f4c0f235 100644 --- a/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/FileAnalyzer.java @@ -18,13 +18,12 @@ */ /* - * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. */ package org.opensolaris.opengrok.analysis; import java.io.IOException; -import java.io.Reader; import java.io.Writer; import java.util.logging.Level; import org.apache.lucene.analysis.Analyzer; @@ -168,18 +167,18 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc } @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { + public TokenStreamComponents createComponents(String fieldName) { switch (fieldName) { case "full": - return new TokenStreamComponents(new PlainFullTokenizer(reader)); + return new TokenStreamComponents(new PlainFullTokenizer()); case "path": case "project": - return new TokenStreamComponents(new PathTokenizer(reader)); + return new TokenStreamComponents(new PathTokenizer()); case "hist": - return new HistoryAnalyzer().createComponents(fieldName, reader); + return new HistoryAnalyzer().createComponents(fieldName); case "refs": case "defs": - return new TokenStreamComponents(new PlainSymbolTokenizer(reader)); + return new TokenStreamComponents(new PlainSymbolTokenizer()); default: OpenGrokLogger.getLogger().log( Level.WARNING, "Have no analyzer for: {0}", fieldName); diff --git a/src/org/opensolaris/opengrok/analysis/HistoryAnalyzer.java b/src/org/opensolaris/opengrok/analysis/HistoryAnalyzer.java index 6d4168adf59..46986a6ef68 100644 --- a/src/org/opensolaris/opengrok/analysis/HistoryAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/HistoryAnalyzer.java @@ -18,11 +18,10 @@ */ /* - * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis; -import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.util.CharArraySet; @@ -54,13 +53,12 @@ public HistoryAnalyzer() { /** * Filters LowerCaseTokenizer with StopFilter. - * @param fieldName name of field for which to create components - * @param reader from which to read data + * @param fieldName name of field for which to create components * @return components for this analyzer */ @Override - protected TokenStreamComponents createComponents(String fieldName, Reader reader) { - final PlainFullTokenizer plainfull = new PlainFullTokenizer(reader); + protected TokenStreamComponents createComponents(String fieldName) { + final PlainFullTokenizer plainfull = new PlainFullTokenizer(); //we are counting position increments, this might affect the queries later and need to be in sync, especially for highlighting of results return new TokenStreamComponents(plainfull, new StopFilter(plainfull, stopWords)); } diff --git a/src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java b/src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java index de82e9819ac..ca384b661c0 100644 --- a/src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java +++ b/src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis; @@ -52,10 +52,15 @@ public abstract class JFlexTokenizer extends Tokenizer { abstract public void yybegin(int newState); - abstract public int yystate(); - - public JFlexTokenizer(Reader input) { - super(input); + abstract public int yystate(); + + //TODO can be removed once we figure out jflex generation of empty constructor + protected JFlexTokenizer(Reader in) { + super(); + } + + protected JFlexTokenizer() { + super(); } /** diff --git a/src/org/opensolaris/opengrok/analysis/PathTokenizer.java b/src/org/opensolaris/opengrok/analysis/PathTokenizer.java index de17c983d33..21e4c90bb6f 100644 --- a/src/org/opensolaris/opengrok/analysis/PathTokenizer.java +++ b/src/org/opensolaris/opengrok/analysis/PathTokenizer.java @@ -18,12 +18,11 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis; import java.io.IOException; -import java.io.Reader; import java.util.Arrays; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -59,8 +58,7 @@ public class PathTokenizer extends Tokenizer { private boolean dot = false; private static final char cdot = '.'; - public PathTokenizer(Reader input) { - super(input); + public PathTokenizer() { this.delimiter = DEFAULT_DELIMITER; } diff --git a/src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java b/src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java index 3a7f07849c9..5e031f99274 100644 --- a/src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java +++ b/src/org/opensolaris/opengrok/analysis/archive/BZip2Analyzer.java @@ -18,14 +18,13 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.archive; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.io.Writer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -107,10 +106,10 @@ public InputStream getStream() throws IOException { } @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { + public TokenStreamComponents createComponents(String fieldName) { if (fa != null) { - return fa.createComponents(fieldName, reader); + return fa.createComponents(fieldName); } - return super.createComponents(fieldName, reader); + return super.createComponents(fieldName); } } diff --git a/src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java b/src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java index a05b5d5ed71..8e161d02b4e 100644 --- a/src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/archive/GZIPAnalyzer.java @@ -18,14 +18,13 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.archive; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.io.Writer; import java.util.logging.Level; import java.util.zip.GZIPInputStream; @@ -108,10 +107,10 @@ public InputStream getStream() throws IOException { } @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { + public TokenStreamComponents createComponents(String fieldName) { if (fa != null) { - return fa.createComponents(fieldName, reader); + return fa.createComponents(fieldName); } - return super.createComponents(fieldName, reader); + return super.createComponents(fieldName); } } diff --git a/src/org/opensolaris/opengrok/analysis/c/CScopeParser.lex b/src/org/opensolaris/opengrok/analysis/c/CScopeParser.lex index b76cd3ac42a..2a4c95ae4c6 100644 --- a/src/org/opensolaris/opengrok/analysis/c/CScopeParser.lex +++ b/src/org/opensolaris/opengrok/analysis/c/CScopeParser.lex @@ -50,7 +50,6 @@ import org.opensolaris.opengrok.analysis.JFlexScopeParser; private void decScope() { if (level > 0) { level--; if (level == 0) { scope.lineTo = yyline; } } } %} -Identifier = [a-zA-Z_] [a-zA-Z0-9_]* %state STRING COMMENT SCOMMENT QSTRING diff --git a/src/org/opensolaris/opengrok/analysis/c/CSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/c/CSymbolTokenizer.lex index f0e03283eda..d35f4e1665e 100644 --- a/src/org/opensolaris/opengrok/analysis/c/CSymbolTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/c/CSymbolTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ /* @@ -33,9 +33,6 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class CSymbolTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} %type boolean %eofval{ return false; diff --git a/src/org/opensolaris/opengrok/analysis/c/CxxSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/c/CxxSymbolTokenizer.lex index fd1913fbbb7..655724a9b53 100644 --- a/src/org/opensolaris/opengrok/analysis/c/CxxSymbolTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/c/CxxSymbolTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.c; @@ -30,9 +30,6 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class CxxSymbolTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} %type boolean %eofval{ return false; diff --git a/src/org/opensolaris/opengrok/analysis/document/TroffAnalyzer.java b/src/org/opensolaris/opengrok/analysis/document/TroffAnalyzer.java index f9517888777..8624b367041 100644 --- a/src/org/opensolaris/opengrok/analysis/document/TroffAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/document/TroffAnalyzer.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.document; @@ -63,11 +63,11 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc } @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { + public TokenStreamComponents createComponents(String fieldName) { if ("full".equals(fieldName)) { - return new TokenStreamComponents(new TroffFullTokenizer(reader)); + return new TokenStreamComponents(new TroffFullTokenizer()); } - return super.createComponents(fieldName, reader); + return super.createComponents(fieldName); } /** diff --git a/src/org/opensolaris/opengrok/analysis/document/TroffFullTokenizer.lex b/src/org/opensolaris/opengrok/analysis/document/TroffFullTokenizer.lex index 2b6258e5f83..f2e9c202fb8 100644 --- a/src/org/opensolaris/opengrok/analysis/document/TroffFullTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/document/TroffFullTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.document; @@ -33,9 +33,11 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class TroffFullTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} +%{ +public TroffFullTokenizer() { +super(); +} +%} %type boolean %eofval{ return false; diff --git a/src/org/opensolaris/opengrok/analysis/java/JavaScopeParser.lex b/src/org/opensolaris/opengrok/analysis/java/JavaScopeParser.lex index 6cea725d3a3..5870bab8fd3 100644 --- a/src/org/opensolaris/opengrok/analysis/java/JavaScopeParser.lex +++ b/src/org/opensolaris/opengrok/analysis/java/JavaScopeParser.lex @@ -50,8 +50,6 @@ import org.opensolaris.opengrok.analysis.JFlexScopeParser; private void decScope() { if (level > 0) { level--; if (level == 0) { scope.lineTo = yyline; } } } %} -Identifier = [a-zA-Z_] [a-zA-Z0-9_]* - %state STRING COMMENT SCOMMENT QSTRING %% diff --git a/src/org/opensolaris/opengrok/analysis/plain/AbstractSourceCodeAnalyzer.java b/src/org/opensolaris/opengrok/analysis/plain/AbstractSourceCodeAnalyzer.java index 094b2960c3d..3f355a1eda3 100644 --- a/src/org/opensolaris/opengrok/analysis/plain/AbstractSourceCodeAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/plain/AbstractSourceCodeAnalyzer.java @@ -49,6 +49,7 @@ public abstract class AbstractSourceCodeAnalyzer extends PlainAnalyzer { /** * Creates a new instance of abstract analyzer + * @param factory for which analyzer to create this */ protected AbstractSourceCodeAnalyzer(FileAnalyzerFactory factory) { super(factory); @@ -74,12 +75,13 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc super.analyze(doc, src, xrefOut); } + Reader dummy=null; @Override - public Analyzer.TokenStreamComponents createComponents(String fieldName, Reader reader) { + public Analyzer.TokenStreamComponents createComponents(String fieldName) { if ("refs".equals(fieldName)) { - return new TokenStreamComponents(newSymbolTokenizer(reader)); + return new TokenStreamComponents(newSymbolTokenizer(dummy)); } - return super.createComponents(fieldName, reader); + return super.createComponents(fieldName); } /** diff --git a/src/org/opensolaris/opengrok/analysis/plain/PlainFullTokenizer.lex b/src/org/opensolaris/opengrok/analysis/plain/PlainFullTokenizer.lex index bffe36efabc..a05fbbfca11 100644 --- a/src/org/opensolaris/opengrok/analysis/plain/PlainFullTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/plain/PlainFullTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.plain; @@ -30,9 +30,11 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class PlainFullTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} +%{ +public PlainFullTokenizer() { +super(); +} +%} %type boolean %eofval{ this.finalOffset=zzEndRead; diff --git a/src/org/opensolaris/opengrok/analysis/plain/PlainSymbolTokenizer.lex b/src/org/opensolaris/opengrok/analysis/plain/PlainSymbolTokenizer.lex index 884b1b03495..5ba8f1a29ea 100644 --- a/src/org/opensolaris/opengrok/analysis/plain/PlainSymbolTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/plain/PlainSymbolTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.plain; @@ -28,9 +28,11 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class PlainSymbolTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} +%{ +public PlainSymbolTokenizer() { +super(); +} +%} %type boolean %eofval{ this.finalOffset=zzEndRead; diff --git a/src/org/opensolaris/opengrok/analysis/uue/UuencodeAnalyzer.java b/src/org/opensolaris/opengrok/analysis/uue/UuencodeAnalyzer.java index 4ef1f8b7b88..1228c2a0fdb 100644 --- a/src/org/opensolaris/opengrok/analysis/uue/UuencodeAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/uue/UuencodeAnalyzer.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.analysis.uue; @@ -62,11 +62,11 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc } @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { + public TokenStreamComponents createComponents(String fieldName) { if ("full".equals(fieldName)) { - return new TokenStreamComponents(new UuencodeFullTokenizer(reader)); + return new TokenStreamComponents(new UuencodeFullTokenizer()); } - return super.createComponents(fieldName, reader); + return super.createComponents(fieldName); } /** diff --git a/src/org/opensolaris/opengrok/analysis/uue/UuencodeFullTokenizer.lex b/src/org/opensolaris/opengrok/analysis/uue/UuencodeFullTokenizer.lex index 5605fd3d8ef..070bf860eb6 100644 --- a/src/org/opensolaris/opengrok/analysis/uue/UuencodeFullTokenizer.lex +++ b/src/org/opensolaris/opengrok/analysis/uue/UuencodeFullTokenizer.lex @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2013 Constantine A. Murenin */ @@ -34,9 +34,11 @@ import org.opensolaris.opengrok.analysis.JFlexTokenizer; %class UuencodeFullTokenizer %extends JFlexTokenizer %unicode -%init{ -super(in); -%init} +%{ +public UuencodeFullTokenizer() { +super(); +} +%} %type boolean %eofval{ return false; diff --git a/src/org/opensolaris/opengrok/index/IndexDatabase.java b/src/org/opensolaris/opengrok/index/IndexDatabase.java index 4eefbca3ccb..152b45d4251 100644 --- a/src/org/opensolaris/opengrok/index/IndexDatabase.java +++ b/src/org/opensolaris/opengrok/index/IndexDatabase.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.index; @@ -75,7 +75,6 @@ import org.opensolaris.opengrok.history.HistoryException; import org.opensolaris.opengrok.history.HistoryGuru; import org.opensolaris.opengrok.search.QueryBuilder; -import org.opensolaris.opengrok.search.SearchEngine; import org.opensolaris.opengrok.util.IOUtils; import org.opensolaris.opengrok.web.Util; @@ -84,7 +83,7 @@ * one index database per project. * * @author Trond Norbye - * @author Lubos Kosco , update for lucene 4.x + * @author Lubos Kosco , update for lucene 4.x , 5.x */ public class IndexDatabase { @@ -130,7 +129,7 @@ public IndexDatabase() throws IOException { */ public IndexDatabase(Project project) throws IOException { this.project = project; - lockfact = new SimpleFSLockFactory(); + lockfact = SimpleFSLockFactory.INSTANCE; initialize(); } @@ -260,9 +259,9 @@ private void initialize() throws IOException { } if (!env.isUsingLuceneLocking()) { - lockfact = NoLockFactory.getNoLockFactory(); + lockfact = NoLockFactory.INSTANCE; } - indexDirectory = FSDirectory.open(indexDir, lockfact); + indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact); ignoredNames = env.getIgnoredNames(); includedNames = env.getIncludedNames(); analyzerGuru = new AnalyzerGuru(); @@ -333,7 +332,7 @@ public void update() throws IOException, HistoryException { try { Analyzer analyzer = AnalyzerGuru.getAnalyzer(); - IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer); + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(RuntimeEnvironment.getInstance().getRamBufferSize()); writer = new IndexWriter(indexDirectory, iwc); @@ -500,7 +499,7 @@ public void optimize() { try { log.info("Optimizing the index ... "); Analyzer analyzer = new StandardAnalyzer(); - IndexWriterConfig conf = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer); + IndexWriterConfig conf = new IndexWriterConfig(analyzer); conf.setOpenMode(OpenMode.CREATE_OR_APPEND); wrt = new IndexWriter(indexDirectory, conf); @@ -626,7 +625,7 @@ private void addFile(File file, String path) throws IOException { } try { - writer.addDocument(doc, fa); + writer.addDocument(doc); } catch (Throwable t) { cleanupResources(doc); throw t; @@ -1090,7 +1089,7 @@ public static IndexReader getIndexReader(String path) { indexDir = new File(indexDir, p.getPath()); } try { - FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory()); + FSDirectory fdir = FSDirectory.open(indexDir.toPath(), NoLockFactory.INSTANCE); if (indexDir.exists() && DirectoryReader.indexExists(fdir)) { ret = DirectoryReader.open(fdir); } diff --git a/src/org/opensolaris/opengrok/search/SearchEngine.java b/src/org/opensolaris/opengrok/search/SearchEngine.java index 7c7dd379055..7aaa51fb431 100644 --- a/src/org/opensolaris/opengrok/search/SearchEngine.java +++ b/src/org/opensolaris/opengrok/search/SearchEngine.java @@ -120,8 +120,7 @@ public class SearchEngine { private final List docs; private final char[] content = new char[1024 * 8]; private String source; - private String data; - private static final boolean docsScoredInOrder = false; + private String data; int hitsPerPage = RuntimeEnvironment.getInstance().getHitsPerPage(); int cachePages = RuntimeEnvironment.getInstance().getCachePages(); int totalHits = 0; @@ -173,13 +172,13 @@ public boolean isValidQuery() { * @throws IOException */ private void searchSingleDatabase(File root, boolean paging) throws IOException { - IndexReader ireader = DirectoryReader.open(FSDirectory.open(root)); + IndexReader ireader = DirectoryReader.open(FSDirectory.open(root.toPath())); searcher = new IndexSearcher(ireader); - collector = TopScoreDocCollector.create(hitsPerPage * cachePages, docsScoredInOrder); + collector = TopScoreDocCollector.create(hitsPerPage * cachePages); searcher.search(query, collector); totalHits = collector.getTotalHits(); if (!paging && totalHits > 0) { - collector = TopScoreDocCollector.create(totalHits, docsScoredInOrder); + collector = TopScoreDocCollector.create(totalHits); searcher.search(query, collector); } hits = collector.topDocs().scoreDocs; @@ -202,7 +201,7 @@ private void searchMultiDatabase(List root, boolean paging) throws IOEx File droot = new File(RuntimeEnvironment.getInstance().getDataRootFile(), IndexDatabase.INDEX_DIR); int ii = 0; for (Project project : root) { - IndexReader ireader = (DirectoryReader.open(FSDirectory.open(new File(droot, project.getPath())))); + IndexReader ireader = (DirectoryReader.open(FSDirectory.open(new File(droot, project.getPath()).toPath()))); subreaders[ii++] = ireader; } MultiReader searchables = new MultiReader(subreaders, true); @@ -213,11 +212,11 @@ private void searchMultiDatabase(List root, boolean paging) throws IOEx } else { searcher = new IndexSearcher(searchables); } - collector = TopScoreDocCollector.create(hitsPerPage * cachePages, docsScoredInOrder); + collector = TopScoreDocCollector.create(hitsPerPage * cachePages); searcher.search(query, collector); totalHits = collector.getTotalHits(); if (!paging && totalHits > 0) { - collector = TopScoreDocCollector.create(totalHits, docsScoredInOrder); + collector = TopScoreDocCollector.create(totalHits); searcher.search(query, collector); } hits = collector.topDocs().scoreDocs; @@ -319,7 +318,7 @@ public void results(int start, int end, List ret) { //TODO check if below fits for if end=old hits.length, or it should include it if (end > hits.length & !allCollected) { //do the requery, we want more than 5 pages - collector = TopScoreDocCollector.create(totalHits, docsScoredInOrder); + collector = TopScoreDocCollector.create(totalHits); try { searcher.search(query, collector); } catch (Exception e) { // this exception should never be hit, since search() will hit this before diff --git a/src/org/opensolaris/opengrok/web/SearchHelper.java b/src/org/opensolaris/opengrok/web/SearchHelper.java index bc7c0d5131d..6fb5e8af35e 100644 --- a/src/org/opensolaris/opengrok/web/SearchHelper.java +++ b/src/org/opensolaris/opengrok/web/SearchHelper.java @@ -19,7 +19,7 @@ /* * Copyright (c) 2011 Jens Elkner. - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. */ package org.opensolaris.opengrok.web; @@ -263,12 +263,12 @@ public SearchHelper prepareExec(SortedSet projects) { this.projects = projects; if (projects.isEmpty()) { //no project setup - FSDirectory dir = FSDirectory.open(indexDir); + FSDirectory dir = FSDirectory.open(indexDir.toPath()); searcher = new IndexSearcher(DirectoryReader.open(dir)); } else if (projects.size() == 1) { // just 1 project selected FSDirectory dir = - FSDirectory.open(new File(indexDir, projects.first())); + FSDirectory.open(new File(indexDir, projects.first()).toPath()); searcher = new IndexSearcher(DirectoryReader.open(dir)); } else { //more projects @@ -277,7 +277,7 @@ public SearchHelper prepareExec(SortedSet projects) { //TODO might need to rewrite to Project instead of // String , need changes in projects.jspf too for (String proj : projects) { - FSDirectory dir = FSDirectory.open(new File(indexDir, proj)); + FSDirectory dir = FSDirectory.open(new File(indexDir, proj).toPath()); subreaders[ii++] = DirectoryReader.open(dir); } MultiReader searchables = new MultiReader(subreaders, true); @@ -310,7 +310,7 @@ public SearchHelper prepareExec(SortedSet projects) { } catch (FileNotFoundException e) { // errorMsg = "Index database(s) not found: " + e.getMessage(); errorMsg = "Index database(s) not found."; - } catch (Exception e) { + } catch (IOException e) { errorMsg = e.getMessage(); } return this; @@ -370,7 +370,7 @@ public SearchHelper executeQuery() { } } catch (BooleanQuery.TooManyClauses e) { errorMsg = "Too many results for wildcard!"; - } catch (Exception e) { + } catch (IOException | ClassNotFoundException e) { errorMsg = e.getMessage(); } return this; @@ -383,14 +383,12 @@ private void getSuggestion(Term term, IndexReader ir, return; } String[] toks = TABSPACE.split(term.text(), 0); - for (int j = 0; j < toks.length; j++) { - //TODO below seems to be case insensitive ... for refs/defs this is bad - SuggestWord[] words=checker.suggestSimilar( - new Term(term.field(),toks[j]), SPELLCHECK_SUGGEST_WORD_COUNT, ir, - SuggestMode.SUGGEST_ALWAYS); - for (SuggestWord w: words) { - result.add(w.string); - } + for (String tok : toks) { + //TODO below seems to be case insensitive ... for refs/defs this is bad + SuggestWord[] words = checker.suggestSimilar(new Term(term.field(), tok), SPELLCHECK_SUGGEST_WORD_COUNT, ir, SuggestMode.SUGGEST_ALWAYS); + for (SuggestWord w: words) { + result.add(w.string); + } } } @@ -425,39 +423,39 @@ public List getSuggestions() { FSDirectory dir; IndexReader ir=null; Term t; - for (int idx = 0; idx < name.length; idx++) { - Suggestion s = new Suggestion(name[idx]); + for (String proj : name) { + Suggestion s = new Suggestion(proj); try { - dir = FSDirectory.open(new File(indexDir, name[idx])); - ir = DirectoryReader.open(dir); - if (builder.getFreetext()!=null && - !builder.getFreetext().isEmpty()) { - t=new Term(QueryBuilder.FULL,builder.getFreetext()); - getSuggestion(t, ir, dummy); - s.freetext = dummy.toArray(new String[dummy.size()]); - dummy.clear(); - } - if (builder.getRefs()!=null && !builder.getRefs().isEmpty()) { - t=new Term(QueryBuilder.REFS,builder.getRefs()); - getSuggestion(t, ir, dummy); - s.refs = dummy.toArray(new String[dummy.size()]); - dummy.clear(); - } + dir = FSDirectory.open(new File(indexDir, proj).toPath()); + ir = DirectoryReader.open(dir); + if (builder.getFreetext()!=null && + !builder.getFreetext().isEmpty()) { + t=new Term(QueryBuilder.FULL,builder.getFreetext()); + getSuggestion(t, ir, dummy); + s.freetext = dummy.toArray(new String[dummy.size()]); + dummy.clear(); + } + if (builder.getRefs()!=null && !builder.getRefs().isEmpty()) { + t=new Term(QueryBuilder.REFS,builder.getRefs()); + getSuggestion(t, ir, dummy); + s.refs = dummy.toArray(new String[dummy.size()]); + dummy.clear(); + } if (builder.getDefs()!=null && !builder.getDefs().isEmpty()) { - t=new Term(QueryBuilder.DEFS,builder.getDefs()); - getSuggestion(t, ir, dummy); - s.defs = dummy.toArray(new String[dummy.size()]); - dummy.clear(); - } - //TODO suggest also for path and history? - if ((s.freetext!=null && s.freetext.length > 0) || - (s.defs!=null && s.defs.length > 0) || - (s.refs!=null && s.refs.length > 0) ) { + t=new Term(QueryBuilder.DEFS,builder.getDefs()); + getSuggestion(t, ir, dummy); + s.defs = dummy.toArray(new String[dummy.size()]); + dummy.clear(); + } + //TODO suggest also for path and history? + if ((s.freetext!=null && s.freetext.length > 0) || + (s.defs!=null && s.defs.length > 0) || + (s.refs!=null && s.refs.length > 0) ) { res.add(s); } - } catch (IOException e) { + }catch (IOException e) { log.log(Level.WARNING, "Got exception while getting " - + "spelling suggestions: ", e); + + "spelling suggestions: ", e); } finally { if (ir != null) { try { @@ -467,8 +465,8 @@ public List getSuggestions() { + "getting spelling suggestions: ", ex); } } - } - } + } + } return res; } diff --git a/test/org/opensolaris/opengrok/analysis/PathTokenizerTest.java b/test/org/opensolaris/opengrok/analysis/PathTokenizerTest.java index 42ea2c93537..0a57cff763a 100644 --- a/test/org/opensolaris/opengrok/analysis/PathTokenizerTest.java +++ b/test/org/opensolaris/opengrok/analysis/PathTokenizerTest.java @@ -42,7 +42,8 @@ public class PathTokenizerTest { public void testIncrementToken() throws Exception { String inputText = "alpha/beta/gamma/delta.ext"; String[] expectedTokens = inputText.split("[/.]"); - PathTokenizer tokenizer = new PathTokenizer(new StringReader(inputText)); + PathTokenizer tokenizer = new PathTokenizer(); + tokenizer.setReader(new StringReader(inputText)); CharTermAttribute term = tokenizer.addAttribute(CharTermAttribute.class); OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class); diff --git a/test/org/opensolaris/opengrok/analysis/php/PhpSymbolTokenizerTest.java b/test/org/opensolaris/opengrok/analysis/php/PhpSymbolTokenizerTest.java index 9b8a41cc016..e0607ef9929 100644 --- a/test/org/opensolaris/opengrok/analysis/php/PhpSymbolTokenizerTest.java +++ b/test/org/opensolaris/opengrok/analysis/php/PhpSymbolTokenizerTest.java @@ -9,6 +9,8 @@ import java.io.StringReader; import java.util.LinkedList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.junit.Test; import org.opensolaris.opengrok.analysis.FileAnalyzer; @@ -34,7 +36,13 @@ private String[] getTermsFor(String s) { private String[] getTermsFor(Reader r) { List l = new LinkedList(); - JFlexTokenizer ts = (JFlexTokenizer) this.analyzer.createComponents("refs", r).getTokenStream(); + JFlexTokenizer ts = (JFlexTokenizer) this.analyzer.createComponents("refs").getTokenStream(); + try { + ts.setReader(r); + } catch (IOException ex) { + Logger.getLogger(PhpSymbolTokenizerTest.class.getName()).log(Level.SEVERE, null, ex); + throw new RuntimeException(ex); + } ts.yyreset(r); CharTermAttribute term = ts.addAttribute(CharTermAttribute.class); try { From 7f9f5df05a522b79dcf288e598e0a2dbf6980cbe Mon Sep 17 00:00:00 2001 From: Lubos Kosco Date: Fri, 20 Feb 2015 19:07:50 +0100 Subject: [PATCH 2/2] lucene 5 mvn, pkgs --- platform/solaris/ips/create.sh | 2 +- platform/solaris/pkgdef/prototype | 8 ++++---- pom.xml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platform/solaris/ips/create.sh b/platform/solaris/ips/create.sh index 2ce0899680b..810f0302152 100755 --- a/platform/solaris/ips/create.sh +++ b/platform/solaris/ips/create.sh @@ -209,7 +209,7 @@ PKG pkgsend add file doc/ctags.config mode=0444 owner=root group=sys path=/usr/o # install libs -LV=4.10.3 +LV=5.0.0 for file in ant.jar bcel-5.2.jar \ lucene-analyzers-common-${LV}.jar lucene-core-${LV}.jar lucene-queryparser-${LV}.jar lucene-suggest-${LV}.jar \ jrcs.jar \ diff --git a/platform/solaris/pkgdef/prototype b/platform/solaris/pkgdef/prototype index 34b0a51cf47..cda4ca43131 100644 --- a/platform/solaris/pkgdef/prototype +++ b/platform/solaris/pkgdef/prototype @@ -60,10 +60,10 @@ f none usr/opengrok/doc/opengrok-agent.properties=platform/solaris/default/openg f none usr/opengrok/lib/opengrok.jar=dist/opengrok.jar 0444 root bin f none usr/opengrok/lib/ant.jar=dist/lib/ant.jar 0444 root bin f none usr/opengrok/lib/bcel-5.2.jar=lib/bcel-5.2.jar 0444 root bin -f none usr/opengrok/lib/lucene-core-4.10.3.jar=lib/lucene-core-4.10.3.jar 0444 root bin -f none usr/opengrok/lib/lucene-analyzers-common-4.10.3.jar=lib/lucene-analyzers-common-4.10.3.jar 0444 root bin -f none usr/opengrok/lib/lucene-queryparser-4.10.3.jar=lib/lucene-queryparser-4.10.3.jar 0444 root bin -f none usr/opengrok/lib/lucene-suggest-4.10.3.jar=lib/lucene-suggest-4.10.3.jar 0444 root bin +f none usr/opengrok/lib/lucene-core-5.0.0.jar=lib/lucene-core-5.0.0.jar 0444 root bin +f none usr/opengrok/lib/lucene-analyzers-common-5.0.0.jar=lib/lucene-analyzers-common-5.0.0.jar 0444 root bin +f none usr/opengrok/lib/lucene-queryparser-5.0.0.jar=lib/lucene-queryparser-5.0.0.jar 0444 root bin +f none usr/opengrok/lib/lucene-suggest-5.0.0.jar=lib/lucene-suggest-5.0.0.jar 0444 root bin f none usr/opengrok/lib/jrcs.jar=lib/jrcs.jar 0444 root bin f none usr/opengrok/lib/swing-layout-0.9.jar=lib/swing-layout-0.9.jar 0444 root bin f none usr/opengrok/lib/json-simple-1.1.1.jar=lib/json-simple-1.1.1.jar 0444 root bin diff --git a/pom.xml b/pom.xml index c2e52c39aa1..3bde894dd6f 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ - 4.10.3 + 5.0.0 2.9.1 1.7 1.7 @@ -47,7 +47,7 @@ junit junit - 4.11 + 4.12 org.apache.bcel @@ -87,12 +87,12 @@ org.apache.ant ant - 1.9.2 + 1.9.4 org.apache.derby derby - 10.10.1.1 + 10.11.1.1