Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
mcpgovernance committed Oct 7, 2012
1 parent 7465980 commit 8a2f3b5
Show file tree
Hide file tree
Showing 17 changed files with 397 additions and 275 deletions.
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -334,12 +334,12 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>2.9.2</version>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers</artifactId>
<version>2.9.2</version>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.jcs</groupId>
Expand Down
52 changes: 52 additions & 0 deletions web/src/main/java/org/fao/geonet/exceptions/SearchExpiredEx.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
//=== This library is free software; you can redistribute it and/or
//=== modify it under the terms of the GNU Lesser General Public
//=== License as published by the Free Software Foundation; either
//=== version 2.1 of the License, or (at your option) any later version.
//===
//=== This library is distributed in the hope that it will be useful,
//=== but WITHOUT ANY WARRANTY; without even the implied warranty of
//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//=== Lesser General Public License for more details.
//===
//=== You should have received a copy of the GNU Lesser General Public
//=== License along with this library; if not, write to the Free Software
//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
//===
//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
//=== Rome - Italy. email: geonetwork@osgeo.org
//==============================================================================

package org.fao.geonet.exceptions;

import jeeves.exceptions.OperationAbortedEx;

//=============================================================================

public class SearchExpiredEx extends OperationAbortedEx
{
//--------------------------------------------------------------------------
//---
//--- Constructor
//---
//--------------------------------------------------------------------------

/**
*
*/
private static final long serialVersionUID = -5759127829247984311L;

public SearchExpiredEx(String name)
{
super("Search expired", name);

id = "search-expired";
}
}

//=============================================================================

Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import jeeves.utils.Log;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.MapFieldSelector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
Expand Down Expand Up @@ -178,13 +178,14 @@ public static List<Element> handlePropertyName(String[] propertyNames,
GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
SearchManager sm = gc.getSearchmanager();

IndexReader reader = sm.getIndexReader();
IndexSearcher searcher = sm.getNewIndexSearcher().two();

try {
Query query = CatalogSearcher.getGroupsQuery(context);
Sort sort = LuceneSearcher.makeSort(Collections.singletonList(Pair.read(Geonet.SearchResult.SortBy.RELEVANCE, true)));
CachingWrapperFilter filter = null;

Pair<TopDocs,Element> searchResults = LuceneSearcher.doSearchAndMakeSummary( maxRecords, 0, maxRecords, Integer.MAX_VALUE, context.getLanguage(), "results", new Element("summary"), reader, query, filter, sort, false);
Pair<TopDocs,Element> searchResults = LuceneSearcher.doSearchAndMakeSummary( maxRecords, 0, maxRecords, Integer.MAX_VALUE, context.getLanguage(), "results", new Element("summary"), searcher, query, filter, sort, false);
TopDocs hits = searchResults.one();

try {
Expand All @@ -195,8 +196,7 @@ public static List<Element> handlePropertyName(String[] propertyNames,
property = indexField;

// check if params asked is in the index using getFieldNames ?
if (!reader.getFieldNames(IndexReader.FieldOption.ALL)
.contains(property))
if (searcher.getIndexReader().getFieldInfos().fieldInfo(property) == null)
continue;

boolean isRange = false;
Expand All @@ -220,7 +220,7 @@ public static List<Element> handlePropertyName(String[] propertyNames,
SortedSet<String> sortedValues = new TreeSet<String>();
HashMap<String, Integer> duplicateValues = new HashMap<String, Integer>();
for (int j = 0; j < hits.scoreDocs.length; j++) {
Document doc = reader.document(hits.scoreDocs[j].doc, selector);
Document doc = searcher.getIndexReader().document(hits.scoreDocs[j].doc, selector);

// Skip templates and subTemplates
String[] isTemplate = doc.getValues("_isTemplate");
Expand Down Expand Up @@ -254,7 +254,7 @@ public static List<Element> handlePropertyName(String[] propertyNames,
domainValuesList.add(domainValues);
}
} finally {
sm.releaseIndexReader(reader);
sm.releaseIndexSearcher(searcher);
}
}
return domainValuesList;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
Expand All @@ -51,6 +52,7 @@
import org.fao.geonet.csw.common.exceptions.CatalogException;
import org.fao.geonet.csw.common.exceptions.InvalidParameterValueEx;
import org.fao.geonet.csw.common.exceptions.NoApplicableCodeEx;
import org.fao.geonet.exceptions.SearchExpiredEx;
import org.fao.geonet.kernel.AccessManager;
import org.fao.geonet.kernel.search.LuceneSearcher;
import org.fao.geonet.kernel.search.LuceneUtils;
Expand Down Expand Up @@ -84,6 +86,7 @@ public class CatalogSearcher {
private CachingWrapperFilter _filter;
private Sort _sort;
private String _lang;
private long _searchToken;

public CatalogSearcher(Element summaryConfig, FieldSelector selector,
FieldSelector uuidselector, HashSet<String> tokenizedFieldSet,
Expand All @@ -97,6 +100,8 @@ public CatalogSearcher(Element summaryConfig, FieldSelector selector,
_selector = selector;
_uuidselector = uuidselector;
_summaryConfig = summaryConfig;
_searchToken = -1L; // means we will get a new IndexSearcher when we
// ask for it first time
}

// ---------------------------------------------------------------------------
Expand All @@ -121,22 +126,41 @@ public Pair<Element, List<ResultItem>> search(ServiceContext context,
checkForErrors(luceneExpr);
remapFields(luceneExpr);
}


GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
SearchManager sm = gc.getSearchmanager();
IndexSearcher searcher = null;
try {
if (luceneExpr != null) {
convertPhrases(luceneExpr);
}

return performSearch(context,
Pair<Long,IndexSearcher> searcherPair = sm.getIndexSearcher(_searchToken);
Log.debug(Geonet.CSW_SEARCH,"Found searcher with "+searcherPair.one()+" comparing with "+_searchToken);
if (_searchToken != -1L && searcherPair.one() != _searchToken) {
throw new SearchExpiredEx("Search has expired/timed out - start a new search");
}
_searchToken = searcherPair.one();
searcher = searcherPair.two();
return performSearch(context,
luceneExpr, filterExpr, filterVersion, sort, resultType,
startPosition, maxRecords, maxHitsInSummary);
startPosition, maxRecords, maxHitsInSummary, searcher);
} catch (Exception e) {
Log.error(Geonet.CSW_SEARCH, "Error while searching metadata ");
Log.error(Geonet.CSW_SEARCH, " (C) StackTrace:\n"
+ Util.getStackTrace(e));

throw new NoApplicableCodeEx(
"Raised exception while searching metadata : " + e);
} finally {
try {
if (searcher != null) sm.releaseIndexSearcher(searcher);
} catch (Exception ex) { // eat it as it probably doesn't matter,
// but say what happened anyway
Log.error(Geonet.CSW_SEARCH, "Error while releasing index searcher ");
Log.error(Geonet.CSW_SEARCH, " (C) StackTrace:\n"
+ Util.getStackTrace(ex));
}
}
}

Expand All @@ -157,11 +181,11 @@ public List<String> getAllUuids(int maxHits, ServiceContext context) throws Exce
GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
SearchManager sm = gc.getSearchmanager();

IndexReader _reader = sm.getIndexReader();
IndexSearcher searcher = sm.getIndexSearcher(_searchToken).two();
try {
Pair<TopDocs, Element> searchResults =
LuceneSearcher.doSearchAndMakeSummary(
maxHits, 0, maxHits, Integer.MAX_VALUE, _lang, ResultType.RESULTS.toString(), _summaryConfig, _reader, _query, _filter, _sort, false);
maxHits, 0, maxHits, Integer.MAX_VALUE, _lang, ResultType.RESULTS.toString(), _summaryConfig, searcher, _query, _filter, _sort, false);
TopDocs tdocs = searchResults.one();
Element summary = searchResults.two();

Expand All @@ -173,13 +197,13 @@ public List<String> getAllUuids(int maxHits, ServiceContext context) throws Exce
List<String> response = new ArrayList<String>();

for ( ScoreDoc sdoc : tdocs.scoreDocs ) {
Document doc = _reader.document(sdoc.doc, _uuidselector);
Document doc = searcher.getIndexReader().document(sdoc.doc, _uuidselector);
String uuid = doc.get("_uuid");
if (uuid != null) response.add(uuid);
}
return response;
} finally {
sm.releaseIndexReader(_reader);
sm.releaseIndexSearcher(searcher);
}
}

Expand Down Expand Up @@ -303,13 +327,12 @@ private void remapFields(Element elem) {
private Pair<Element, List<ResultItem>> performSearch(
ServiceContext context, Element luceneExpr, Element filterExpr,
String filterVersion, Sort sort, ResultType resultType,
int startPosition, int maxRecords, int maxHitsInSummary) throws Exception {
int startPosition, int maxRecords, int maxHitsInSummary,
IndexSearcher searcher) throws Exception {

GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
SearchManager sm = gc.getSearchmanager();

IndexReader indexReader = sm.getIndexReader();
try {
if (luceneExpr != null) {
Log.debug(Geonet.CSW_SEARCH, "Search criteria:\n" + Xml.getString(luceneExpr));
}
Expand Down Expand Up @@ -365,7 +388,7 @@ private Pair<Element, List<ResultItem>> performSearch(
_lang = context.getLanguage();

Pair<TopDocs,Element> searchResults = LuceneSearcher.doSearchAndMakeSummary(numHits, startPosition - 1,
maxRecords, Integer.MAX_VALUE, context.getLanguage(), resultType.toString(), _summaryConfig, indexReader, query, cFilter,
maxRecords, Integer.MAX_VALUE, context.getLanguage(), resultType.toString(), _summaryConfig, searcher, query, cFilter,
sort, buildSummary
);
TopDocs hits = searchResults.one();
Expand All @@ -389,7 +412,7 @@ private Pair<Element, List<ResultItem>> performSearch(
}

for (;i < iMax; i++) {
Document doc = indexReader.document(hits.scoreDocs[i].doc, _selector);
Document doc = searcher.getIndexReader().document(hits.scoreDocs[i].doc, _selector);
String id = doc.get("_id");

ResultItem ri = new ResultItem(id);
Expand All @@ -406,9 +429,6 @@ private Pair<Element, List<ResultItem>> performSearch(
summary.setName("Summary");
summary.setNamespace(Csw.NAMESPACE_GEONET);
return Pair.read(summary, results);
} finally {
sm.releaseIndexReader(indexReader);
}
}

// ---------------------------------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import jeeves.server.UserSession;
import jeeves.server.context.ServiceContext;
import jeeves.utils.Log;
import jeeves.utils.Util;
import jeeves.utils.Xml;
import org.apache.commons.lang.StringUtils;
Expand All @@ -43,7 +44,7 @@
import org.fao.geonet.csw.common.exceptions.NoApplicableCodeEx;
import org.fao.geonet.kernel.SelectionManager;
import org.fao.geonet.kernel.search.spatial.Pair;
import org.fao.geonet.util.spring.StringUtils;
//import org.fao.geonet.util.spring.StringUtils;
import org.jdom.Element;

import javax.xml.transform.Source;
Expand Down Expand Up @@ -165,7 +166,26 @@ public final FieldSelectorResult accept(String name) {
//---------------------------------------------------------------------------

/**
* Performs the general search tasks.
* Performs the general search tasks. Brief explanation of what happens
* here:
*
* This is a stateful search because when a search is
* first initiated a snapshot of the Lucene index must be captured (via an
* IndexSearcher session id) and held in the user session
* so that the user can page through/request pages of results from the
* original search result without effects being introduced by changes to
* the Lucene index after the original search (eg. user adds a new
* metadata record, makes changes, deletes one or more records -
* these should not appear in the search results otherwise funny
* things happen such as more or less records returned than the number
* originally found etc).
*
* If no previous search in this user session then create a new
* CatalogSearcher (which captures a view of the Index). If there is a
* previous CatalogSearcher in the session then retrieve and use it.
* If the query is different from the one we have in the user session
* then we throw away the CatalogSearcher and get a new one with a new view
* of the Lucene index.
*
* @param context
* @param startPos
Expand All @@ -181,32 +201,44 @@ public final FieldSelectorResult accept(String name) {
* @return
* @throws CatalogException
*/
public Pair<Element, Element> search(ServiceContext context, int startPos, int maxRecords,
ResultType resultType, OutputSchema outSchema, ElementSetName setName,
Element filterExpr, String filterVersion, Sort sort,
Set<String> elemNames, int maxHitsFromSummary) throws CatalogException {
public Pair<Element, Element> search(ServiceContext context, int startPos, int maxRecords, ResultType resultType, OutputSchema outSchema, ElementSetName setName, Element filterExpr, String filterVersion, Sort sort, Set<String> elemNames, int maxHitsFromSummary) throws CatalogException {


Element results = new Element("SearchResults", Csw.NAMESPACE_CSW);
CatalogSearcher searcher = new CatalogSearcher(_summaryConfig, _selector, _uuidselector, _tokenizedFieldSet, _longFieldSet, _integerFieldSet, _floatFieldSet, _doubleFieldSet);

UserSession session = context.getUserSession();
session.setProperty(Geonet.Session.SEARCH_RESULT, searcher);
UserSession session = context.getUserSession();

// search for results, filtered and sorted
Pair<Element, List<ResultItem>> summaryAndSearchResults = searcher.search(context, filterExpr, filterVersion, sort, resultType, startPos, maxRecords, maxHitsFromSummary);

// clear selection from session when query filter change
// build an id from the query filter to see whether the filter has changed
// from the one we already had
String requestId = Util.scramble(Xml.getString(filterExpr));
String sessionRequestId = (String) session.getProperty(Geonet.Session.SEARCH_REQUEST_ID);

// possibly close old selection if at least one previous search
if (sessionRequestId != null && !sessionRequestId.equals(requestId)) {
// possibly close old selection
SelectionManager oldSelection = (SelectionManager)session.getProperty(Geonet.Session.SELECTED_RESULT);
if (sessionRequestId != null) {
SelectionManager oldSelection = (SelectionManager)session.getProperty(Geonet.Session.SELECTED_RESULT);

if (oldSelection != null){
oldSelection.close();
oldSelection = null;
}
}
}

// initialize catalog searcher and a new view of the index and set into
// user session if there is no session or this is a new query
CatalogSearcher searcher = (CatalogSearcher)session.getProperty(Geonet.Session.SEARCH_RESULT);
if (searcher == null || !sessionRequestId.equals(requestId)) {
Log.debug(Geonet.CSW_SEARCH,"Creating new catalog searcher");

if (oldSelection != null){
oldSelection.close();
oldSelection = null;
}
searcher = new CatalogSearcher(_summaryConfig, _selector, _uuidselector, _tokenizedFieldSet, _longFieldSet, _integerFieldSet, _floatFieldSet, _doubleFieldSet);
session.setProperty(Geonet.Session.SEARCH_RESULT, searcher);
} else {
Log.debug(Geonet.CSW_SEARCH,"Using existing catalog searcher");
}
session.setProperty(Geonet.Session.SEARCH_REQUEST_ID, requestId);

// search for results, filtered and sorted
Pair<Element, List<ResultItem>> summaryAndSearchResults = searcher.search(context, filterExpr, filterVersion, sort, resultType, startPos, maxRecords, maxHitsFromSummary);

List<ResultItem> resultsList = summaryAndSearchResults.two();
int counter = Math.min(maxRecords,resultsList.size());
Expand Down Expand Up @@ -376,7 +408,7 @@ private static String createElementNameTransformation(Set<String> elemNames) {
"<xsl:copy>\n";

for (String xpath : elemNames) {
if(StringUtils.hasLength(xpath)) {
if(org.fao.geonet.util.spring.StringUtils.hasLength(xpath)) {
result += "<xsl:apply-templates select=\"" + xpath + "\"/>\n";
}
}
Expand Down
Loading

0 comments on commit 8a2f3b5

Please sign in to comment.