From c5d689d452de83a4c3043e4d0d4dcecd1b626b1f Mon Sep 17 00:00:00 2001
From: Gert Schmeltz Pedersen <gspe@dtic.dtu.dk>
Date: Thu, 22 Dec 2011 12:48:06 +0100
Subject: [PATCH] FCREPO-1009 Interaction with the Resource Index

---
 FedoraGenericSearch/src/css/fgseu.css         | 32 +++++++
 .../src/html/fedoragsearch-doc.html           |  3 +-
 .../defxws/fedoragsearch/server/Config.java   |  9 +-
 .../server/GenericOperationsImpl.java         |  6 +-
 .../rest/gsearchToRisearch.xslt               |  4 +-
 .../dk/defxws/fgslucene/OperationsImpl.java   | 65 ++++++++------
 .../java/dk/defxws/fgslucene/Statement.java   | 17 ++++
 .../dk/defxws/fgssolr/OperationsImpl.java     | 89 ++++++++++++++-----
 README.txt                                    |  2 +-
 9 files changed, 167 insertions(+), 60 deletions(-)

diff --git a/FedoraGenericSearch/src/css/fgseu.css b/FedoraGenericSearch/src/css/fgseu.css
index d65d1a0..04ebc45 100644
--- a/FedoraGenericSearch/src/css/fgseu.css
+++ b/FedoraGenericSearch/src/css/fgseu.css
@@ -224,6 +224,38 @@
 	.fgseuHitscore {
 		color: green;
 	}
+
+.hitno { 
+    font-family: Arial, Helvetica, sans-serif;
+    font-size: 14px;
+}
+
+.hitscore { 
+    font-family: Arial, Helvetica, sans-serif;
+    font-size: 12px;
+    margin: 10px;
+}
+
+.hittitle { 
+    font-family: Verdana;
+    font-wight: bold;
+    font-size: 20px;
+}
+
+.hitfield { 
+    font-family: Arial, Helvetica, sans-serif;
+    font-size: 14px;
+    font-style: italic;
+	text-align: center;
+    margin: 10px;
+    color: green;
+}
+
+.hitsnippet { 
+    font-family: Arial, Helvetica, sans-serif;
+    font-size: 14px;
+    margin: 10px;
+}
 	
 	/** Right Column Area**/
 	#fgseuRightColumnArea {
diff --git a/FedoraGenericSearch/src/html/fedoragsearch-doc.html b/FedoraGenericSearch/src/html/fedoragsearch-doc.html
index a6f20dc..0b8e8e6 100644
--- a/FedoraGenericSearch/src/html/fedoragsearch-doc.html
+++ b/FedoraGenericSearch/src/html/fedoragsearch-doc.html
@@ -741,7 +741,7 @@ <h4><a href="http://www.indexdata.dk/zebra/">Zebra</a></h4>
 						</ul>
 						<ul>
 							<li>RISEARCH with GSEARCH: 
-							<a href="rest?operation=gfindObjects&restXslt=copyXml&query=(::RISEARCH::type=tuples%26lang=itql%26format=Sparql%26query=select+$obj1+from+%3C%23ri%3E+where+$obj1+%3Cinfo:fedora/fedora-system:def/relations-external%23isMemberOf%3E+%3Cinfo:fedora/demo:SmileyStuff%3E+and+(::GSEARCH::xsltName/gsearchToRisearch?operation=gfindObjects%26restXslt=copyXml%26query=dc.creator:apache::GSEARCH::)::RISEARCH::)">an itql search with embedded GSearch</a>
+							<a href="rest?operation=gfindObjects&restXslt=copyXml&query=(::RISEARCH::type=tuples%26lang=itql%26format=Sparql%26query=select+$obj1+from+%3C%23ri%3E+where+$obj1+%3Cinfo:fedora/fedora-system:def/relations-external%23isMemberOf%3E+%3Cinfo:fedora/demo:SmileyStuff%3E+or+(::GSEARCH::xsltName/gsearchToRisearch?operation=gfindObjects%26restXslt=copyXml%26query=dc.creator:apache::GSEARCH::)::RISEARCH::)">an itql search with embedded GSearch</a>
 							</li>
 						</ul>
 						<ul>
@@ -786,7 +786,6 @@ <h4><a href="http://www.indexdata.dk/zebra/">Zebra</a></h4>
     ant junit-lucene  
     ant junit-testsonlucene
     ant junit-fgs23  
-    ant junit-lucene-fgs24_1009 
     ant junit-lucene-fgs24_1010 
     ant junit-lucene-fgs24_1019  
 </pre>
diff --git a/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/Config.java b/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/Config.java
index 86ad505..b092f57 100644
--- a/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/Config.java
+++ b/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/Config.java
@@ -962,8 +962,13 @@ public String getGfindObjectsResultXslt(String indexName, String resultPageXslt)
     }
 
 	public String getSortFields(String indexName, String sortFields) {
-        if (sortFields==null || sortFields.equals("")) 
-            return (getIndexProps(indexName)).getProperty("fgsindex.defaultSortFields");
+        if (sortFields==null || sortFields.equals("")) {
+            String sf = (getIndexProps(indexName)).getProperty("fgsindex.defaultSortFields");
+            if (sf==null) {
+            	return "";
+            }
+            return sf;
+        }
         else 
             return sortFields;
     }
diff --git a/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/GenericOperationsImpl.java b/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/GenericOperationsImpl.java
index b023d64..0938ec8 100644
--- a/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/GenericOperationsImpl.java
+++ b/FedoraGenericSearch/src/java/dk/defxws/fedoragsearch/server/GenericOperationsImpl.java
@@ -217,6 +217,8 @@ public String gfindObjects(
         params[9] = indexName;
         params[10] = "SORTFIELDS";
         params[11] = sortFields;
+        params[12] = "RESULTPAGEXSLT";
+        params[13] = resultPageXslt;
         params[14] = "FGSUSERNAME";
         params[15] = fgsUserName;
         params[16] = "SRFTYPE";
@@ -374,7 +376,7 @@ private String processEmbeddedQuery(String embedType, String embeddedQuery)
 		}
         if (logger.isDebugEnabled())
             logger.debug("processEmbeddedQuery after get content");
-        params = new String[12];
+        String[] params = new String[12];
         params[0] = "OPERATION";
         params[1] = "gfindObjects";
         params[2] = "ACTION";
@@ -393,7 +395,7 @@ private String processEmbeddedQuery(String embedType, String embeddedQuery)
         		new StreamSource(content),
     			config.getURIResolver(embeddedIndexName),
                 params);
-        String newQueryPart = resultXml.toString();
+        String newQueryPart = resultXml.toString().replaceAll("&lt;", "<").replaceAll("&gt;", ">");
         String findString = "result:newQueryPart xmlns:result=\"http://www.w3.org/2001/sw/DataAccess/rf1/result\">";
         i = resultXml.indexOf(findString);
         if (i>-1) {
diff --git a/FgsConfig/FgsConfigTemplate/rest/gsearchToRisearch.xslt b/FgsConfig/FgsConfigTemplate/rest/gsearchToRisearch.xslt
index c302fae..f15be48 100644
--- a/FgsConfig/FgsConfigTemplate/rest/gsearchToRisearch.xslt
+++ b/FgsConfig/FgsConfigTemplate/rest/gsearchToRisearch.xslt
@@ -10,8 +10,8 @@
 			<xsl:text>(</xsl:text>
 			<xsl:for-each select="//object"> 
 				<xsl:if test="position()>1"> or </xsl:if>
-				<xsl:text>$obj1 &lt;mulgara:is&gt;</xsl:text> 
-				<xsl:text>&lt;info:fedora/</xsl:text><xsl:value-of select="field[@name='PID']"/><xsl:text>&gt;</xsl:text>
+				<xsl:text>$obj1 %3Cmulgara:is%3E </xsl:text> 
+				<xsl:text>%3Cinfo:fedora/</xsl:text><xsl:value-of select="field[@name='PID']"/><xsl:text>%3E</xsl:text>
 			</xsl:for-each>
 			<xsl:text>)</xsl:text>
 		</result:newQueryPart>
diff --git a/FgsLucene/src/java/dk/defxws/fgslucene/OperationsImpl.java b/FgsLucene/src/java/dk/defxws/fgslucene/OperationsImpl.java
index b040bcc..4d40d8f 100644
--- a/FgsLucene/src/java/dk/defxws/fgslucene/OperationsImpl.java
+++ b/FgsLucene/src/java/dk/defxws/fgslucene/OperationsImpl.java
@@ -90,27 +90,44 @@ public String gfindObjects(
                         " fgsUserName="+fgsUserName+
                         " usingQuery="+usingQuery);
         }
-        ResultSet resultSet = (new Connection()).createStatement().executeQuery(
-        		usingQuery,
-                hitPageStart,
-                hitPageSize,
-                snippetsMax,
-                fieldMaxLength,
-                getQueryAnalyzer(usingIndexName),
-                config.getDefaultQueryFields(usingIndexName),
-                config.getAllowLeadingWildcard(usingIndexName),
-                config.getIndexDir(usingIndexName),
-                usingIndexName,
-                config.getSnippetBegin(usingIndexName),
-                config.getSnippetEnd(usingIndexName),
-                config.getSortFields(usingIndexName, sortFields));
+        ResultSet resultSet = null;
+		try {
+			resultSet = (new Connection()).createStatement().executeQuery(
+					usingQuery,
+			        hitPageStart,
+			        hitPageSize,
+			        snippetsMax,
+			        fieldMaxLength,
+			        getQueryAnalyzer(usingIndexName),
+			        config.getDefaultQueryFields(usingIndexName),
+			        config.getAllowLeadingWildcard(usingIndexName),
+			        config.getIndexDir(usingIndexName),
+			        usingIndexName,
+			        config.getSnippetBegin(usingIndexName),
+			        config.getSnippetEnd(usingIndexName),
+			        config.getSortFields(usingIndexName, sortFields));
+		} catch (Exception e) {
+            throw new GenericSearchException("gfindObjects executeQuery error:\n" + e.toString());
+		}
         params[12] = "RESULTPAGEXSLT";
         params[13] = resultPageXslt;
-        String xsltPath = config.getConfigName()+"/index/"+usingIndexName+"/"+config.getGfindObjectsResultXslt(usingIndexName, resultPageXslt);
-        StringBuffer resultXml = (new GTransformer()).transform(
-        		xsltPath,
-        		resultSet.getResultXml(),
-                params);
+        String xsltPath = null;
+		try {
+			xsltPath = config.getConfigName()+"/index/"+usingIndexName+"/"+config.getGfindObjectsResultXslt(usingIndexName, resultPageXslt);
+		} catch (Exception e) {
+            throw new GenericSearchException("gfindObjects xsltPath error:\n" + e.toString());
+		}
+        if (logger.isDebugEnabled())
+            logger.debug("gfindObjects xsltPath=\n"+xsltPath+" resultSet="+resultSet);
+        StringBuffer resultXml = null;
+		try {
+			resultXml = (new GTransformer()).transform(
+					xsltPath,
+					resultSet.getResultXml(),
+			        params);
+		} catch (Exception e) {
+            throw new GenericSearchException("gfindObjects transform error:\n" + e.toString());
+		}
         if (srf != null && config.isSearchResultFilteringActive("postsearch")) {
         	resultXml = srf.filterResultsetForPostsearch(fgsUserName, resultXml, fgsUserAttributes, config);
             if (logger.isDebugEnabled())
@@ -675,16 +692,6 @@ private void getIndexWriter(String indexName)
     	    try {
                 iw = new IndexWriter(dir, iwconfig);
             } catch (Exception e) {
-//             	iw = null;
-//                if (e.toString().indexOf("/segments")>-1) {
-//                    try {
-//        				Directory dir = new SimpleFSDirectory(new File(config.getIndexDir(indexName)));
-//                        iw = new IndexWriter(dir, getAnalyzer(indexName), true, IndexWriter.MaxFieldLength.LIMITED);
-//                    } catch (IOException e2) {
-//                        throw new GenericSearchException("IndexWriter new error, creating index indexName=" + indexName+ " :\n", e2);
-//                    }
-//                }
-//                else
                     throw new GenericSearchException("IndexWriter new error indexName=" + indexName+ " :\n", e);
             }
     	}
diff --git a/FgsLucene/src/java/dk/defxws/fgslucene/Statement.java b/FgsLucene/src/java/dk/defxws/fgslucene/Statement.java
index 81c5631..6fd3027 100644
--- a/FgsLucene/src/java/dk/defxws/fgslucene/Statement.java
+++ b/FgsLucene/src/java/dk/defxws/fgslucene/Statement.java
@@ -145,6 +145,8 @@ public ResultSet executeQuery(
     	TopDocs hits = getHits(query, start+maxResults-1, sortFields);
     	ScoreDoc[] docs = hits.scoreDocs;
     	int end = Math.min(hits.totalHits, start + maxResults - 1);
+        if (logger.isDebugEnabled())
+        	logger.debug("executeQuery hits.totalHits="+hits.totalHits);
     	StringBuffer resultXml = new StringBuffer();
     	resultXml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
     	String queryStringEncoded = null;
@@ -214,19 +216,34 @@ public ResultSet executeQuery(
     		resultXml.append("</hit>");
     	}
     	resultXml.append("</lucenesearch>");
+        if (logger.isDebugEnabled()) {
+        	int size = 500;
+        	if (resultXml.length()<size)
+        		size = resultXml.length();
+        	String debugString = resultXml.substring(0, size);
+        	if (resultXml.length()>size)
+        		debugString += "...";
+        	logger.debug("executeQuery resultXml="+debugString);
+        }
     	rs = new ResultSet(resultXml);
     	if (searcher!=null) {
     		try {
     			searcher.close();
+    			searcher = null;
     		} catch (IOException e) {
     		}
     	}
+        if (logger.isDebugEnabled()) 
+        	logger.debug("executeQuery searcher="+searcher);
 		if (ir!=null) {
     		try {
 				ir.close();
+				ir = null;
 			} catch (IOException e) {
 			}
 		}
+        if (logger.isDebugEnabled()) 
+        	logger.debug("executeQuery ir="+ir);
     	return rs;
     }
 
diff --git a/FgsSolr2/src/java/dk/defxws/fgssolr/OperationsImpl.java b/FgsSolr2/src/java/dk/defxws/fgssolr/OperationsImpl.java
index 90657b2..65621ab 100644
--- a/FgsSolr2/src/java/dk/defxws/fgssolr/OperationsImpl.java
+++ b/FgsSolr2/src/java/dk/defxws/fgssolr/OperationsImpl.java
@@ -46,6 +46,7 @@
 
 import dk.defxws.fedoragsearch.server.GTransformer;
 import dk.defxws.fedoragsearch.server.GenericOperationsImpl;
+import dk.defxws.fedoragsearch.server.errors.ConfigException;
 import dk.defxws.fedoragsearch.server.errors.GenericSearchException;
 
 import org.fcrepo.server.utilities.StreamUtility;
@@ -104,8 +105,6 @@ public String gfindObjects(
                 config.getSnippetBegin(usingIndexName),
                 config.getSnippetEnd(usingIndexName),
                 config.getSortFields(usingIndexName, sortFields));
-        params[12] = "RESULTPAGEXSLT";
-        params[13] = resultPageXslt;
         String xsltPath = config.getConfigName()+"/index/"+usingIndexName+"/"+config.getGfindObjectsResultXslt(usingIndexName, resultPageXslt);
         StringBuffer resultXml = (new GTransformer()).transform(
         		xsltPath,
@@ -459,39 +458,85 @@ private void indexDoc(
     
     public Analyzer getAnalyzer(String indexName)
     throws GenericSearchException {
-    	String analyzerClassName= config.getAnalyzer(indexName);
+    	return getAnalyzer(indexName, "");
+    }
+    
+    public Analyzer getAnalyzer(String indexName, String className)
+    throws GenericSearchException {
+    	String analyzerClassName = className;
+    	if (analyzerClassName.length()==0) {
+        	analyzerClassName= config.getAnalyzer(indexName);
+    	}
 		String stopwordsLocation = config.getStopwordsLocation(indexName); 
         if (logger.isDebugEnabled())
-            logger.debug("analyzerClassName=" + analyzerClassName+ " stopwordsLocation="+stopwordsLocation);
+            logger.debug("getAnalyzer analyzerClassName=" + analyzerClassName+ " stopwordsLocation="+stopwordsLocation);
         Analyzer analyzer = null;
-		try {
-			Version version = Version.LUCENE_35;
-			Class analyzerClass = Class.forName(analyzerClassName);
-            if (logger.isDebugEnabled())
-                logger.debug("analyzerClass=" + analyzerClass.toString());
-			if (stopwordsLocation == null || stopwordsLocation.equals("")) {
-				analyzer = (Analyzer) analyzerClass.getConstructor(new Class[] { Version.class})
-				.newInstance(new Object[] { version });
-			} else {
-				analyzer = (Analyzer) analyzerClass.getConstructor(new Class[] { Version.class, File.class})
-				.newInstance(new Object[] { version, new File(stopwordsLocation) });
-			}
-            if (logger.isDebugEnabled())
-                logger.debug("analyzer=" + analyzer.toString());
-        } catch (Exception e) {
-            throw new GenericSearchException(analyzerClassName
-                    + ": instantiation error.\n", e);
+        if ("org.apache.lucene.analysis.KeywordAnalyzer".equals(analyzerClassName)) {
+        	analyzer = new KeywordAnalyzer();
+        } else {
+    		try {
+    			Version version = Version.LUCENE_35;
+    			Class analyzerClass = Class.forName(analyzerClassName);
+                if (logger.isDebugEnabled())
+                    logger.debug("getAnalyzer analyzerClass=" + analyzerClass.toString());
+    			if (stopwordsLocation == null || stopwordsLocation.equals("")) {
+    				analyzer = (Analyzer) analyzerClass.getConstructor(new Class[] { Version.class})
+    				.newInstance(new Object[] { version });
+    			} else {
+    				analyzer = (Analyzer) analyzerClass.getConstructor(new Class[] { Version.class, File.class})
+    				.newInstance(new Object[] { version, new File(stopwordsLocation) });
+    			}
+            } catch (Exception e) {
+                throw new GenericSearchException(analyzerClassName
+                        + ": instantiation error.\n", e);
+            }
         }
+        if (logger.isDebugEnabled())
+            logger.debug("getAnalyzer analyzer=" + analyzer.toString());
         return analyzer;
     }
     
     public Analyzer getQueryAnalyzer(String indexName)
     throws GenericSearchException {
+        if (logger.isDebugEnabled())
+            logger.debug("getQueryAnalyzer indexName=" + indexName);
         Analyzer analyzer = getAnalyzer(indexName);
         Map<String,Analyzer> fieldAnalyzers = new HashMap<String, Analyzer>();
+        String configFieldAnalyzers = "";
+        try {
+			configFieldAnalyzers = config.getFieldAnalyzers(indexName);
+		} catch (Exception e) {
+            throw new ConfigException("getQueryAnalyzer config.getFieldAnalyzers "+" :\n", e);
+		}
+        if (logger.isDebugEnabled())
+            logger.debug("getQueryAnalyzer configFieldAnalyzers=" + configFieldAnalyzers);
+        if (configFieldAnalyzers != null && configFieldAnalyzers.length()>0) {
+            StringTokenizer stConfigFieldAnalyzers = new StringTokenizer(configFieldAnalyzers);
+        	while (stConfigFieldAnalyzers.hasMoreElements()) {
+        		String fieldAnalyzer = stConfigFieldAnalyzers.nextToken();
+                if (logger.isDebugEnabled())
+                    logger.debug("getQueryAnalyzer fieldAnalyzer=" + fieldAnalyzer);
+        		int i = fieldAnalyzer.indexOf("::");
+        		if (i<0) {
+                    throw new ConfigException("getQueryAnalyzer fgsindex.fieldAnalyzer="+fieldAnalyzer+ " missing '::'");
+        		}
+    			String fieldName = "-";
+    			String analyzerClassName = "-";
+        		try {
+    				fieldName = fieldAnalyzer.substring(0, i);
+    				analyzerClassName = fieldAnalyzer.substring(i+2);
+    				fieldAnalyzers.put(fieldName, getAnalyzer(indexName, analyzerClassName));
+    			} catch (Exception e) {
+    	            throw new ConfigException("getQueryAnalyzer getAnalyzer fieldName="+fieldName+" analyzerClassName="+analyzerClassName+" :\n", e);
+    			}
+        	}
+        }
     	StringTokenizer untokenizedFields = new StringTokenizer(config.getUntokenizedFields(indexName));
     	while (untokenizedFields.hasMoreElements()) {
-    		fieldAnalyzers.put(untokenizedFields.nextToken(), new KeywordAnalyzer());
+    		String fieldName = untokenizedFields.nextToken();
+    		if (!fieldAnalyzers.containsKey(fieldName)) {
+        		fieldAnalyzers.put(fieldName, new KeywordAnalyzer());
+    		}
     	}
         PerFieldAnalyzerWrapper pfanalyzer = new PerFieldAnalyzerWrapper(analyzer, fieldAnalyzers);
         if (logger.isDebugEnabled())
diff --git a/README.txt b/README.txt
index 7e91ac9..97f63d5 100644
--- a/README.txt
+++ b/README.txt
@@ -1,5 +1,5 @@
 -------------------------------------------------------------------
-FedoraGenericSearch (GSearch) Version 2.3 
+FedoraGenericSearch (GSearch) Version 2.4 
 -------------------------------------------------------------------
 
  * License and Copyright: FedoraGenericSearch is subject to the same open source