Skip to content

Commit

Permalink
Merge pull request #27 from ga-gn/es-newfields
Browse files Browse the repository at this point in the history
Add new index fields in elastic search - authors, pid, ecatid, keywor…
  • Loading branch information
josephjohn136 committed Aug 26, 2020
2 parents c906d47 + b9b08f2 commit 06d521e
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 4 deletions.
52 changes: 48 additions & 4 deletions core/src/main/java/org/fao/geonet/kernel/ECatOperationManager.java
@@ -1,16 +1,23 @@
package org.fao.geonet.kernel;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.StringJoiner;
import java.util.stream.Collectors;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.domain.Metadata;
import org.fao.geonet.domain.MetadataIndexedField;
import org.fao.geonet.exceptions.BatchEditException;
import org.fao.geonet.kernel.search.IndexAndTaxonomy;
import org.fao.geonet.kernel.search.IndexFields;
import org.fao.geonet.kernel.search.LuceneConfig;
import org.fao.geonet.kernel.search.LuceneQueryBuilder;
import org.fao.geonet.kernel.search.LuceneQueryInput;
Expand Down Expand Up @@ -56,7 +63,6 @@ public Metadata getMetadataFromECatId(ApplicationContext context, String eCatId)
Document doc = getLuceneTopDocs(element, Geonet.IndexFieldNames.UUID);

String uuid = doc.get(Geonet.IndexFieldNames.UUID);
Log.debug(Geonet.SEARCH_ENGINE, "ECatOperationManager >> getMetadataByLuceneSearch --> uuid: " + uuid);

if (uuid != null) {
Metadata md = mdRepo.findOneByUuid(uuid);
Expand All @@ -70,6 +76,46 @@ public Metadata getMetadataFromECatId(ApplicationContext context, String eCatId)
return null;
}

/**
* Search metadata using lucene index
* @param context
* @param srvContext
* @param request
* @return
* @throws JDOMException
* @throws IOException
* @throws BatchEditException
*/
public MetadataIndexedField getMetadataIndexedFieldsFromECatId(String eCatId) throws Exception {

Element element = Xml
.loadString("<request><isAdmin>true</isAdmin><_isTemplate>n</_isTemplate><eCatId>"
+ eCatId + "</eCatId><fast>index</fast></request>", false);

try {

Document doc = getLuceneTopDocs(element, IndexFields.UUID, IndexFields.KEYWORD, IndexFields.PID, IndexFields.AUTHOR);

String uuid = doc.get(IndexFields.UUID).trim();
String pid = doc.get(IndexFields.PID).trim();
String authors = Arrays.asList(doc.getFields(IndexFields.AUTHOR)).stream().map(IndexableField::stringValue).collect(Collectors.joining(", "));
String keywords = Arrays.asList(doc.getFields(IndexFields.KEYWORD)).stream().map(IndexableField::stringValue).collect(Collectors.joining(", "));

MetadataIndexedField indexField = new MetadataIndexedField();
indexField.setUuid(uuid);
indexField.seteCatId(eCatId);
indexField.setAuthors(authors);
indexField.setKeywords(keywords);
indexField.setPid(pid);

return indexField;


} catch (Exception e) {
throw new Exception("failed to get MetadataIndexedFields using lucene search" + e.getMessage());
}
}

/**
* Search metadata using lucene index
* @param context
Expand All @@ -92,8 +138,6 @@ public String getECatIdFromUUID(ApplicationContext context, String uuid) throws
Document doc = getLuceneTopDocs(element, Geonet.IndexFieldNames.ECAT_ID);
String eCatId = doc.get(Geonet.IndexFieldNames.ECAT_ID);

Log.debug(Geonet.SEARCH_ENGINE, "ECatOperationManager >> getECatIdFromUUID --> eCatId: " + eCatId);

return eCatId;

} catch (Exception e) {
Expand All @@ -103,7 +147,7 @@ public String getECatIdFromUUID(ApplicationContext context, String uuid) throws

}

private Document getLuceneTopDocs(Element element, String fieldVisitor) throws IOException {
private Document getLuceneTopDocs(Element element, String... fieldVisitor) throws IOException {
IndexAndTaxonomy indexAndTaxonomy = null;
try {
indexAndTaxonomy = this.tracker.acquire("eng", -1);
Expand Down
Expand Up @@ -77,6 +77,8 @@ public class IndexFields {
public static final String INSPIRE_THEME = "inspiretheme";
public static final String IS_TEMPLATE = "_isTemplate";
public static final String KEYWORD = "keyword";
public static final String AUTHOR = "author";
public static final String PID = "PID";
public static final String METADATA_STANDARD_NAME = "metadataStandardName";
public static final String NORTH = "northBL";
public static final String OPERATESON = "operatesOn";
Expand Down
Expand Up @@ -28,7 +28,9 @@
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.domain.MetadataIndexedField;
import org.fao.geonet.es.EsClient;
import org.fao.geonet.kernel.ECatOperationManager;
import org.fao.geonet.utils.Log;
import org.springframework.beans.factory.NoSuchBeanDefinitionException;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -41,13 +43,20 @@
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* A bean representing a query request. Performs the mapping with the database.
*
* @author nicolas
*/
public class QueryRequest {

ECatOperationManager eCatManager;

Pattern pattern = Pattern.compile("\\+eCatId:(.*?)\\s");

/**
* the date format when inserting date into database
*/
Expand Down Expand Up @@ -360,6 +369,20 @@ public boolean storeToEs(String index, String indexType) {
doc.put("hits", this.hits);
doc.put("language", this.language);
doc.put("query", this.luceneQuery);
Matcher matcher = pattern.matcher(this.luceneQuery);
if (matcher.find()) {
String eCatId = matcher.group(1);
if(eCatManager == null)
eCatManager = ApplicationContextHolder.get().getBean(ECatOperationManager.class);

MetadataIndexedField field = eCatManager.getMetadataIndexedFieldsFromECatId(eCatId);
doc.put("eCatId", field.geteCatId());
doc.put("uuid", field.getUuid());
doc.put("pid", field.getPid());
doc.put("author", field.getAuthors());
doc.put("keyword", field.getKeywords());
}

doc.put("recordType", this.mdType.name());
doc.put("sortBy", this.sortBy);
doc.put("spatialFilter", this.spatialFilter);
Expand Down
@@ -0,0 +1,62 @@
package org.fao.geonet.domain;

public class MetadataIndexedField {

private String eCatId;
private String uuid;
private String keywords;
private String pid;
private String authors;

public String geteCatId() {
return eCatId;
}

public void seteCatId(String eCatId) {
this.eCatId = eCatId;
}

public String getUuid() {
return uuid;
}

public void setUuid(String uuid) {
this.uuid = uuid;
}

public String getPid() {
return pid;
}

public void setPid(String pid) {
this.pid = pid;
}

public String getKeywords() {
return keywords;
}

public void setKeywords(String keywords) {
this.keywords = keywords;
}

public String getAuthors() {
return authors;
}

public void setAuthors(String authors) {
this.authors = authors;
}

@Override
public String toString() {

StringBuilder sb = new StringBuilder();
sb.append("eCatId: " + this.eCatId + ", ");
sb.append("uuid: " + this.uuid + ", ");
sb.append("pid: " + this.pid + ", ");
sb.append("keywords: " + this.keywords + ", ");
sb.append("author: " + this.authors + ", ");
return sb.toString();
}
}

0 comments on commit 06d521e

Please sign in to comment.