Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/3.0.x' into 3.2.x
Browse files Browse the repository at this point in the history
  • Loading branch information
Jose Garcia authored and Jose Garcia committed Jun 9, 2017
2 parents 372921f + ea774a1 commit 1ebeb74
Show file tree
Hide file tree
Showing 12 changed files with 230 additions and 43 deletions.
41 changes: 40 additions & 1 deletion harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/HarvesterJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
package org.fao.geonet.kernel.harvest.harvester;

import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.Logger;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.utils.Log;
import org.quartz.*;
import org.springframework.context.ConfigurableApplicationContext;

Expand All @@ -44,10 +47,13 @@ public class HarvesterJob implements Job, InterruptableJob {
public static final String ID_FIELD = "harvesterId";
String harvesterId;
AbstractHarvester<?> harvester;
private Thread _this = null;
protected Logger log = Log.createLogger(Geonet.HARVESTER);

@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
try {
_this = Thread.currentThread();
harvester.harvest();
} catch (Throwable t) {
throw new JobExecutionException(t, false);
Expand All @@ -70,9 +76,42 @@ public void setHarvester(AbstractHarvester<?> harvester) {
this.harvester = harvester;
}


@Override
public void interrupt() throws UnableToInterruptJobException {
harvester.cancelMonitor.set(true);

// Following the suggestion of InterruptableJob
// Sometimes the harvester is frozen
// give some time, but if it does not finish properly...
// just kill it!!
new Thread(){
@Override
public void run() {
super.run();

//Wait for proper shutdown (a minute, more than enough!)
try {
Thread.sleep(60 * 1000);
} catch (InterruptedException e) {
log.error(e);
}

//Still running?
if(_this.isAlive()) {
//Then kill it!
log.error("Forcefully stopping harvester thread '" +
getHarvesterId() + "'.");
try {
_this.interrupt();
} catch (Throwable e) {
log.error(e);
}
}
}
}.start();
}

public Thread getThread() {
return _this;
}
}
10 changes: 10 additions & 0 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ private Set<RecordInfo> search(CswServer server, Search s) throws Exception {
Set<RecordInfo> records = new HashSet<RecordInfo>();

while (true) {
if(this.cancelMonitor.get()) {
log.error("Harvester stopped in the middle of running!");
//Returning whatever, we have to move on and finish!
return records;
}
request.setStartPosition(start);
Element response = doSearch(request, start, GETRECORDS_REQUEST_MAXRECORDS);
if (log.isDebugEnabled()) {
Expand All @@ -296,6 +301,11 @@ private Set<RecordInfo> search(CswServer server, Search s) throws Exception {
}
}

if(this.cancelMonitor.get()) {
log.error("Harvester stopped in the middle of running!");
//Returning whatever, we have to move on and finish!
return records;
}
@SuppressWarnings("unchecked")
List<Element> list = results.getChildren();
int foundCnt = 0;
Expand Down
44 changes: 29 additions & 15 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import jeeves.server.context.ServiceContext;

import org.apache.commons.lang.StringUtils;
import org.eclipse.emf.common.command.AbortExecutionException;
import org.fao.geonet.GeonetContext;
import org.fao.geonet.Logger;
Expand All @@ -34,16 +35,12 @@
import org.fao.geonet.domain.Metadata;
import org.fao.geonet.domain.MetadataType;
import org.fao.geonet.domain.OperationAllowedId_;
import org.fao.geonet.domain.Pair;
import org.fao.geonet.exceptions.OperationAbortedEx;
import org.fao.geonet.kernel.DataManager;
import org.fao.geonet.kernel.UpdateDatestamp;
import org.fao.geonet.kernel.harvest.BaseAligner;
import org.fao.geonet.kernel.harvest.harvester.CategoryMapper;
import org.fao.geonet.kernel.harvest.harvester.GroupMapper;
import org.fao.geonet.kernel.harvest.harvester.HarvestError;
import org.fao.geonet.kernel.harvest.harvester.HarvestResult;
import org.fao.geonet.kernel.harvest.harvester.IHarvester;
import org.fao.geonet.kernel.harvest.harvester.UUIDMapper;
import org.fao.geonet.kernel.harvest.harvester.*;
import org.fao.geonet.lib.Lib;
import org.fao.geonet.repository.MetadataRepository;
import org.fao.geonet.repository.OperationAllowedRepository;
Expand All @@ -63,11 +60,7 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;

//=============================================================================
Expand Down Expand Up @@ -285,6 +278,11 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
localGroups = new GroupMapper(context);
localUuids = new UUIDMapper(context.getBean(MetadataRepository.class), params.getUuid());

Pair<String, Map<String, Object>> filter =
HarvesterUtil.parseXSLFilter(params.xslfilter, log);
String processName = filter.one();
Map<String, Object> processParams = filter.two();

dataMan.flush();

//-----------------------------------------------------------------------
Expand Down Expand Up @@ -322,9 +320,9 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
String id = localUuids.getID(ri.id);

if (id == null) {
addMetadata(t, ri);
addMetadata(t, ri, processName, processParams);
} else {
updateMetadata(t, ri, id);
updateMetadata(t, ri, id, processName, processParams);
}
}

Expand All @@ -345,7 +343,7 @@ private boolean exists(Set<RecordInfo> records, String uuid) {
return false;
}

private void addMetadata(XmlRequest t, RecordInfo ri) throws Exception {
private void addMetadata(XmlRequest t, RecordInfo ri, String processName, Map<String, Object> processParams) throws Exception {
Element md = retrieveMetadata(t, ri);

if (md == null)
Expand All @@ -357,6 +355,13 @@ private void addMetadata(XmlRequest t, RecordInfo ri) throws Exception {

if (log.isDebugEnabled()) log.debug(" - Adding metadata with remote id : " + ri.id);


// Apply the xsl filter choosed by UI
if (StringUtils.isNotEmpty(params.xslfilter)) {
md = HarvesterUtil.processMetadata(dataMan.getSchema(schema),
md, processName, processParams, log);
}

//
// insert metadata
//
Expand Down Expand Up @@ -471,7 +476,7 @@ private Element toDublinCore(Element md) {
}
}

private void updateMetadata(XmlRequest t, RecordInfo ri, String id) throws Exception {
private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String processName, Map<String, Object> processParams) throws Exception {
String date = localUuids.getChangeDate(ri.id);

if (!ri.isMoreRecentThan(date)) {
Expand All @@ -487,6 +492,15 @@ private void updateMetadata(XmlRequest t, RecordInfo ri, String id) throws Excep
if (md == null)
return;

// The schema of the metadata
String schema = dataMan.autodetectSchema(md, null);

// Apply the xsl filter choosed by UI
if (StringUtils.isNotEmpty(params.xslfilter)) {
md = HarvesterUtil.processMetadata(dataMan.getSchema(schema),
md, processName, processParams, log);
}

//
// update metadata
//
Expand Down
1 change: 1 addition & 0 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/OaiPmhHarvester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ protected void storeNodeExtra(AbstractParams p, String path,

settingMan.add("id:" + siteId, "url", params.url);
settingMan.add("id:" + siteId, "icon", params.icon);
settingMan.add("id:" + siteId, "xslfilter", params.xslfilter);

settingMan.add("id:" + optionsId, "validate", params.getValidate());

Expand Down
13 changes: 13 additions & 0 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/OaiPmhParams.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ public class OaiPmhParams extends AbstractParams {

public String url;

/**
* The filter is a process (see schema/process folder) which depends on the schema.
* It could be composed of parameter which will be sent to XSL transformation using
* the following syntax :
* <pre>
* anonymizer?protocol=MYLOCALNETWORK:FILEPATH&email=gis@organisation.org&thesaurus=MYORGONLYTHEASURUS
* </pre>
*/
public String xslfilter;

//---------------------------------------------------------------------------
//---
//--- Create : called when a new entry must be added. Reads values from the
Expand Down Expand Up @@ -77,6 +87,7 @@ public void create(Element node) throws BadInputEx {

url = Util.getParam(site, "url", "");
icon = Util.getParam(site, "icon", "");
xslfilter = Util.getParam(site, "xslfilter", "");

addSearches(searches);
}
Expand All @@ -91,6 +102,7 @@ public void update(Element node) throws BadInputEx {

url = Util.getParam(site, "url", url);
icon = Util.getParam(site, "icon", icon);
xslfilter = Util.getParam(site, "xslfilter", "");

//--- if some search queries are given, we drop the previous ones and
//--- set these new ones
Expand Down Expand Up @@ -125,6 +137,7 @@ public OaiPmhParams copy() {

copy.url = url;
copy.icon = icon;
copy.xslfilter = xslfilter;

copy.setValidate(getValidate());

Expand Down
31 changes: 19 additions & 12 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Search.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@

package org.fao.geonet.kernel.harvest.harvester.oaipmh;

import java.util.Date;

import org.apache.commons.lang.StringUtils;
import org.fao.geonet.Util;
import org.fao.geonet.domain.ISODate;
import org.fao.geonet.exceptions.BadInputEx;
Expand Down Expand Up @@ -73,26 +76,30 @@ public Search(Element search) throws BadInputEx {
ISODate fromDate = null;
ISODate untilDate = null;

try {
if (!from.equals("")) {
fromDate = new ISODate(from);
from = fromDate.getDateAsString();
}
try {
if (StringUtils.isNotEmpty(from) && !from.equalsIgnoreCase("Invalid Date")) {
fromDate = new ISODate(from);
from = fromDate.getDateAsString();
} else {
from = "";
}

} catch (Exception e) {
throw new BadParameterEx("from", from);
}

//--- check until parameter

try {
if (!until.equals("")) {
untilDate = new ISODate(until);
until = untilDate.getDateAsString();
try {
if (StringUtils.isNotEmpty(until) && !until.equalsIgnoreCase("Invalid Date")) {
untilDate = new ISODate(until);
until = untilDate.getDateAsString();
} else {
until = "";
}
} catch (Exception e) {
throw new BadParameterEx("until", until);
}
} catch(Exception e) {
throw new BadParameterEx("until", until);
}

//--- check from <= until

Expand Down
45 changes: 45 additions & 0 deletions schemas/dublin-core/src/main/plugin/dublin-core/index-fields/default.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,51 @@

<Field name="keyword" string="{$place}" store="true" index="true"/>
</xsl:when>

<!-- Latitude 4.817 to 88.225; Longitude -179.950 to 180.000 -->
<xsl:when test="starts-with(., 'Latitude')">
<xsl:variable name="s" select="substring-after($coverage,'Latitude')"/>
<xsl:variable name="south" select="normalize-space(substring-before($s, 'to'))"/>
<xsl:variable name="n" select="substring-after($s,'to')"/>
<xsl:variable name="north" select="normalize-space(substring-before($n, ';'))"/>
<xsl:variable name="w" select="substring-after($n,'Longitude')"/>
<xsl:variable name="west" select="normalize-space(substring-before($w, 'to'))"/>
<xsl:variable name="e" select="substring-after($w,'to')"/>
<xsl:variable name="east" select="normalize-space($e)"/>

<Field name="westBL" string="{$west}" store="false" index="true"/>
<Field name="eastBL" string="{$east}" store="false" index="true"/>
<Field name="southBL" string="{$south}" store="false" index="true"/>
<Field name="northBL" string="{$north}" store="false" index="true"/>
<Field name="geoBox" string="{concat($west, '|',
$south, '|',
$east, '|',
$north
)}" store="true" index="false"/>
</xsl:when>

<!-- 4.817,-179.95 / 88.225,180 (min, max Latitude / min, max Longitude) -->
<xsl:when test="ends-with(., '(min, max Latitude / min, max Longitude)')">
<xsl:variable name="s" select="$coverage"/>
<xsl:variable name="south" select="normalize-space(substring-before($s, ','))"/>
<xsl:variable name="w" select="substring-after($s,',')"/>
<xsl:variable name="west" select="normalize-space(substring-before($w, '/'))"/>
<xsl:variable name="n" select="substring-after($w,'/')"/>
<xsl:variable name="north" select="normalize-space(substring-before($n, ','))"/>
<xsl:variable name="e" select="substring-after($n,',')"/>
<xsl:variable name="east" select="normalize-space(substring-before($e, '(min, max Latitude / min, max Longitude)'))"/>

<Field name="westBL" string="{$west}" store="false" index="true"/>
<Field name="eastBL" string="{$east}" store="false" index="true"/>
<Field name="southBL" string="{$south}" store="false" index="true"/>
<Field name="northBL" string="{$north}" store="false" index="true"/>
<Field name="geoBox" string="{concat($west, '|',
$south, '|',
$east, '|',
$north
)}" store="true" index="false"/>
</xsl:when>

<xsl:otherwise>
<Field name="keyword" string="{.}" store="true" index="true"/>
</xsl:otherwise>
Expand Down
22 changes: 18 additions & 4 deletions web-ui/src/main/resources/catalog/components/utility/UtilityDirective.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,11 +1055,25 @@
};
}]);
module.filter('newlines', function() {
return function(text) {
if (text) {
return text.replace(/(\r)?\n/g, '<br/>');
return function(value) {
if(angular.isArray(value)) {
var finalText = '';
angular.forEach(value, function(value, key) {
if(value) {
finalText += '<p>' + value + '</p>';
}
});

return finalText;

} else if(angular.isString(value)) {
if (value) {
return value.replace(/(\r)?\n/g, '<br/>');
} else {
return value;
}
} else {
return text;
return value;
}
}
});
Expand Down
Loading

0 comments on commit 1ebeb74

Please sign in to comment.