From e0d2b0ff00f3f217b3000580a3ff4aa9a46456e9 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Thu, 7 Sep 2023 13:23:57 -0700 Subject: [PATCH 01/22] Update AgpOutputBroker.java check-in for issue #203 --- .../harvester/agp/AgpOutputBroker.java | 113 ++++++++++++++---- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 1ff49acb..23c87605 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -187,7 +187,20 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { //Node titleNode = markdownParser.parse(title); //title = htmlRenderer.render(titleNode); Node descriptionNode = markdownParser.parse(description); + description = htmlRenderer.render(descriptionNode); + description = description.replaceAll("[*] ","
  • "); + description = description.replaceAll("\\\\n","
    "); + description = replaceSpecialCharacters(description); + //description = description.replaceAll("[^\\x00-\\x7F]", ""); + + // also update arcgisMetadata with the new description as idAbs + // only if there is exactly 1 idAbs element + String[] metadataPieces = arcgisMetadata.split("idAbs>"); + if (metadataPieces.length == 3) { + metadataPieces[1] = "idAbs>"; + arcgisMetadata = metadataPieces[0] + metadataPieces[1] + metadataPieces[2]; + } } String sThumbnailUrl = StringUtils.trimToNull(getAttributeValue(attributes, WKAConstants.WKA_THUMBNAIL_URL, null)); String resourceUrl = getAttributeValue(attributes, WKAConstants.WKA_RESOURCE_URL, null); @@ -373,33 +386,46 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } // update item if does exist - ItemResponse response = updateItem( - itemEntry.id, - itemEntry.owner, - itemEntry.ownerFolder, - title, - description, - new URL(resourceUrl), - sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, - itemType, - extractEnvelope(bbox), - typeKeywords, - fileToUpload, - metadataFile, - token - ); + + // if the item url is the same as the resourceUrl, proceed. + // otherwise the metadata is for a sublayer, but the item is the parent + if (resourceUrl.equals(itemEntry.url)) { + ItemResponse response = updateItem( + itemEntry.id, + itemEntry.owner, + itemEntry.ownerFolder, + title, + description, + new URL(resourceUrl), + sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, + itemType, + extractEnvelope(bbox), + typeKeywords, + fileToUpload, + metadataFile, + token + ); + + if (response == null || !response.success) { + String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; + throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); + } else { + // String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); + System.out.print("updateItem -> " + response.toString()); + } - if (response == null || !response.success) { - String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; - throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); + existing.remove(itemEntry.id); + + return PublishingStatus.UPDATED; } else { - // String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); - System.out.print("updateItem -> " + response.toString()); + // the metadata is apparently for a sublayer + + // DO SOMETHING ELSE + String metadataUpdateURI = resourceUrl + "/metadata/update"; + System.out.println("update metadata at " + metadataUpdateURI); + + return PublishingStatus.SKIPPED; } - - existing.remove(itemEntry.id); - - return PublishingStatus.UPDATED; } } catch (MalformedURLException ex) { return PublishingStatus.SKIPPED; @@ -450,6 +476,27 @@ private ItemEntry searchForItem(String src_uri_s) throws URISyntaxException, IOE QueryResponse search = client.search(String.format("url:%s", String.format("%s", src_uri_s)), 0, 0, token); ItemEntry itemEntry = search != null && search.results != null && search.results.length > 0 ? search.results[0] : null; + // if no results found and the source uri (src_uri_s) ends in /0, /1, ... this may be a sublayer + // look for the parent service + if (itemEntry == null) { + String[] uri_parts = src_uri_s.split("/"); + + // iff the last part is a number, this could be a sublayer + if (StringUtils.isNumeric(uri_parts[uri_parts.length-1])) { + String regex = "\\/" + uri_parts[uri_parts.length-1] + "$"; + String mother_uri = ""; + try { + mother_uri = src_uri_s.replaceFirst(regex, ""); + } catch (Exception e) { + System.out.println(e.toString()); + } + + // now search for mother uri + search = client.search(String.format("url:%s", String.format("%s", mother_uri)), 0, 0, token); + itemEntry = search != null && search.results != null && search.results.length > 0 ? search.results[0] : null; + } + } + return itemEntry; } @@ -819,6 +866,24 @@ private FileName getFileNameFromUrl(String resourceUrl) { return new FileName(name, ext); } + private String replaceSpecialCharacters(String inputText) { + String outputText = ""; + String sourceText = inputText; + + HashMap extendedCharacters = new HashMap<>(); + + extendedCharacters.put("\\x91", "‘"); + extendedCharacters.put("\\x92", "’"); + extendedCharacters.put("\\x93", "“"); + extendedCharacters.put("\\x94", "”"); + + for (HashMap.Entry val: extendedCharacters.entrySet()) { + outputText = sourceText.replaceAll(val.getKey(), val.getValue()); + sourceText = outputText; + } + return outputText; + } + private static class FileName { public final String name; public final String ext; From 57ab8c8232bd84f7ce90fbea51f206ff80a38ec3 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Wed, 13 Sep 2023 09:00:39 -0700 Subject: [PATCH 02/22] harvest full metadata from ArcGIS Server services and layers added code to get the full metadata from ArcGIS Server services and layers as opposed to relying on the item description. --- .../commons/ags/client/AgsClient.java | 25 +++++ .../commons/ags/client/ItemInfo.java | 2 + .../commons/ags/client/LayerInfo.java | 2 + .../commons/ags/client/ServerResponse.java | 4 +- .../commons/meta/util/WKAConstants.java | 4 +- .../geoportal/harvester/ags/AgsBroker.java | 97 ++++++++++++++++++- 6 files changed, 129 insertions(+), 5 deletions(-) diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java index 2c4d0a94..db59418a 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/AgsClient.java @@ -177,6 +177,19 @@ public ServerResponse readServiceInformation(URL url) throws IOException { response.url = url.toExternalForm(); response.json = responseContent; response.itemInfo = readItemInfo(new URL(url + "/info/itemInfo")); + + response.hasMetadata = false; + response.metadataXML = ""; + String metadataURL = url + "/info/metadata"; + HttpGet getXML = new HttpGet(metadataURL); + try (CloseableHttpResponse httpResponseXML = httpClient.execute(getXML); InputStream contentStreamXML = httpResponseXML.getEntity().getContent();) { + if (httpResponseXML.getStatusLine().getStatusCode()<400) { + String responseContentXML = IOUtils.toString(contentStreamXML, "UTF-8"); + response.metadataXML = responseContentXML; + response.hasMetadata = true; + } + } + return response; } } @@ -201,6 +214,7 @@ public ItemInfo readItemInfo(URL url) throws IOException { mapper.configure(Feature.ALLOW_NON_NUMERIC_NUMBERS, true); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); ItemInfo response = mapper.readValue(responseContent, ItemInfo.class); + return response; } } @@ -228,6 +242,17 @@ public LayerInfo readLayerInformation(String folder, ServiceInfo si, LayerRef lR mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); LayerInfo response = mapper.readValue(responseContent, LayerInfo.class); + if (response.hasMetadata) { + HttpGet getXML = new HttpGet(url + String.format("/metadata", "text/xml")); + try (CloseableHttpResponse httpResponseXML = httpClient.execute(getXML); InputStream contentStreamXML = httpResponseXML.getEntity().getContent();) { + if (httpResponseXML.getStatusLine().getStatusCode()>=400) { + throw new HttpResponseException(httpResponseXML.getStatusLine().getStatusCode(), httpResponseXML.getStatusLine().getReasonPhrase()); + } + String responseContentXML = IOUtils.toString(contentStreamXML, "UTF-8"); + response.metadataXML = responseContentXML; + } + + } response.url = url; response.json = responseContent; return response; diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java index c24006f4..56d1a3fa 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ItemInfo.java @@ -36,4 +36,6 @@ public final class ItemInfo { public String spatialReference; public String accessInformation; public String licenseInfo; + public boolean hasMetadata; + public String metadataXML; } diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java index e92ba1f2..a77eece4 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/LayerInfo.java @@ -26,4 +26,6 @@ public class LayerInfo { public String description; public ExtentInfo extent; public String json; + public boolean hasMetadata; + public String metadataXML; } diff --git a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java index ecbac372..4158abbd 100644 --- a/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java +++ b/geoportal-commons/geoportal-commons-ags-client/src/main/java/com/esri/geoportal/commons/ags/client/ServerResponse.java @@ -23,6 +23,8 @@ public final class ServerResponse { public String url; public String json; + public boolean hasMetadata; + public String metadataXML; public String mapName; public String serviceDescription; @@ -36,6 +38,6 @@ public final class ServerResponse { @Override public String toString() { - return String.format("{ \"mapName\": \"%s\", \"serviceDescription\": \"%s\", \"spatialReference\": %s, \"initialExtent\": %s, \"fullExtent\": %s}", mapName, serviceDescription, spatialReference, initialExtent, fullExtent); + return String.format("{ \"mapName\": \"%s\", \"serviceDescription\": \"%s\", \"spatialReference\": %s, \"initialExtent\": %s, \"fullExtent\": %s, \"metadata\": %s}", mapName, serviceDescription, spatialReference, initialExtent, fullExtent, metadataXML); } } diff --git a/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java b/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java index e76a9875..5bbafd2f 100644 --- a/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java +++ b/geoportal-commons/geoportal-commons-meta/src/main/java/com/esri/geoportal/commons/meta/util/WKAConstants.java @@ -33,10 +33,12 @@ public final class WKAConstants { public static final String WKA_BBOX = "bbox"; public static final String WKA_MODIFIED = "modified"; public static final String WKA_REFERENCES = "references"; + public static final String WKA_METADATA_XML = "metadataXML"; private static final Set all = new HashSet(Arrays.asList(new String[]{ WKA_IDENTIFIER, WKA_TITLE, WKA_DESCRIPTION, WKA_RESOURCE_URL, - WKA_RESOURCE_URL_SCHEME, WKA_BBOX, WKA_THUMBNAIL_URL, WKA_MODIFIED, WKA_REFERENCES + WKA_RESOURCE_URL_SCHEME, WKA_BBOX, WKA_THUMBNAIL_URL, WKA_MODIFIED, WKA_REFERENCES, + WKA_METADATA_XML })); /** diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java index 4a630bb9..7e0b892a 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java @@ -50,14 +50,22 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathFactory; import org.apache.commons.lang3.StringUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import com.esri.geoportal.commons.utils.XmlUtils; import com.esri.geoportal.geoportal.commons.geometry.GeometryService; import com.esri.geoportal.harvester.api.DataContent; @@ -69,10 +77,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.StringReader; import java.net.URL; import java.util.Arrays; import java.util.stream.Collectors; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; import org.apache.http.impl.client.LaxRedirectStrategy; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * Ags broker. @@ -171,6 +184,7 @@ private ServerResponse layerInfoToServerResponse(LayerInfo layerInfo) { response.description = layerInfo.description; response.fullExtent = layerInfo.extent; response.initialExtent = layerInfo.extent; + response.metadataXML = layerInfo.metadataXML; return response; } @@ -253,6 +267,7 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx String itemInfoDescription = trimHtml(serverResponse.itemInfo!=null? serverResponse.itemInfo.description: null); String serverDescription = trimHtml(StringUtils.defaultString(StringUtils.defaultIfBlank(serverResponse.description, serverResponse.serviceDescription))); String description = StringUtils.defaultIfBlank(itemInfoDescription, serverDescription); + String metadataXML = serverResponse.metadataXML!=null? serverResponse.metadataXML: null; HashMap attributes = new HashMap<>(); attributes.put(WKAConstants.WKA_IDENTIFIER, new StringAttribute(serverResponse.url)); @@ -260,6 +275,7 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx attributes.put(WKAConstants.WKA_DESCRIPTION, new StringAttribute(description)); attributes.put(WKAConstants.WKA_RESOURCE_URL, new StringAttribute(serverResponse.url)); attributes.put(WKAConstants.WKA_RESOURCE_URL_SCHEME, new StringAttribute("urn:x-esri:specification:ServiceType:ArcGIS:" + (serviceType != null ? serviceType : "Unknown"))); + attributes.put(WKAConstants.WKA_METADATA_XML, new StringAttribute(metadataXML)); if (serverResponse.fullExtent != null) { normalizeExtent(serverResponse.fullExtent, 4326); @@ -269,10 +285,85 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx } } - MapAttribute attrs = new MapAttribute(attributes); - Document document = metaBuilder.create(attrs); - byte[] bytes = XmlUtils.toString(document).getBytes("UTF-8"); + Document document = null; + byte[] bytes = null; + + if (metadataXML != null && !metadataXML.trim().isEmpty()) { + bytes = metadataXML.getBytes("UTF-8"); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(false); + DocumentBuilder builder = null; + try { + builder = factory.newDocumentBuilder(); + document = builder.parse(new InputSource(new StringReader(metadataXML))); + + XPath xpath = XPathFactory.newInstance().newXPath(); + NodeList thumbnailNodes = (NodeList) xpath.compile("/metadata/Binary/Thumbnail/Data").evaluate(document, XPathConstants.NODESET); + if (thumbnailNodes.getLength() > 0) { + LOG.debug(thumbnailNodes.item(0).getTextContent()); + } + + NodeList serviceUrlNodes = (NodeList) xpath.compile("/metadata/distInfo/distributor/distorTran/onLineSrc/linkage").evaluate(document, XPathConstants.NODESET); + if (serviceUrlNodes.getLength() > 0) { + serviceUrlNodes.item(0).setNodeValue(serverResponse.url); + } else { + // no onLineSrc/linkage nodes + + // get /metadata + NodeList metadataNodes = (NodeList) xpath.compile("/metadata").evaluate(document, XPathConstants.NODESET); + + // get /metadata/distInfo + NodeList distinfoNodes = (NodeList) xpath.compile("/metadata/distinfo").evaluate(document, XPathConstants.NODESET); + if (distinfoNodes.getLength() == 0) { + Element distinfoElement = document.createElement("distinfo"); + metadataNodes.item(0).appendChild(distinfoElement); + } + + // get /metadata/distInfo/distributor + NodeList distributorNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor").evaluate(document, XPathConstants.NODESET); + if (distributorNodes.getLength() == 0) { + distinfoNodes = (NodeList) xpath.compile("/metadata/distinfo").evaluate(document, XPathConstants.NODESET); + Element distributorElement = document.createElement("distributor"); + distinfoNodes.item(0).appendChild(distributorElement); + } + + // get /metadata/distInfo/distributor/distorTran + NodeList distorTranNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran").evaluate(document, XPathConstants.NODESET); + if (distorTranNodes.getLength() == 0) { + distributorNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor").evaluate(document, XPathConstants.NODESET); + Element distorTranElement = document.createElement("distorTran"); + distributorNodes.item(0).appendChild(distorTranElement); + } + + // get /metadata/distInfo/distributor/distorTran/onLineSrc + NodeList onLineSrcNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc").evaluate(document, XPathConstants.NODESET); + if (onLineSrcNodes.getLength() == 0) { + distorTranNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran").evaluate(document, XPathConstants.NODESET); + Element onLineSrcElement = document.createElement("onLineSrc"); + distorTranNodes.item(0).appendChild(onLineSrcElement); + } + + // get /metadata/distInfo/distributor/distorTran/onLineSrc/linkage + NodeList linkageNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc/linkage").evaluate(document, XPathConstants.NODESET); + if (linkageNodes.getLength() == 0) { + onLineSrcNodes = (NodeList) xpath.compile("/metadata/distinfo/distributor/distorTran/onLineSrc").evaluate(document, XPathConstants.NODESET); + Element linkageElement = document.createElement("linkage"); + linkageElement.setTextContent(serverResponse.url); + onLineSrcNodes.item(0).appendChild(linkageElement); + } + } + + } catch (Exception ex) { + LOG.error(String.format("Error geting XML document. "), ex); + } + } else { + MapAttribute attrs = new MapAttribute(attributes); + document = metaBuilder.create(attrs); + } + bytes = XmlUtils.toString(document).getBytes("UTF-8"); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), getEntityDefinition().getLabel(), serverResponse.url, null, URI.create(serverResponse.url), td.getSource().getRef(), td.getRef()); attributes.entrySet().forEach(entry -> { ref.getAttributesMap().put(entry.getKey(), entry.getValue()); From 83ec07cd250dd9728b45f9f282890503b282b5fb Mon Sep 17 00:00:00 2001 From: cont_anki Date: Tue, 19 Sep 2023 16:25:21 +0530 Subject: [PATCH 03/22] #209 Portal Input broker- file name should be title in place of item id --- .../api/base/DataReferenceWrapper.java | 4 ++++ .../harvester/api/base/SimpleDataReference.java | 17 +++++++++++++++++ .../geoportal/harvester/api/DataReference.java | 2 ++ .../geoportal/commons/agp/client/ItemEntry.java | 2 +- .../harvester/agpsrc/AgpInputBroker.java | 4 ++-- .../harvester/folder/FolderBroker.java | 7 ++++--- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java index e5e0cd87..6a2af73b 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java +++ b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/DataReferenceWrapper.java @@ -75,6 +75,10 @@ public Set getContentType() { public String getId() { return baseRef.getId(); } + @Override + public String getTitle() { + return baseRef.getTitle(); + } @Override public String getFetchableId() { diff --git a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java index bde0a2cc..f1efa0b7 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java +++ b/geoportal-SDK/geoportal-harvester-api-base/src/main/java/com/esri/geoportal/harvester/api/base/SimpleDataReference.java @@ -34,6 +34,7 @@ public class SimpleDataReference implements DataReference { private final URI brokerUri; private final String brokerName; private final String id; + private String title = ""; private final Date lastModifiedDate; private final URI sourceUri; private final String inputBrokerRef; @@ -54,6 +55,16 @@ public class SimpleDataReference implements DataReference { * @param taskRef task reference of null if ad-hoc */ public SimpleDataReference(URI brokerUri, String brokerName, String id, Date lastModifiedDate, URI sourceUri, String inputBrokerRef, String taskRef) { + this.brokerUri = brokerUri; + this.brokerName = brokerName; + this.id = id; + this.lastModifiedDate = lastModifiedDate; + this.sourceUri = sourceUri; + this.inputBrokerRef = inputBrokerRef; + this.taskRef = taskRef; + } + + public SimpleDataReference(URI brokerUri, String brokerName, String id, Date lastModifiedDate, URI sourceUri, String inputBrokerRef, String taskRef,String title) { this.brokerUri = brokerUri; this.brokerName = brokerName; this.id = id; @@ -61,6 +72,7 @@ public SimpleDataReference(URI brokerUri, String brokerName, String id, Date las this.sourceUri = sourceUri; this.inputBrokerRef = inputBrokerRef; this.taskRef = taskRef; + this.title = title; } /** @@ -101,6 +113,11 @@ public Date getLastModifiedDate() { public URI getSourceUri() { return sourceUri; } + + @Override + public String getTitle() { + return title; + } @Override public byte[] getContent(MimeType...mimeType) throws IOException { diff --git a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java index b6cb2474..5839f021 100644 --- a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java +++ b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/DataReference.java @@ -86,4 +86,6 @@ public interface DataReference extends Serializable, DataContent { * @return task reference of null if ad-hoc */ String getTaskRef(); + + String getTitle(); } diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java index 1e966e2b..100f736e 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/ItemEntry.java @@ -19,7 +19,7 @@ * Item entry. */ public final class ItemEntry { - public String id; + public String id; public String owner; public long created; public long modified; diff --git a/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java b/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java index 1880e812..6fe7697a 100644 --- a/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-source/src/main/java/com/esri/geoportal/harvester/agpsrc/AgpInputBroker.java @@ -196,7 +196,7 @@ public void terminate() { public DataContent readContent(String id) throws DataInputException { try { ItemEntry itemEntry = client.readItem(id, client.generateToken(1).token); - SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef()); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef(),itemEntry.title); ref.addContext(MimeType.APPLICATION_JSON, mapper.writeValueAsString(itemEntry).getBytes("UTF-8")); return ref; } catch (IOException|URISyntaxException ex) { @@ -228,7 +228,7 @@ private DataReference createReference(ItemEntry itemEntry) throws URISyntaxExcep props.put(WKAConstants.WKA_BBOX, sBox); } - SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef()); + SimpleDataReference ref = new SimpleDataReference(getBrokerUri(), definition.getEntityDefinition().getLabel(), itemEntry.id, new Date(itemEntry.modified), URI.create(itemEntry.id), td.getSource().getRef(), td.getRef(),itemEntry.title); if (definition.getEmitXml()) { String orgMeta = null; diff --git a/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java b/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java index 0837a7e8..5f190c7a 100644 --- a/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java +++ b/geoportal-connectors/geoportal-harvester-folder/src/main/java/com/esri/geoportal/harvester/folder/FolderBroker.java @@ -131,8 +131,9 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { try { for (MimeType ct: ref.getContentType()) { String extension = MimeTypeUtils.findExtensions(ct).stream().findFirst().orElse(null); - if (extension!=null) { - Path f = generateFileName(ref.getBrokerUri(), ref.getSourceUri(), ref.getId(), extension); + if (extension!=null) { + + Path f = generateFileName(ref.getBrokerUri(), ref.getSourceUri(), ref.getTitle() , extension); boolean created = !Files.exists(f); Files.createDirectories(f.getParent()); try (OutputStream output = Files.newOutputStream(f)) { @@ -178,7 +179,7 @@ private Path generateFileName(URI brokerUri, URI sourceUri, String id, String ex subFolder.remove(0); } for (String sf : subFolder) { - fileName = Paths.get(fileName.toString(), sf); + fileName = Paths.get(fileName.toString(), (id.isBlank() ? sf :id)); } if (!fileName.getFileName().toString().endsWith(extension)) { fileName = fileName.getParent().resolve(fileName.getFileName() + "." + extension); From 53fa4d535dab9fd159c84e083023f08a647e55f8 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Thu, 21 Sep 2023 13:06:04 +0530 Subject: [PATCH 04/22] Removed /item in elastic search url to support 8.x --- .../main/java/com/esri/geoportal/commons/gpt/client/Client.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java index d7dbe582..11e63c4b 100644 --- a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java +++ b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java @@ -81,7 +81,7 @@ public class Client implements Closeable { private static final String DEFAULT_INDEX = "metadata"; private static final String REST_ITEM_URL = "rest/metadata/item"; - private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/item/_search"; + private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/_search"; private static final String ELASTIC_SCROLL_URL = "elastic/_search/scroll"; private static final String TOKEN_URL = "oauth/token"; From 295915ae12663401366ccb8251a5138db05d9b2a Mon Sep 17 00:00:00 2001 From: cont_anki Date: Thu, 28 Sep 2023 13:52:10 +0530 Subject: [PATCH 05/22] Read json from _source if _source._json not available --- .../geoportal/commons/gpt/client/Client.java | 1447 +++++++++-------- 1 file changed, 726 insertions(+), 721 deletions(-) diff --git a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java index 11e63c4b..f0aca05d 100644 --- a/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java +++ b/geoportal-commons/geoportal-commons-gpt-client/src/main/java/com/esri/geoportal/commons/gpt/client/Client.java @@ -76,768 +76,773 @@ */ public class Client implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(Client.class); - private static final int BATCH_SIZE = 500; - - private static final String DEFAULT_INDEX = "metadata"; - private static final String REST_ITEM_URL = "rest/metadata/item"; - private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/_search"; - private static final String ELASTIC_SCROLL_URL = "elastic/_search/scroll"; - private static final String TOKEN_URL = "oauth/token"; - - private final CloseableHttpClient httpClient; - private final URL url; - private final SimpleCredentials cred; - private final String index; - private final String collectionsFieldName; - - private TokenInfo tokenInfo; - - private final ObjectMapper mapper = new ObjectMapper(); - - /** - * Creates instance of the client. - * - * @param httpClient HTTP client - * @param url URL of the GPT REST end point - * @param cred credentials - * @param index index name - * @param collectionsFieldName collections field name - */ - public Client(CloseableHttpClient httpClient, URL url, SimpleCredentials cred, String index, String collectionsFieldName) { - this.httpClient = httpClient; - this.url = url; - this.cred = cred; - this.index = StringUtils.defaultIfBlank(index, DEFAULT_INDEX); - this.collectionsFieldName = collectionsFieldName; - - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - } - - /** - * Creates instance of the client. - * - * @param url URL of the GPT REST end point - * @param cred credentials - * @param index index name - * @param collectionsFieldName collections field name - */ - public Client(URL url, SimpleCredentials cred, String index, String collectionsFieldName) { - this(HttpClientBuilder.create().useSystemProperties().setRedirectStrategy(LaxRedirectStrategy.INSTANCE).build(), url, cred, index, collectionsFieldName); - } - - /** - * Publishes a document. - * - * @param data data to publish - * @param attributes extra attributes - * @param id custom id - * @param xml xml - * @param json json - * @param forceAdd true to force add. - * @param collections list of collections - * @return response information - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public PublishResponse publish( - PublishRequest data, - Map attributes, - String id, - String xml, String json, - boolean forceAdd, - String [] collections) throws IOException, URISyntaxException { - - ObjectNode jsonRequest = mapper.convertValue(data, ObjectNode.class); - if (xml != null) { - jsonRequest.put("xml", xml); - } - if (json != null) { - try { - ObjectNode jsonValue = mapper.readValue(json, ObjectNode.class); - jsonRequest.set("_json", jsonValue); - if (jsonValue.isObject()) { - Iterator> fldIter = jsonValue.fields(); - while (fldIter.hasNext()) { - Map.Entry fld = fldIter.next(); - - if (fld.getKey().equals("fullExtent")) { - JsonNode fullExtent = fld.getValue(); - - Double xmin = fullExtent.path("xmin").asDouble(); - Double ymin = fullExtent.path("ymin").asDouble(); - Double xmax = fullExtent.path("xmax").asDouble(); - Double ymax = fullExtent.path("ymax").asDouble(); - - if (attributes.containsKey(WKAConstants.WKA_BBOX)) { - Object boxObj = attributes.get(WKAConstants.WKA_BBOX); - if (boxObj!=null && boxObj instanceof Attribute) { - String parts [] = ((Attribute)boxObj).getValue().split(","); - if (parts!=null && parts.length==2) { - String ll[] = parts[0].split(" "); - String ur[] = parts[1].split(" "); - if (ll!=null && ll.length==2 && ur!=null && ur.length==2) { - Double b_xmin = parseDouble(ll[0]); - Double b_ymin = parseDouble(ll[1]); - Double b_xmax = parseDouble(ur[0]); - Double b_ymax = parseDouble(ur[1]); - - if (b_xmin!=null && b_ymin!=null && b_xmax!=null && b_ymax!=null) { - xmin = b_xmin; - ymin = b_ymin; - xmax = b_xmax; - ymax = b_ymax; - } + private static final Logger LOG = LoggerFactory.getLogger(Client.class); + private static final int BATCH_SIZE = 500; + + private static final String DEFAULT_INDEX = "metadata"; + private static final String REST_ITEM_URL = "rest/metadata/item"; + private static final String ELASTIC_SEARCH_URL = "elastic/{metadata}/_search"; + private static final String ELASTIC_SCROLL_URL = "elastic/_search/scroll"; + private static final String TOKEN_URL = "oauth/token"; + + private final CloseableHttpClient httpClient; + private final URL url; + private final SimpleCredentials cred; + private final String index; + private final String collectionsFieldName; + + private TokenInfo tokenInfo; + + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Creates instance of the client. + * + * @param httpClient HTTP client + * @param url URL of the GPT REST end point + * @param cred credentials + * @param index index name + * @param collectionsFieldName collections field name + */ + public Client(CloseableHttpClient httpClient, URL url, SimpleCredentials cred, String index, String collectionsFieldName) { + this.httpClient = httpClient; + this.url = url; + this.cred = cred; + this.index = StringUtils.defaultIfBlank(index, DEFAULT_INDEX); + this.collectionsFieldName = collectionsFieldName; + + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + /** + * Creates instance of the client. + * + * @param url URL of the GPT REST end point + * @param cred credentials + * @param index index name + * @param collectionsFieldName collections field name + */ + public Client(URL url, SimpleCredentials cred, String index, String collectionsFieldName) { + this(HttpClientBuilder.create().useSystemProperties().setRedirectStrategy(LaxRedirectStrategy.INSTANCE).build(), url, cred, index, collectionsFieldName); + } + + /** + * Publishes a document. + * + * @param data data to publish + * @param attributes extra attributes + * @param id custom id + * @param xml xml + * @param json json + * @param forceAdd true to force add. + * @param collections list of collections + * @return response information + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public PublishResponse publish( + PublishRequest data, + Map attributes, + String id, + String xml, String json, + boolean forceAdd, + String[] collections) throws IOException, URISyntaxException { + + ObjectNode jsonRequest = mapper.convertValue(data, ObjectNode.class); + if (xml != null) { + jsonRequest.put("xml", xml); + } + if (json != null) { + try { + ObjectNode jsonValue = mapper.readValue(json, ObjectNode.class); + jsonRequest.set("_json", jsonValue); + if (jsonValue.isObject()) { + Iterator> fldIter = jsonValue.fields(); + while (fldIter.hasNext()) { + Map.Entry fld = fldIter.next(); + + if (fld.getKey().equals("fullExtent")) { + JsonNode fullExtent = fld.getValue(); + + Double xmin = fullExtent.path("xmin").asDouble(); + Double ymin = fullExtent.path("ymin").asDouble(); + Double xmax = fullExtent.path("xmax").asDouble(); + Double ymax = fullExtent.path("ymax").asDouble(); + + if (attributes.containsKey(WKAConstants.WKA_BBOX)) { + Object boxObj = attributes.get(WKAConstants.WKA_BBOX); + if (boxObj != null && boxObj instanceof Attribute) { + String parts[] = ((Attribute) boxObj).getValue().split(","); + if (parts != null && parts.length == 2) { + String ll[] = parts[0].split(" "); + String ur[] = parts[1].split(" "); + if (ll != null && ll.length == 2 && ur != null && ur.length == 2) { + Double b_xmin = parseDouble(ll[0]); + Double b_ymin = parseDouble(ll[1]); + Double b_xmax = parseDouble(ur[0]); + Double b_ymax = parseDouble(ur[1]); + + if (b_xmin != null && b_ymin != null && b_xmax != null && b_ymax != null) { + xmin = b_xmin; + ymin = b_ymin; + xmax = b_xmax; + ymax = b_ymax; + } + } + } + } + } + + ObjectNode envelope_geo = mapper.createObjectNode(); + envelope_geo.put("type", "envelope"); + ArrayNode coordinates = mapper.createArrayNode(); + ArrayNode southWest = coordinates.addArray(); + southWest.add(Math.max(xmin, -180.0)); + southWest.add(Math.min(ymax, 90.0)); + ArrayNode northEast = coordinates.addArray(); + northEast.add(Math.min(xmax, 180.0)); + northEast.add(Math.max(ymin, -90.0)); + envelope_geo.set("coordinates", coordinates); + + jsonRequest.set("envelope_geo", envelope_geo); + + double lon = (xmin + xmax) / 2.0; + double lat = (ymin + ymax) / 2.0; + + ObjectNode envelope_cen_pt = mapper.createObjectNode(); + envelope_cen_pt.put("lon", lon); + envelope_cen_pt.put("lat", lat); + + jsonRequest.set("envelope_cen_pt", envelope_cen_pt); + } + + switch (fld.getValue().getNodeType()) { + case STRING: + String s_format = "%s_txt"; + switch (fld.getKey()) { + case "allowedUploadFileTypes": + case "capabilities": + case "configuredState": + case "clusterName": + case "executionType": + case "geometryType": + case "htmlPopupType": + case "isolationLevel": + case "loadBalancing": + case "supportedQueryFormats": + case "tags": + case "type": + case "typeName": + case "units": + s_format = "%s_s"; + break; + case "title": + case "description": + case "fileid": + s_format = "%s"; + break; + } + jsonRequest.put(String.format(s_format, fld.getKey()), fld.getValue().asText()); + break; + case NUMBER: + jsonRequest.put(String.format("%s_d", fld.getKey()), fld.getValue().asDouble()); + break; + case BOOLEAN: + jsonRequest.put(String.format("%s_b", fld.getKey()), fld.getValue().asBoolean()); + break; + case ARRAY: + jsonRequest.set(String.format("%s", fld.getKey()), fld.getValue()); + break; + case OBJECT: +// jsonRequest.set(String.format("%s_obj", fld.getKey()), fld.getValue()); + break; + } } - } } - } + } catch (Exception ex) { + LOG.debug(String.format("Invalid json received.", json), ex); + } + } + + for (Map.Entry entry : attributes.entrySet()) { + if (entry.getValue() == null) { + jsonRequest.putNull(entry.getKey()); + } else { + if (entry.getValue() instanceof String) { + jsonRequest.put(entry.getKey(), (String) entry.getValue()); + } else if (entry.getValue() instanceof Double) { + jsonRequest.put(entry.getKey(), (Double) entry.getValue()); + } else if (entry.getValue() instanceof BigDecimal) { + jsonRequest.put(entry.getKey(), ((BigDecimal) entry.getValue()).doubleValue()); + } else if (entry.getValue() instanceof Float) { + jsonRequest.put(entry.getKey(), (Float) entry.getValue()); + } else if (entry.getValue() instanceof Long) { + jsonRequest.put(entry.getKey(), (Long) entry.getValue()); + } else if (entry.getValue() instanceof BigInteger) { + jsonRequest.put(entry.getKey(), ((BigInteger) entry.getValue()).longValue()); + } else if (entry.getValue() instanceof Integer) { + jsonRequest.put(entry.getKey(), (Integer) entry.getValue()); + } else if (entry.getValue() instanceof Boolean) { + jsonRequest.put(entry.getKey(), (Boolean) entry.getValue()); + } else if (entry.getValue() instanceof JsonNode) { + jsonRequest.set(entry.getKey(), (JsonNode) entry.getValue()); + } + } + } - ObjectNode envelope_geo = mapper.createObjectNode(); - envelope_geo.put("type", "envelope"); - ArrayNode coordinates = mapper.createArrayNode(); - ArrayNode southWest = coordinates.addArray(); - southWest.add(Math.max(xmin, -180.0)); - southWest.add(Math.min(ymax, 90.0)); - ArrayNode northEast = coordinates.addArray(); - northEast.add(Math.min(xmax, 180.0)); - northEast.add(Math.max(ymin, -90.0)); - envelope_geo.set("coordinates", coordinates); + if (collections != null) { + List collectionsList = Arrays.stream(collections) + .map(StringUtils::trimToNull) + .filter(collection -> collection != null) + .collect(Collectors.toList()); - jsonRequest.set("envelope_geo", envelope_geo); + if (!collectionsList.isEmpty()) { + ArrayNode collectionsArray = jsonRequest.putArray(collectionsFieldName); + collectionsList.forEach(collectionsArray::add); + } + } - double lon = (xmin + xmax) / 2.0; - double lat = (ymin + ymax) / 2.0; + String strRequest = mapper.writeValueAsString(jsonRequest); + StringEntity entity = new StringEntity(strRequest, "UTF-8"); - ObjectNode envelope_cen_pt = mapper.createObjectNode(); - envelope_cen_pt.put("lon", lon); - envelope_cen_pt.put("lat", lat); + List ids = !forceAdd ? queryIds("src_uri_s", data.src_uri_s, 1) : Collections.emptyList(); - jsonRequest.set("envelope_cen_pt", envelope_cen_pt); + URI pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); + try { + return publish(pubUri, entity, data.sys_owner_s); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); + return publish(pubUri, entity, data.sys_owner_s); + } else { + throw ex; } + } + } - switch (fld.getValue().getNodeType()) { - case STRING: - String s_format = "%s_txt"; - switch (fld.getKey()) { - case "allowedUploadFileTypes": - case "capabilities": - case "configuredState": - case "clusterName": - case "executionType": - case "geometryType": - case "htmlPopupType": - case "isolationLevel": - case "loadBalancing": - case "supportedQueryFormats": - case "tags": - case "type": - case "typeName": - case "units": - s_format = "%s_s"; - break; - case "title": - case "description": - case "fileid": - s_format = "%s"; - break; + private Double parseDouble(String val) { + try { + return Double.parseDouble(val); + } catch (Exception ex) { + return null; + } + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public String readXml(String id) throws URISyntaxException, IOException { + URI xmlUri = createXmlUri(id); + try { + return readContent(xmlUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + xmlUri = createXmlUri(id); + return readContent(xmlUri); + } else { + throw ex; + } + } + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public String readJson(String id) throws URISyntaxException, IOException { + URI jsonUri = createJsonUri(id); + try { + String content = readContent(jsonUri); + try { + JsonNode root = mapper.readTree(content); + JsonNode json = null; + if (root.isObject() && root.has("_source") && root.get("_source").isObject() && root.get("_source").has("_json")) { + json = root.get("_source").get("_json"); + return mapper.writeValueAsString(json); + } //if _source._json not there then take _source + else if (root.isObject() && root.has("_source") && json == null) { + json = root.get("_source"); + return mapper.writeValueAsString(json); } - jsonRequest.put(String.format(s_format, fld.getKey()), fld.getValue().asText()); - break; - case NUMBER: - jsonRequest.put(String.format("%s_d", fld.getKey()), fld.getValue().asDouble()); - break; - case BOOLEAN: - jsonRequest.put(String.format("%s_b", fld.getKey()), fld.getValue().asBoolean()); - break; - case ARRAY: - jsonRequest.set(String.format("%s", fld.getKey()), fld.getValue()); - break; - case OBJECT: -// jsonRequest.set(String.format("%s_obj", fld.getKey()), fld.getValue()); - break; + } catch (IOException ex) { + // ignore + } + return null; + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + jsonUri = createJsonUri(id); + return readContent(jsonUri); + } else { + throw ex; } - } } - } catch (Exception ex) { - LOG.debug(String.format("Invalid json received.", json), ex); - } - } - - for (Map.Entry entry : attributes.entrySet()) { - if (entry.getValue() == null) { - jsonRequest.putNull(entry.getKey()); - } else { - if (entry.getValue() instanceof String) { - jsonRequest.put(entry.getKey(), (String) entry.getValue()); - } else if (entry.getValue() instanceof Double) { - jsonRequest.put(entry.getKey(), (Double) entry.getValue()); - } else if (entry.getValue() instanceof BigDecimal) { - jsonRequest.put(entry.getKey(), ((BigDecimal) entry.getValue()).doubleValue()); - } else if (entry.getValue() instanceof Float) { - jsonRequest.put(entry.getKey(), (Float) entry.getValue()); - } else if (entry.getValue() instanceof Long) { - jsonRequest.put(entry.getKey(), (Long) entry.getValue()); - } else if (entry.getValue() instanceof BigInteger) { - jsonRequest.put(entry.getKey(), ((BigInteger) entry.getValue()).longValue()); - } else if (entry.getValue() instanceof Integer) { - jsonRequest.put(entry.getKey(), (Integer) entry.getValue()); - } else if (entry.getValue() instanceof Boolean) { - jsonRequest.put(entry.getKey(), (Boolean) entry.getValue()); - } else if (entry.getValue() instanceof JsonNode) { - jsonRequest.set(entry.getKey(), (JsonNode) entry.getValue()); + } + + /** + * Reads metadata. + * + * @param id id of the metadata + * @return string representing metadata + * @throws URISyntaxException if invalid URI + * @throws IOException if reading metadata fails + */ + public EntryRef readItem(String id) throws URISyntaxException, IOException { + URI itemUri = createItemUri(id); + try { + return readItem(itemUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + itemUri = createItemUri(id); + return readItem(itemUri); + } else { + throw ex; + } } - } - } - - if (collections!=null) { - List collectionsList = Arrays.stream(collections) - .map(StringUtils::trimToNull) - .filter(collection -> collection!=null) - .collect(Collectors.toList()); - - if (!collectionsList.isEmpty()) { - ArrayNode collectionsArray = jsonRequest.putArray(collectionsFieldName); - collectionsList.forEach(collectionsArray::add); - } - } - - String strRequest = mapper.writeValueAsString(jsonRequest); - StringEntity entity = new StringEntity(strRequest, "UTF-8"); - - List ids = !forceAdd ? queryIds("src_uri_s", data.src_uri_s, 1) : Collections.emptyList(); - - URI pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); - try { - return publish(pubUri, entity, data.sys_owner_s); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - pubUri = id != null ? createItemUri(id) : !ids.isEmpty() ? createItemUri(ids.get(0)) : createItemsUri(); - return publish(pubUri, entity, data.sys_owner_s); - } else { - throw ex; - } - } - } - - private Double parseDouble(String val) { - try { - return Double.parseDouble(val); - } catch (Exception ex) { - return null; - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public String readXml(String id) throws URISyntaxException, IOException { - URI xmlUri = createXmlUri(id); - try { - return readContent(xmlUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - xmlUri = createXmlUri(id); - return readContent(xmlUri); - } else { - throw ex; - } - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public String readJson(String id) throws URISyntaxException, IOException { - URI jsonUri = createJsonUri(id); - try { - String content = readContent(jsonUri); - try { - JsonNode root = mapper.readTree(content); - if (root.isObject() && root.has("_source") && root.get("_source").isObject() && root.get("_source").has("_json")) { - JsonNode json = root.get("_source").get("_json"); - return mapper.writeValueAsString(json); + } + + /** + * Returns listIds of ids. + * + * @return listIds of ids or null if no more ids. + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public List listIds() throws URISyntaxException, IOException { + return queryIds(null, null, BATCH_SIZE); + } + + @Override + public void close() throws IOException { + if (httpClient instanceof Closeable) { + ((Closeable) httpClient).close(); } - } catch (IOException ex) { - // ignore - } - return null; - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - jsonUri = createJsonUri(id); - return readContent(jsonUri); - } else { - throw ex; - } - } - } - - /** - * Reads metadata. - * - * @param id id of the metadata - * @return string representing metadata - * @throws URISyntaxException if invalid URI - * @throws IOException if reading metadata fails - */ - public EntryRef readItem(String id) throws URISyntaxException, IOException { - URI itemUri = createItemUri(id); - try { - return readItem(itemUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - itemUri = createItemUri(id); - return readItem(itemUri); - } else { - throw ex; - } - } - } - - /** - * Returns listIds of ids. - * - * @return listIds of ids or null if no more ids. - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public List listIds() throws URISyntaxException, IOException { - return queryIds(null, null, BATCH_SIZE); - } - - @Override - public void close() throws IOException { - if (httpClient instanceof Closeable) { - ((Closeable) httpClient).close(); - } - } - - /** - * Query items by src_source_uri_s. - * - * @param src_source_uri_s query - * @return query response - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public List queryBySource(String src_source_uri_s) throws IOException, URISyntaxException { - return queryIds("src_source_uri_s", src_source_uri_s, BATCH_SIZE); - } - - /** - * Deletes record by id. - * - * @param id record id - * @return publish response - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - public PublishResponse delete(String id) throws URISyntaxException, IOException { - URI deleteUri = createItemUri(id); - try { - return delete(deleteUri); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - deleteUri = createItemUri(id); - return delete(deleteUri); - } else { - throw ex; - } - } - } - - private PublishResponse publish(URI uri, StringEntity entity, String owner) throws IOException, URISyntaxException { - HttpPut put = new HttpPut(uri); - put.setConfig(DEFAULT_REQUEST_CONFIG); - put.setEntity(entity); - put.setHeader("Content-Type", "application/json; charset=UTF-8"); - put.setHeader("User-Agent", HttpConstants.getUserAgent()); - - PublishResponse response = execute(put, PublishResponse.class); - if (response.getError() == null && owner != null) { - changeOwner(response.getId(), owner); - } - - return response; - } - - private EntryRef readItem(URI uri) throws URISyntaxException, IOException { - HttpGet get = new HttpGet(uri); - get.setConfig(DEFAULT_REQUEST_CONFIG); - get.setHeader("User-Agent", HttpConstants.getUserAgent()); - Hit hit = execute(get, Hit.class); - return new EntryRef(hit._id, readUri(hit._source, uri), readLastUpdated(hit._source, new Date())); - } - - private String readContent(URI uri) throws URISyntaxException, IOException { - HttpGet get = new HttpGet(uri); - get.setConfig(DEFAULT_REQUEST_CONFIG); - get.setHeader("User-Agent", HttpConstants.getUserAgent()); - - try (CloseableHttpResponse httpResponse = httpClient.execute(get); InputStream contentStream = httpResponse.getEntity().getContent();) { - if (httpResponse.getStatusLine().getStatusCode() >= 400) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); - } - String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); - String responseContent = IOUtils.toString(contentStream, "UTF-8"); - LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); - return responseContent; - } - } - - private PublishResponse changeOwner(String id, String owner) throws IOException, URISyntaxException { - URI uri = createChangeOwnerUri(id, owner); - HttpPut put = new HttpPut(uri); - put.setConfig(DEFAULT_REQUEST_CONFIG); - put.setHeader("Content-Type", "application/json; charset=UTF-8"); - put.setHeader("User-Agent", HttpConstants.getUserAgent()); - - return execute(put, PublishResponse.class); - } - - private URI readUri(QueryResponse.Source source, URI defUri) { - if (source != null && source.src_uri_s != null) { - try { - return new URI(source.src_uri_s); - } catch (Exception ex) { - } - } - return defUri; - } - - private Date readLastUpdated(QueryResponse.Source source, Date defDate) { - if (source != null && source.src_lastupdate_dt != null) { - try { - return Date.from(ZonedDateTime.from(DateTimeFormatter.ISO_DATE_TIME.parse(source.src_lastupdate_dt)).toInstant()); - } catch (Exception ex) { - } - } - return defDate; - } - - private PublishResponse delete(URI uri) throws URISyntaxException, IOException { - HttpDelete del = new HttpDelete(uri); - del.setConfig(DEFAULT_REQUEST_CONFIG); - del.setHeader("User-Agent", HttpConstants.getUserAgent()); - return execute(del, PublishResponse.class); - } - - private URI createItemsUri() throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createItemUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createChangeOwnerUri(String id, String owner) throws URISyntaxException, IOException { - return new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/owner/" + owner)) - .addParameter("access_token", getAccessToken()) - .build(); - } - - private URI createXmlUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/xml")); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - private URI createJsonUri(String id) throws URISyntaxException, IOException { - URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); - if (cred != null && !cred.isEmpty()) { - b.addParameter("access_token", getAccessToken()); - } - return b.build(); - } - - /** - * Query ids. - * - * @param term term to query - * @param value value of the term - * @param batchSize batch size (note: size 1 indicates looking for the first - * only) - * @return listIds of ids - * @throws IOException if reading response fails - * @throws URISyntaxException if URL has invalid syntax - */ - private List queryIds(String term, String value, long batchSize) throws IOException, URISyntaxException { - Set ids = new HashSet<>(); - String search_after = null; - - ObjectNode root = mapper.createObjectNode(); - root.put("size", batchSize); - root.set("_source", mapper.createArrayNode().add("_id")); - root.set("sort", mapper.createArrayNode().add(mapper.createObjectNode().put("_id", "asc"))); - if (term != null && value != null) { - root.set("query", mapper.createObjectNode().set("match", mapper.createObjectNode().put(term, value))); - } - - do { - URIBuilder builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())); - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); - } - - if (search_after != null) { - root.set("search_after", mapper.createArrayNode().add(search_after)); - } - - String json = mapper.writeValueAsString(root); - HttpEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON); - - QueryResponse response = query(builder, entity); - if (response!=null && response.status!=null && response.status==400 && search_after==null) { - // This indicates it could be an old version of Elastic Search behind the Geoportal. - // Fall back to using scroll API - return queryIdsScroll(term, value, batchSize); - } - - search_after = null; - if (response != null && response.hasHits()) { - List responseIds = response.hits.hits.stream().map(hit -> hit._id).collect(Collectors.toList()); - ids.addAll(responseIds); - - // if argument 'size' is 1 that means looking for the first one only; otherwise looking for every possible - search_after = batchSize > 1 ? responseIds.get(responseIds.size() - 1) : null; - } - } while (search_after != null && !Thread.currentThread().isInterrupted()); - - return ids.stream().collect(Collectors.toList()); - } - - private List queryIdsScroll(String term, String value, long size) throws IOException, URISyntaxException { - ArrayList ids = new ArrayList<>(); - SearchContext searchContext = new SearchContext(); - - while (!Thread.currentThread().isInterrupted()) { - QueryResponse response = query(term, value, size, searchContext); - if (Thread.currentThread().isInterrupted()) { - break; - } - if (response.hits == null || response.hits.hits == null || response.hits.hits.isEmpty()) { - break; - } - ids.addAll(response.hits.hits.stream() - .map(h -> h._id) - .filter(id -> id != null) - .collect(Collectors.toList())); - } - - return ids; - } - - private void clearToken() { - tokenInfo = null; - } - - private QueryResponse query(String term, String value, long size, SearchContext searchContext) throws IOException, URISyntaxException { - URI uri = createQueryUri(searchContext); - HttpEntity httpEntity = createQueryEntity(term, value, size, searchContext); - try { - QueryResponse response = query(uri, httpEntity); - searchContext._scroll_id = response._scroll_id; - return response; - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); - uri = createQueryUri(searchContext); - httpEntity = createQueryEntity(term, value, size, searchContext); - QueryResponse response = query(uri, httpEntity); - searchContext._scroll_id = response._scroll_id; + } + + /** + * Query items by src_source_uri_s. + * + * @param src_source_uri_s query + * @return query response + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public List queryBySource(String src_source_uri_s) throws IOException, URISyntaxException { + return queryIds("src_source_uri_s", src_source_uri_s, BATCH_SIZE); + } + + /** + * Deletes record by id. + * + * @param id record id + * @return publish response + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + public PublishResponse delete(String id) throws URISyntaxException, IOException { + URI deleteUri = createItemUri(id); + try { + return delete(deleteUri); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + deleteUri = createItemUri(id); + return delete(deleteUri); + } else { + throw ex; + } + } + } + + private PublishResponse publish(URI uri, StringEntity entity, String owner) throws IOException, URISyntaxException { + HttpPut put = new HttpPut(uri); + put.setConfig(DEFAULT_REQUEST_CONFIG); + put.setEntity(entity); + put.setHeader("Content-Type", "application/json; charset=UTF-8"); + put.setHeader("User-Agent", HttpConstants.getUserAgent()); + + PublishResponse response = execute(put, PublishResponse.class); + if (response.getError() == null && owner != null) { + changeOwner(response.getId(), owner); + } + return response; - } else { - throw ex; - } } - } - private QueryResponse query(URIBuilder builder, HttpEntity entity) throws IOException, URISyntaxException { - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + private EntryRef readItem(URI uri) throws URISyntaxException, IOException { + HttpGet get = new HttpGet(uri); + get.setConfig(DEFAULT_REQUEST_CONFIG); + get.setHeader("User-Agent", HttpConstants.getUserAgent()); + Hit hit = execute(get, Hit.class); + return new EntryRef(hit._id, readUri(hit._source, uri), readLastUpdated(hit._source, new Date())); + } + + private String readContent(URI uri) throws URISyntaxException, IOException { + HttpGet get = new HttpGet(uri); + get.setConfig(DEFAULT_REQUEST_CONFIG); + get.setHeader("User-Agent", HttpConstants.getUserAgent()); + + try ( CloseableHttpResponse httpResponse = httpClient.execute(get); InputStream contentStream = httpResponse.getEntity().getContent();) { + if (httpResponse.getStatusLine().getStatusCode() >= 400) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + return responseContent; + } + } + + private PublishResponse changeOwner(String id, String owner) throws IOException, URISyntaxException { + URI uri = createChangeOwnerUri(id, owner); + HttpPut put = new HttpPut(uri); + put.setConfig(DEFAULT_REQUEST_CONFIG); + put.setHeader("Content-Type", "application/json; charset=UTF-8"); + put.setHeader("User-Agent", HttpConstants.getUserAgent()); + + return execute(put, PublishResponse.class); + } + + private URI readUri(QueryResponse.Source source, URI defUri) { + if (source != null && source.src_uri_s != null) { + try { + return new URI(source.src_uri_s); + } catch (Exception ex) { + } + } + return defUri; } - QueryResponse response = null; - try { - response = query(builder.build(), entity); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() == 401) { - clearToken(); + private Date readLastUpdated(QueryResponse.Source source, Date defDate) { + if (source != null && source.src_lastupdate_dt != null) { + try { + return Date.from(ZonedDateTime.from(DateTimeFormatter.ISO_DATE_TIME.parse(source.src_lastupdate_dt)).toInstant()); + } catch (Exception ex) { + } + } + return defDate; + } + + private PublishResponse delete(URI uri) throws URISyntaxException, IOException { + HttpDelete del = new HttpDelete(uri); + del.setConfig(DEFAULT_REQUEST_CONFIG); + del.setHeader("User-Agent", HttpConstants.getUserAgent()); + return execute(del, PublishResponse.class); + } + + private URI createItemsUri() throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL)); if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + b.addParameter("access_token", getAccessToken()); } - response = query(builder.build(), entity); - } else { - throw ex; - } + return b.build(); } - return response; - } + private URI createItemUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } - private QueryResponse query(URI uri, HttpEntity httpEntity) throws IOException, URISyntaxException { - HttpPost request = new HttpPost(uri); - request.setEntity(httpEntity); + private URI createChangeOwnerUri(String id, String owner) throws URISyntaxException, IOException { + return new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/owner/" + owner)) + .addParameter("access_token", getAccessToken()) + .build(); + } - request.setConfig(DEFAULT_REQUEST_CONFIG); - request.setHeader("Content-Type", "application/json"); - request.setHeader("User-Agent", HttpConstants.getUserAgent()); + private URI createXmlUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id + "/xml")); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } + + private URI createJsonUri(String id) throws URISyntaxException, IOException { + URIBuilder b = new URIBuilder(url.toURI().resolve(REST_ITEM_URL + "/" + id)); + if (cred != null && !cred.isEmpty()) { + b.addParameter("access_token", getAccessToken()); + } + return b.build(); + } - return execute(request, QueryResponse.class); - } + /** + * Query ids. + * + * @param term term to query + * @param value value of the term + * @param batchSize batch size (note: size 1 indicates looking for the first + * only) + * @return listIds of ids + * @throws IOException if reading response fails + * @throws URISyntaxException if URL has invalid syntax + */ + private List queryIds(String term, String value, long batchSize) throws IOException, URISyntaxException { + Set ids = new HashSet<>(); + String search_after = null; + + ObjectNode root = mapper.createObjectNode(); + root.put("size", batchSize); + root.set("_source", mapper.createArrayNode().add("_id")); + root.set("sort", mapper.createArrayNode().add(mapper.createObjectNode().put("_id", "asc"))); + if (term != null && value != null) { + root.set("query", mapper.createObjectNode().set("match", mapper.createObjectNode().put(term, value))); + } - private String createElasticSearchUrl() { - return ELASTIC_SEARCH_URL.replaceAll("\\{metadata\\}", index); - } + do { + URIBuilder builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())); + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } - private HttpEntity createQueryEntity(String term, String value, long size, SearchContext searchContext) { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode node = mapper.createObjectNode(); - if (searchContext._scroll_id == null) { - node.put("size", size); - if (term != null && value != null) { - ObjectNode query = mapper.createObjectNode(); - node.set("query", query); + if (search_after != null) { + root.set("search_after", mapper.createArrayNode().add(search_after)); + } - ObjectNode match = mapper.createObjectNode(); - query.set("match", match); + String json = mapper.writeValueAsString(root); + HttpEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON); - match.put(term, value); - } - } else { - node.put("scroll", "1m"); - node.put("scroll_id", searchContext._scroll_id); + QueryResponse response = query(builder, entity); + if (response != null && response.status != null && response.status == 400 && search_after == null) { + // This indicates it could be an old version of Elastic Search behind the Geoportal. + // Fall back to using scroll API + return queryIdsScroll(term, value, batchSize); + } + + search_after = null; + if (response != null && response.hasHits()) { + List responseIds = response.hits.hits.stream().map(hit -> hit._id).collect(Collectors.toList()); + ids.addAll(responseIds); + + // if argument 'size' is 1 that means looking for the first one only; otherwise looking for every possible + search_after = batchSize > 1 ? responseIds.get(responseIds.size() - 1) : null; + } + } while (search_after != null && !Thread.currentThread().isInterrupted()); + + return ids.stream().collect(Collectors.toList()); } - return new StringEntity(node.toString(), ContentType.APPLICATION_JSON); - } + private List queryIdsScroll(String term, String value, long size) throws IOException, URISyntaxException { + ArrayList ids = new ArrayList<>(); + SearchContext searchContext = new SearchContext(); - private URI createQueryUri(SearchContext searchContext) throws IOException, URISyntaxException { - URIBuilder builder; + while (!Thread.currentThread().isInterrupted()) { + QueryResponse response = query(term, value, size, searchContext); + if (Thread.currentThread().isInterrupted()) { + break; + } + if (response.hits == null || response.hits.hits == null || response.hits.hits.isEmpty()) { + break; + } + ids.addAll(response.hits.hits.stream() + .map(h -> h._id) + .filter(id -> id != null) + .collect(Collectors.toList())); + } - if (searchContext._scroll_id == null) { - builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())) - .addParameter("scroll", "1m"); - } else { - builder = new URIBuilder(url.toURI().resolve(ELASTIC_SCROLL_URL)) - .addParameter("scroll_id", searchContext._scroll_id) - .addParameter("scroll", "1m"); + return ids; } - if (cred != null && !cred.isEmpty()) { - builder = builder.addParameter("access_token", getAccessToken()); + private void clearToken() { + tokenInfo = null; } - return builder.build(); - } + private QueryResponse query(String term, String value, long size, SearchContext searchContext) throws IOException, URISyntaxException { + URI uri = createQueryUri(searchContext); + HttpEntity httpEntity = createQueryEntity(term, value, size, searchContext); + try { + QueryResponse response = query(uri, httpEntity); + searchContext._scroll_id = response._scroll_id; + return response; + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + uri = createQueryUri(searchContext); + httpEntity = createQueryEntity(term, value, size, searchContext); + QueryResponse response = query(uri, httpEntity); + searchContext._scroll_id = response._scroll_id; + return response; + } else { + throw ex; + } + } + } - private T execute(HttpUriRequest req, Class clazz) throws IOException, URISyntaxException { - try (CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { - String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); - String responseContent = IOUtils.toString(contentStream, "UTF-8"); - LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + private QueryResponse query(URIBuilder builder, HttpEntity entity) throws IOException, URISyntaxException { + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } - if (httpResponse.getStatusLine().getStatusCode() >= 400) { - T value = null; + QueryResponse response = null; try { - value = mapper.readValue(responseContent, clazz); - } catch (Exception ex) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + response = query(builder.build(), entity); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() == 401) { + clearToken(); + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } + response = query(builder.build(), entity); + } else { + throw ex; + } + } + + return response; + } + + private QueryResponse query(URI uri, HttpEntity httpEntity) throws IOException, URISyntaxException { + HttpPost request = new HttpPost(uri); + request.setEntity(httpEntity); + + request.setConfig(DEFAULT_REQUEST_CONFIG); + request.setHeader("Content-Type", "application/json"); + request.setHeader("User-Agent", HttpConstants.getUserAgent()); + + return execute(request, QueryResponse.class); + } + + private String createElasticSearchUrl() { + return ELASTIC_SEARCH_URL.replaceAll("\\{metadata\\}", index); + } + + private HttpEntity createQueryEntity(String term, String value, long size, SearchContext searchContext) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode node = mapper.createObjectNode(); + if (searchContext._scroll_id == null) { + node.put("size", size); + if (term != null && value != null) { + ObjectNode query = mapper.createObjectNode(); + node.set("query", query); + + ObjectNode match = mapper.createObjectNode(); + query.set("match", match); + + match.put(term, value); + } + } else { + node.put("scroll", "1m"); + node.put("scroll_id", searchContext._scroll_id); + } + + return new StringEntity(node.toString(), ContentType.APPLICATION_JSON); + } + + private URI createQueryUri(SearchContext searchContext) throws IOException, URISyntaxException { + URIBuilder builder; + + if (searchContext._scroll_id == null) { + builder = new URIBuilder(url.toURI().resolve(createElasticSearchUrl())) + .addParameter("scroll", "1m"); + } else { + builder = new URIBuilder(url.toURI().resolve(ELASTIC_SCROLL_URL)) + .addParameter("scroll_id", searchContext._scroll_id) + .addParameter("scroll", "1m"); + } + + if (cred != null && !cred.isEmpty()) { + builder = builder.addParameter("access_token", getAccessToken()); + } + + return builder.build(); + } + + private T execute(HttpUriRequest req, Class clazz) throws IOException, URISyntaxException { + try ( CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { + String reasonMessage = httpResponse.getStatusLine().getReasonPhrase(); + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + LOG.trace(String.format("RESPONSE: %s, %s", responseContent, reasonMessage)); + + if (httpResponse.getStatusLine().getStatusCode() >= 400) { + T value = null; + try { + value = mapper.readValue(responseContent, clazz); + } catch (Exception ex) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + if (value == null) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + return value; + } + + return mapper.readValue(responseContent, clazz); + } + } + + private String getAccessToken() throws URISyntaxException, IOException { + LocalDateTime now = LocalDateTime.now(); + if (tokenInfo == null || tokenInfo.validTill.minusSeconds(60).isBefore(now)) { + Token token = generateToken(); + if (token.access_token == null) { + throw new IOException("Error obtaining access token"); + } + TokenInfo ti = new TokenInfo(); + ti.token = token; + ti.validTill = now.plusSeconds(token.expires_in); + tokenInfo = ti; } - if (value == null) { - throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + return tokenInfo.token.access_token; + } + + private Token generateToken() throws URISyntaxException, UnsupportedEncodingException, IOException { + HttpPost post = new HttpPost(url.toURI().resolve(TOKEN_URL)); + post.setConfig(DEFAULT_REQUEST_CONFIG); + post.setHeader("User-Agent", HttpConstants.getUserAgent()); + post.setHeader("Content-Type", "application/x-www-form-urlencoded"); + post.setHeader("Accept", "application/json"); + HashMap params = new HashMap<>(); + if (cred != null) { + params.put("username", StringUtils.trimToEmpty(cred.getUserName())); + params.put("password", StringUtils.trimToEmpty(cred.getPassword())); } - return value; - } - - return mapper.readValue(responseContent, clazz); - } - } - - private String getAccessToken() throws URISyntaxException, IOException { - LocalDateTime now = LocalDateTime.now(); - if (tokenInfo == null || tokenInfo.validTill.minusSeconds(60).isBefore(now)) { - Token token = generateToken(); - if (token.access_token == null) { - throw new IOException("Error obtaining access token"); - } - TokenInfo ti = new TokenInfo(); - ti.token = token; - ti.validTill = now.plusSeconds(token.expires_in); - tokenInfo = ti; - } - return tokenInfo.token.access_token; - } - - private Token generateToken() throws URISyntaxException, UnsupportedEncodingException, IOException { - HttpPost post = new HttpPost(url.toURI().resolve(TOKEN_URL)); - post.setConfig(DEFAULT_REQUEST_CONFIG); - post.setHeader("User-Agent", HttpConstants.getUserAgent()); - post.setHeader("Content-Type", "application/x-www-form-urlencoded"); - post.setHeader("Accept", "application/json"); - HashMap params = new HashMap<>(); - if (cred != null) { - params.put("username", StringUtils.trimToEmpty(cred.getUserName())); - params.put("password", StringUtils.trimToEmpty(cred.getPassword())); - } - params.put("grant_type", "password"); - params.put("client_id", "geoportal-client"); - HttpEntity entity = new UrlEncodedFormEntity(params.entrySet().stream() - .map(e -> new BasicNameValuePair(e.getKey(), e.getValue())).collect(Collectors.toList())); - post.setEntity(entity); - - return execute(post, Token.class); - } - - /** - * Search context. - */ - public static class SearchContext { - - public String _scroll_id; - } - - /** - * Access token. - */ - public static class Token { - - public String access_token; - public String token_type; - public Long expires_in; - public String scope; - public String jti; - } - - private static class TokenInfo { - - public Token token; - public LocalDateTime validTill; - } + params.put("grant_type", "password"); + params.put("client_id", "geoportal-client"); + HttpEntity entity = new UrlEncodedFormEntity(params.entrySet().stream() + .map(e -> new BasicNameValuePair(e.getKey(), e.getValue())).collect(Collectors.toList())); + post.setEntity(entity); + + return execute(post, Token.class); + } + + /** + * Search context. + */ + public static class SearchContext { + + public String _scroll_id; + } + + /** + * Access token. + */ + public static class Token { + + public String access_token; + public String token_type; + public Long expires_in; + public String scope; + public String jti; + } + + private static class TokenInfo { + + public Token token; + public LocalDateTime validTill; + } } From af76395a98ad2cb8580544aa7bebee32b3f3fcf5 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Thu, 28 Sep 2023 13:07:01 -0700 Subject: [PATCH 06/22] use full metadata vs item info from ArcGIS Server services resolves https://github.com/Esri/geoportal-server-harvester/issues/211 --- .../com/esri/geoportal/harvester/ags/AgsBroker.java | 6 ++---- .../harvester/ags/AgsBrokerDefinitionAdaptor.java | 12 ++++++++++++ .../esri/geoportal/harvester/ags/AgsConnector.java | 1 + .../esri/geoportal/harvester/ags/AgsConstants.java | 1 + .../src/main/resources/AgsResource.properties | 5 +++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java index 7e0b892a..c4acc797 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBroker.java @@ -286,11 +286,9 @@ private DataReference createReference(ServerResponse serverResponse) throws IOEx } Document document = null; - byte[] bytes = null; + byte[] bytes; - if (metadataXML != null && !metadataXML.trim().isEmpty()) { - bytes = metadataXML.getBytes("UTF-8"); - + if ((this.definition.getUseServiceXml()) && (metadataXML != null && !metadataXML.trim().isEmpty())) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(false); DocumentBuilder builder = null; diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java index e8a2b99f..ac87629b 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsBrokerDefinitionAdaptor.java @@ -41,6 +41,7 @@ public class AgsBrokerDefinitionAdaptor extends BrokerDefinitionAdaptor { private boolean enableLayers; private boolean emitXml = true; private boolean emitJson = false; + private boolean useServiceXML = false; /** * Creates instance of the adaptor. @@ -64,6 +65,7 @@ public AgsBrokerDefinitionAdaptor(EntityDefinition def) throws InvalidDefinition enableLayers = BooleanUtils.toBoolean(get(P_ENABLE_LAYERS)); emitXml = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_EMIT_XML)), true); emitJson = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_EMIT_JSON)), false); + useServiceXML = BooleanUtils.toBooleanDefaultIfNull(BooleanUtils.toBooleanObject(get(P_USE_FULL_XML)), false); } } @@ -73,6 +75,7 @@ public void override(Map params) { consume(params,P_ENABLE_LAYERS); consume(params,P_EMIT_XML); consume(params,P_EMIT_JSON); + consume(params,P_USE_FULL_XML); credAdaptor.override(params); botsAdaptor.override(params); } @@ -161,4 +164,13 @@ public void setEmitJson(boolean emitJson) { set(P_EMIT_JSON, BooleanUtils.toStringTrueFalse(emitJson)); } + public boolean getUseServiceXml() { + return useServiceXML; + } + + public void setUseServiceXML(boolean useServiceXML) { + this.useServiceXML = useServiceXML; + set(P_USE_FULL_XML, BooleanUtils.toStringTrueFalse(useServiceXML)); + } + } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java index 317bb400..27c5769a 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConnector.java @@ -71,6 +71,7 @@ public boolean isPassword() { args.add(new UITemplate.BooleanArgument(P_ENABLE_LAYERS, bundle.getString("ags.enableLayers"))); args.add(new UITemplate.BooleanArgument(P_EMIT_XML, bundle.getString("ags.emit.xml"),false, Boolean.TRUE)); args.add(new UITemplate.BooleanArgument(P_EMIT_JSON, bundle.getString("ags.emit.json"),false, Boolean.FALSE)); + args.add(new UITemplate.BooleanArgument(P_USE_FULL_XML, bundle.getString("ags.use_full_xml"),false, Boolean.FALSE)); return new UITemplate(getType(), bundle.getString("ags"), args); } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java index ec03ad46..1e93cd3f 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/java/com/esri/geoportal/harvester/ags/AgsConstants.java @@ -23,4 +23,5 @@ public static final String P_ENABLE_LAYERS = "ags-enable-layers"; public static final String P_EMIT_XML = "ags-emit-xml"; public static final String P_EMIT_JSON = "ags-emit-json"; + public static final String P_USE_FULL_XML = "ags-use-full-xml"; } diff --git a/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties b/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties index f8bc2c72..1dd94a22 100644 --- a/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties +++ b/geoportal-connectors/geoportal-harvester-ags/src/main/resources/AgsResource.properties @@ -17,6 +17,7 @@ ags.url = URL ags.username = User name ags.password = User password ags.enableLayers = Enable layers -ags.hint = http://sampleserver1.arcgisonline.com/ArcGIS +ags.hint = https://services.arcgisonline.com/ArcGIS ags.emit.xml = Emit XML -ags.emit.json = Emit JSON \ No newline at end of file +ags.emit.json = Emit JSON +ags.use_full_xml = Use metadata instead of item info \ No newline at end of file From 9c6a260b2117ae5a6a292701802e6f4a64bdc5a2 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Thu, 12 Oct 2023 13:32:04 -0700 Subject: [PATCH 07/22] update sub-layer metadata in ArcGIS Online/Portal updates metadata for a sub-layer in ArcGIS Online. --- .../commons/agp/client/AgpClient.java | 42 ++++++++++++++++++ .../harvester/agp/AgpOutputBroker.java | 43 ++++--------------- 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java index 62de43bd..115582d9 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java @@ -306,6 +306,48 @@ public String writeItemMetadata(String itemId, String metadataXML, String token) throw ex; } } + + /** + * Writes layer metadata. + * @param resourceURL + * @param fileToUpload metadata + * @param token token + * @return true metadata if and only if update successful + * @throws URISyntaxException if invalid URL + * @throws IOException if operation fails + */ + public boolean writeSubLayerMetadata(String resourceURL, File fileToUpload, String token) throws IOException, URISyntaxException { + URIBuilder builder = new URIBuilder(resourceURL); + HttpPost req = new HttpPost(builder.build()); + + Map params = new HashMap<>(); + params.put("f", "json"); + params.put("token", token); + params.put("metadataUploadFormat", "xml"); + params.put("overwrite", "true"); + + try { + HttpEntity entity = createEntity(params, fileToUpload); + req.setEntity(entity); + + try (CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { + if (httpResponse.getStatusLine().getStatusCode()>=400) { + throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); + } + String responseContent = IOUtils.toString(contentStream, "UTF-8"); + System.out.println(responseContent); + if (responseContent.contains("error")) { + return false; + } + } + + } catch (IOException ex) { + System.out.println(ex.getMessage()); + return false; + } + + return true; + } /** * Sharing item. diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 23c87605..9121e350 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -21,8 +21,8 @@ import com.esri.geoportal.commons.agp.client.FolderEntry; import com.esri.geoportal.commons.agp.client.ItemEntry; import com.esri.geoportal.commons.agp.client.ItemResponse; -import com.esri.geoportal.commons.constants.ItemType; import com.esri.geoportal.commons.agp.client.QueryResponse; +import com.esri.geoportal.commons.constants.ItemType; import com.esri.geoportal.commons.constants.MimeType; import com.esri.geoportal.commons.doc.DocUtils; import com.esri.geoportal.commons.meta.ArrayAttribute; @@ -48,7 +48,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.StringReader; -// import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -76,13 +75,8 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -//import org.apache.http.HttpEntity; -//import org.apache.http.client.HttpResponseException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -//import org.apache.http.client.methods.HttpPost; -//import org.apache.http.client.utils.URIBuilder; -//import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.LaxRedirectStrategy; @@ -394,8 +388,8 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { itemEntry.id, itemEntry.owner, itemEntry.ownerFolder, - title, - description, + title, + description, new URL(resourceUrl), sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, itemType, @@ -421,8 +415,9 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { // the metadata is apparently for a sublayer // DO SOMETHING ELSE - String metadataUpdateURI = resourceUrl + "/metadata/update"; - System.out.println("update metadata at " + metadataUpdateURI); + String metadataUpdateURI = resourceUrl + "/metadata/update/"; + boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, metadataFile, token); + System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); return PublishingStatus.SKIPPED; } @@ -442,6 +437,8 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } } } + + private ItemType createItemType(String resourceUrl) { resourceUrl = StringUtils.trimToEmpty(resourceUrl); @@ -551,33 +548,9 @@ private MapAttribute extractMapAttributes(DataReference ref, byte [] content) th String sXml = ""; if (ref.getContentType().contains(MimeType.APPLICATION_XML) || ref.getContentType().contains(MimeType.TEXT_XML)) { sXml = new String(ref.getContent(MimeType.APPLICATION_XML, MimeType.TEXT_XML), "UTF-8"); - //DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - //factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - //factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - //factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - //factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - //factory.setXIncludeAware(false); - //factory.setExpandEntityReferences(false); - //factory.setNamespaceAware(true); - //DocumentBuilder builder = factory.newDocumentBuilder(); - //doc = builder.parse(new InputSource(new StringReader(sXml))); } else if (content!=null) { sXml = new String(content, "UTF-8"); } - /* - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - factory.setFeature("http://xml.org/sax/features/validation", false); - factory.setFeature("http://apache.org/xml/features/validation/schema", false); - factory.setXIncludeAware(false); - factory.setExpandEntityReferences(false); - factory.setNamespaceAware(true); - DocumentBuilder builder = factory.newDocumentBuilder(); - doc = builder.parse(new InputSource(new StringReader(sXml))); - */ doc = stringToDoc(sXml); if (doc!=null) { From 8d4597f3f089ed5bafd431a55934fda3ddc1be59 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Wed, 18 Oct 2023 10:52:11 -0700 Subject: [PATCH 08/22] update terminate function to count actual deletes --- .../esri/geoportal/harvester/gpt/GptBroker.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java index 0597553f..484f70b4 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java +++ b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java @@ -47,7 +47,9 @@ import com.esri.geoportal.harvester.api.ex.DataProcessorException; import com.esri.geoportal.harvester.api.specs.OutputBroker; import com.esri.geoportal.harvester.api.specs.OutputConnector; +import java.util.ArrayList; import java.util.Arrays; +import java.util.ListIterator; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -122,10 +124,16 @@ public void onError(DataException ex) { public void terminate() { try { if (client != null && definition.getCleanup() && !preventCleanup) { + int deleted = 0; + for (String id : existing) { - client.delete(id); - } - LOG.info(String.format("%d records has been removed during cleanup.", existing.size())); + PublishResponse deleteResult = client.delete(id); + if ((deleteResult != null) && (deleteResult.getStatus() != null)) { + deleted += 1; + } + } + + LOG.info(String.format("%d records has been removed during cleanup.", deleted)); } } catch (URISyntaxException | IOException ex) { LOG.error(String.format("Error terminating broker."), ex); From 500c9ffc4b1045cf3104ea8c36f303fb89206fa5 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Wed, 1 Nov 2023 11:35:46 +0530 Subject: [PATCH 09/22] Revert "update terminate function to count actual deletes" This reverts commit 8d4597f3f089ed5bafd431a55934fda3ddc1be59. --- .../esri/geoportal/harvester/gpt/GptBroker.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java index 484f70b4..0597553f 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java +++ b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java @@ -47,9 +47,7 @@ import com.esri.geoportal.harvester.api.ex.DataProcessorException; import com.esri.geoportal.harvester.api.specs.OutputBroker; import com.esri.geoportal.harvester.api.specs.OutputConnector; -import java.util.ArrayList; import java.util.Arrays; -import java.util.ListIterator; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -124,16 +122,10 @@ public void onError(DataException ex) { public void terminate() { try { if (client != null && definition.getCleanup() && !preventCleanup) { - int deleted = 0; - for (String id : existing) { - PublishResponse deleteResult = client.delete(id); - if ((deleteResult != null) && (deleteResult.getStatus() != null)) { - deleted += 1; - } - } - - LOG.info(String.format("%d records has been removed during cleanup.", deleted)); + client.delete(id); + } + LOG.info(String.format("%d records has been removed during cleanup.", existing.size())); } } catch (URISyntaxException | IOException ex) { LOG.error(String.format("Error terminating broker."), ex); From e4cbc1e22ac1a8e85aaae9986ae91bd743096251 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Wed, 1 Nov 2023 12:00:48 +0530 Subject: [PATCH 10/22] Update version to 2.7.1 --- geoportal-application/geoportal-harvester-war/pom.xml | 2 +- .../src/main/webapp/hrv/ui/main/templates/App.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/geoportal-application/geoportal-harvester-war/pom.xml b/geoportal-application/geoportal-harvester-war/pom.xml index 6aa1c772..59aea817 100644 --- a/geoportal-application/geoportal-harvester-war/pom.xml +++ b/geoportal-application/geoportal-harvester-war/pom.xml @@ -4,7 +4,7 @@ geoportal-application com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-harvester-war war diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html index e6b539f6..119d17e1 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/templates/App.html @@ -4,7 +4,7 @@
    -
    Ver. 2.7.0
    +
    Ver. 2.7.1
    From 4670ef2247e8d7f073147a48579197fe48410d43 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Wed, 1 Nov 2023 12:01:25 +0530 Subject: [PATCH 11/22] Update version to 2.7.1 --- .../geoportal-harvester-agp-publisher/pom.xml | 4 ++-- geoportal-connectors/geoportal-harvester-agp-source/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-ags/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-ckan/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-console/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-csw/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-dcat/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-folder-big/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-folder/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-gpt/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-gptsrc/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-jdbc/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-migration/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-sink/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-stac/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-thredds/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-unc/pom.xml | 2 +- geoportal-connectors/geoportal-harvester-waf/pom.xml | 2 +- geoportal-connectors/pom.xml | 4 ++-- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml b/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml index 1dad8b22..5ac6223b 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-agp-publisher Esri :: Geoportal Server :: Harvester :: Data Publisher :: ArcGIS Portal @@ -14,7 +14,7 @@ ${project.groupId} geoportal-commons-agp-client - 2.7.0 + 2.7.1 jar diff --git a/geoportal-connectors/geoportal-harvester-agp-source/pom.xml b/geoportal-connectors/geoportal-harvester-agp-source/pom.xml index b359b12f..21521e40 100644 --- a/geoportal-connectors/geoportal-harvester-agp-source/pom.xml +++ b/geoportal-connectors/geoportal-harvester-agp-source/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-agp-source Esri :: Geoportal Server :: Harvester :: Data Source :: ArcGIS Portal diff --git a/geoportal-connectors/geoportal-harvester-ags/pom.xml b/geoportal-connectors/geoportal-harvester-ags/pom.xml index bffa0331..ab98819d 100644 --- a/geoportal-connectors/geoportal-harvester-ags/pom.xml +++ b/geoportal-connectors/geoportal-harvester-ags/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-ags Esri :: Geoportal Server :: Harvester :: Data Source :: ArcGIS Server diff --git a/geoportal-connectors/geoportal-harvester-ckan/pom.xml b/geoportal-connectors/geoportal-harvester-ckan/pom.xml index 82ca1671..053003b2 100644 --- a/geoportal-connectors/geoportal-harvester-ckan/pom.xml +++ b/geoportal-connectors/geoportal-harvester-ckan/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-ckan jar diff --git a/geoportal-connectors/geoportal-harvester-console/pom.xml b/geoportal-connectors/geoportal-harvester-console/pom.xml index 20b546f3..4e91cc78 100644 --- a/geoportal-connectors/geoportal-harvester-console/pom.xml +++ b/geoportal-connectors/geoportal-harvester-console/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-console Esri :: Geoportal Server :: Harvester :: Data Publisher :: Console diff --git a/geoportal-connectors/geoportal-harvester-csw/pom.xml b/geoportal-connectors/geoportal-harvester-csw/pom.xml index 73e083ce..6c089cc2 100644 --- a/geoportal-connectors/geoportal-harvester-csw/pom.xml +++ b/geoportal-connectors/geoportal-harvester-csw/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-csw Esri :: Geoportal Server :: Harvester :: Data Source :: Csw diff --git a/geoportal-connectors/geoportal-harvester-dcat/pom.xml b/geoportal-connectors/geoportal-harvester-dcat/pom.xml index de0619c5..c5104fe0 100644 --- a/geoportal-connectors/geoportal-harvester-dcat/pom.xml +++ b/geoportal-connectors/geoportal-harvester-dcat/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-dcat jar diff --git a/geoportal-connectors/geoportal-harvester-folder-big/pom.xml b/geoportal-connectors/geoportal-harvester-folder-big/pom.xml index dae5ce0e..748e4272 100644 --- a/geoportal-connectors/geoportal-harvester-folder-big/pom.xml +++ b/geoportal-connectors/geoportal-harvester-folder-big/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-folder-big Esri :: Geoportal Server :: Harvester :: Data Publisher :: Folder :: Big diff --git a/geoportal-connectors/geoportal-harvester-folder/pom.xml b/geoportal-connectors/geoportal-harvester-folder/pom.xml index a4d99059..e4d247e0 100644 --- a/geoportal-connectors/geoportal-harvester-folder/pom.xml +++ b/geoportal-connectors/geoportal-harvester-folder/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-folder Esri :: Geoportal Server :: Harvester :: Data Publisher :: Folder diff --git a/geoportal-connectors/geoportal-harvester-gpt/pom.xml b/geoportal-connectors/geoportal-harvester-gpt/pom.xml index 7d965aa4..a269cb0d 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/pom.xml +++ b/geoportal-connectors/geoportal-harvester-gpt/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-gpt Esri :: Geoportal Server :: Harvester :: Data Publisher :: Geoportal Rest diff --git a/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml b/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml index 55547b73..dabc93cf 100644 --- a/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-gptsrc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-gptsrc Esri :: Geoportal Server :: Harvester :: Data Source :: Geoportal Rest diff --git a/geoportal-connectors/geoportal-harvester-jdbc/pom.xml b/geoportal-connectors/geoportal-harvester-jdbc/pom.xml index dfda8aef..c6c4383f 100644 --- a/geoportal-connectors/geoportal-harvester-jdbc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-jdbc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-jdbc Esri :: Geoportal Server :: Harvester :: Data Source :: JDBC diff --git a/geoportal-connectors/geoportal-harvester-migration/pom.xml b/geoportal-connectors/geoportal-harvester-migration/pom.xml index c393e373..ae1b2239 100644 --- a/geoportal-connectors/geoportal-harvester-migration/pom.xml +++ b/geoportal-connectors/geoportal-harvester-migration/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-migration Esri :: Geoportal Server :: Harvester :: Data Source :: Migration Tool diff --git a/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml b/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml index b04d9e98..ef740bfc 100644 --- a/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml +++ b/geoportal-connectors/geoportal-harvester-oai-pmh/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-oai-pmh Esri :: Geoportal Server :: Harvester :: Data Source :: OAI-PMH diff --git a/geoportal-connectors/geoportal-harvester-sink/pom.xml b/geoportal-connectors/geoportal-harvester-sink/pom.xml index 1f35c05b..39bc365a 100644 --- a/geoportal-connectors/geoportal-harvester-sink/pom.xml +++ b/geoportal-connectors/geoportal-harvester-sink/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-sink Esri :: Geoportal Server :: Harvester :: Data Source :: Sink diff --git a/geoportal-connectors/geoportal-harvester-stac/pom.xml b/geoportal-connectors/geoportal-harvester-stac/pom.xml index 2b689e3f..43043550 100644 --- a/geoportal-connectors/geoportal-harvester-stac/pom.xml +++ b/geoportal-connectors/geoportal-harvester-stac/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-stac jar diff --git a/geoportal-connectors/geoportal-harvester-thredds/pom.xml b/geoportal-connectors/geoportal-harvester-thredds/pom.xml index 4f00e038..bb80222d 100644 --- a/geoportal-connectors/geoportal-harvester-thredds/pom.xml +++ b/geoportal-connectors/geoportal-harvester-thredds/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-thredds jar diff --git a/geoportal-connectors/geoportal-harvester-unc/pom.xml b/geoportal-connectors/geoportal-harvester-unc/pom.xml index f8da7804..6dab131e 100644 --- a/geoportal-connectors/geoportal-harvester-unc/pom.xml +++ b/geoportal-connectors/geoportal-harvester-unc/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-unc Esri :: Geoportal Server :: Harvester :: Data Source :: Unc diff --git a/geoportal-connectors/geoportal-harvester-waf/pom.xml b/geoportal-connectors/geoportal-harvester-waf/pom.xml index 44a26246..a21eae8c 100644 --- a/geoportal-connectors/geoportal-harvester-waf/pom.xml +++ b/geoportal-connectors/geoportal-harvester-waf/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-connectors - 2.7.0 + 2.7.1 geoportal-harvester-waf Esri :: Geoportal Server :: Harvester :: Data Source :: Waf diff --git a/geoportal-connectors/pom.xml b/geoportal-connectors/pom.xml index 526d4971..1f07a677 100644 --- a/geoportal-connectors/pom.xml +++ b/geoportal-connectors/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-connectors pom @@ -16,7 +16,6 @@ geoportal-harvester-dcat geoportal-harvester-folder geoportal-harvester-gpt - geoportal-harvester-waf geoportal-harvester-unc geoportal-harvester-ags geoportal-harvester-agp-publisher @@ -30,5 +29,6 @@ geoportal-harvester-jdbc geoportal-harvester-thredds geoportal-harvester-stac + geoportal-harvester-waf From 93e1ee983cd5c0e3ffa290f4479984e54ac49c7a Mon Sep 17 00:00:00 2001 From: cont_anki Date: Wed, 1 Nov 2023 12:01:39 +0530 Subject: [PATCH 12/22] Update version to 2.7.1 --- README.md | 2 +- geoportal-SDK/geoportal-harvester-api-base/pom.xml | 2 +- geoportal-SDK/geoportal-harvester-api/pom.xml | 4 ++-- geoportal-SDK/pom.xml | 2 +- geoportal-application/geoportal-harvester-cli/pom.xml | 2 +- geoportal-application/geoportal-harvester-engine/pom.xml | 2 +- geoportal-application/pom.xml | 2 +- geoportal-commons/geoportal-commons-agp-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-ags-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-ckan-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-constants/pom.xml | 2 +- geoportal-commons/geoportal-commons-csw-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-dcat-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-doc/pom.xml | 2 +- geoportal-commons/geoportal-commons-geometry/pom.xml | 2 +- geoportal-commons/geoportal-commons-gpt-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-meta/pom.xml | 2 +- geoportal-commons/geoportal-commons-oai-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-pdf/pom.xml | 2 +- geoportal-commons/geoportal-commons-robots/pom.xml | 2 +- geoportal-commons/geoportal-commons-stac-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-thredds-client/pom.xml | 2 +- geoportal-commons/geoportal-commons-utils/pom.xml | 2 +- geoportal-commons/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b1ddf7b0..db010c4a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ To report an issue, please go to [issues](https://github.com/Esri/geoportal-serv The nature of the Harvester application is, as the name suggests, to harvest metadata from whatever web endpoints it is provided. The list(s) of endpoints to download metadata from can also be provided by external entities over the internet. Neither the metadata being harvested nor the list(s) of endpoints provided by external entities are vetted or checked by the Harvester. **Users who wish to limit the scope of the Harvester's reach should configure the network or machine where the Harvester is located with allow lists or deny lists of web endpoints to prevent the Harvester from reaching undesirable locations.** ## Releases and Downloads -- 2.7.0 - June 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.7.0) for release notes and downloads. +- 2.7.1 - June 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.7.1) for release notes and downloads. - 2.6.5 - July 13, 2021, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.6.5) for release notes and downloads. - 2.6.4 - July 8, 2020, click [here](https://github.com/Esri/geoportal-server-harvester/releases/tag/v2.6.4) for release notes and downloads. diff --git a/geoportal-SDK/geoportal-harvester-api-base/pom.xml b/geoportal-SDK/geoportal-harvester-api-base/pom.xml index c966bd0c..af19d12e 100644 --- a/geoportal-SDK/geoportal-harvester-api-base/pom.xml +++ b/geoportal-SDK/geoportal-harvester-api-base/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-SDK - 2.7.0 + 2.7.1 harvester-api-base Esri :: Geoportal Server :: Harvester :: Api Base diff --git a/geoportal-SDK/geoportal-harvester-api/pom.xml b/geoportal-SDK/geoportal-harvester-api/pom.xml index 3ee20895..8ca7330a 100644 --- a/geoportal-SDK/geoportal-harvester-api/pom.xml +++ b/geoportal-SDK/geoportal-harvester-api/pom.xml @@ -4,12 +4,12 @@ geoportal-SDK com.esri.geoportal - 2.7.0 + 2.7.1 harvester-api Esri :: Geoportal Server :: Harvester :: Api Definitions of all basic elements of the Harvester (interfaces, final classes, etc.). - 2.7.0 + 2.7.1 jar diff --git a/geoportal-SDK/pom.xml b/geoportal-SDK/pom.xml index 3652bc1e..2088d0bb 100644 --- a/geoportal-SDK/pom.xml +++ b/geoportal-SDK/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-SDK pom diff --git a/geoportal-application/geoportal-harvester-cli/pom.xml b/geoportal-application/geoportal-harvester-cli/pom.xml index 6fc3d692..9d78426e 100644 --- a/geoportal-application/geoportal-harvester-cli/pom.xml +++ b/geoportal-application/geoportal-harvester-cli/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-application - 2.7.0 + 2.7.1 geoportal-harvester-cli jar diff --git a/geoportal-application/geoportal-harvester-engine/pom.xml b/geoportal-application/geoportal-harvester-engine/pom.xml index b92d56e9..efd007cf 100644 --- a/geoportal-application/geoportal-harvester-engine/pom.xml +++ b/geoportal-application/geoportal-harvester-engine/pom.xml @@ -4,7 +4,7 @@ geoportal-application com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-harvester-engine Esri :: Geoportal Server :: Harvester :: Application :: Engine diff --git a/geoportal-application/pom.xml b/geoportal-application/pom.xml index 2d7641be..eb338b8e 100644 --- a/geoportal-application/pom.xml +++ b/geoportal-application/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-application pom diff --git a/geoportal-commons/geoportal-commons-agp-client/pom.xml b/geoportal-commons/geoportal-commons-agp-client/pom.xml index a51da88b..64348fb6 100644 --- a/geoportal-commons/geoportal-commons-agp-client/pom.xml +++ b/geoportal-commons/geoportal-commons-agp-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-agp-client Esri :: Geoportal Server :: Commons :: ArcGIS Portal Client diff --git a/geoportal-commons/geoportal-commons-ags-client/pom.xml b/geoportal-commons/geoportal-commons-ags-client/pom.xml index 914f5794..a46acd57 100644 --- a/geoportal-commons/geoportal-commons-ags-client/pom.xml +++ b/geoportal-commons/geoportal-commons-ags-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-ags-client Esri :: Geoportal Server :: Commons :: ArcGIS Server Client diff --git a/geoportal-commons/geoportal-commons-ckan-client/pom.xml b/geoportal-commons/geoportal-commons-ckan-client/pom.xml index f58dd4d3..3bc44bb0 100644 --- a/geoportal-commons/geoportal-commons-ckan-client/pom.xml +++ b/geoportal-commons/geoportal-commons-ckan-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-ckan-client Esri :: Geoportal Server :: Commons :: CKAN Lightweight Client diff --git a/geoportal-commons/geoportal-commons-constants/pom.xml b/geoportal-commons/geoportal-commons-constants/pom.xml index 76880082..49631bfb 100644 --- a/geoportal-commons/geoportal-commons-constants/pom.xml +++ b/geoportal-commons/geoportal-commons-constants/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-constants jar diff --git a/geoportal-commons/geoportal-commons-csw-client/pom.xml b/geoportal-commons/geoportal-commons-csw-client/pom.xml index fb4521c0..4fafbf19 100644 --- a/geoportal-commons/geoportal-commons-csw-client/pom.xml +++ b/geoportal-commons/geoportal-commons-csw-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-csw-client Esri :: Geoportal Server :: Commons :: Csw Client diff --git a/geoportal-commons/geoportal-commons-dcat-client/pom.xml b/geoportal-commons/geoportal-commons-dcat-client/pom.xml index a184aa49..c771281c 100644 --- a/geoportal-commons/geoportal-commons-dcat-client/pom.xml +++ b/geoportal-commons/geoportal-commons-dcat-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-dcat-client jar diff --git a/geoportal-commons/geoportal-commons-doc/pom.xml b/geoportal-commons/geoportal-commons-doc/pom.xml index bce527e5..c823f28e 100644 --- a/geoportal-commons/geoportal-commons-doc/pom.xml +++ b/geoportal-commons/geoportal-commons-doc/pom.xml @@ -6,7 +6,7 @@ geoportal-commons com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons-doc jar diff --git a/geoportal-commons/geoportal-commons-geometry/pom.xml b/geoportal-commons/geoportal-commons-geometry/pom.xml index b1570540..cca39d02 100644 --- a/geoportal-commons/geoportal-commons-geometry/pom.xml +++ b/geoportal-commons/geoportal-commons-geometry/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-geometry Esri :: Geoportal Server :: Commons :: Geometry Utils diff --git a/geoportal-commons/geoportal-commons-gpt-client/pom.xml b/geoportal-commons/geoportal-commons-gpt-client/pom.xml index 930df735..9e9a1a51 100644 --- a/geoportal-commons/geoportal-commons-gpt-client/pom.xml +++ b/geoportal-commons/geoportal-commons-gpt-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-gpt-client Esri :: Geoportal Server :: Commons :: Geoportal Rest Client diff --git a/geoportal-commons/geoportal-commons-meta/pom.xml b/geoportal-commons/geoportal-commons-meta/pom.xml index c3a80124..89e7e7f7 100644 --- a/geoportal-commons/geoportal-commons-meta/pom.xml +++ b/geoportal-commons/geoportal-commons-meta/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-meta Esri :: Geoportal Server :: Commons :: Meta diff --git a/geoportal-commons/geoportal-commons-oai-client/pom.xml b/geoportal-commons/geoportal-commons-oai-client/pom.xml index 248d7189..887b3627 100644 --- a/geoportal-commons/geoportal-commons-oai-client/pom.xml +++ b/geoportal-commons/geoportal-commons-oai-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-oai-client Esri :: Geoportal Server :: Commons :: OAI-PMH Client diff --git a/geoportal-commons/geoportal-commons-pdf/pom.xml b/geoportal-commons/geoportal-commons-pdf/pom.xml index 65eb222f..13f63105 100644 --- a/geoportal-commons/geoportal-commons-pdf/pom.xml +++ b/geoportal-commons/geoportal-commons-pdf/pom.xml @@ -6,7 +6,7 @@ geoportal-commons com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons-pdf jar diff --git a/geoportal-commons/geoportal-commons-robots/pom.xml b/geoportal-commons/geoportal-commons-robots/pom.xml index 80bb09f2..d6071470 100644 --- a/geoportal-commons/geoportal-commons-robots/pom.xml +++ b/geoportal-commons/geoportal-commons-robots/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 commons-robots Esri :: Geoportal Server :: Commons :: Robots diff --git a/geoportal-commons/geoportal-commons-stac-client/pom.xml b/geoportal-commons/geoportal-commons-stac-client/pom.xml index c5aa48ec..208cb9f2 100644 --- a/geoportal-commons/geoportal-commons-stac-client/pom.xml +++ b/geoportal-commons/geoportal-commons-stac-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-stac-client jar diff --git a/geoportal-commons/geoportal-commons-thredds-client/pom.xml b/geoportal-commons/geoportal-commons-thredds-client/pom.xml index 7df822c2..381e496f 100644 --- a/geoportal-commons/geoportal-commons-thredds-client/pom.xml +++ b/geoportal-commons/geoportal-commons-thredds-client/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 geoportal-commons-thredds-client Esri :: Geoportal Server :: Commons :: THREDDS Client diff --git a/geoportal-commons/geoportal-commons-utils/pom.xml b/geoportal-commons/geoportal-commons-utils/pom.xml index ba6a9ff0..5b300eca 100644 --- a/geoportal-commons/geoportal-commons-utils/pom.xml +++ b/geoportal-commons/geoportal-commons-utils/pom.xml @@ -4,7 +4,7 @@ com.esri.geoportal geoportal-commons - 2.7.0 + 2.7.1 commons-utils jar diff --git a/geoportal-commons/pom.xml b/geoportal-commons/pom.xml index 3b9ee04a..a3e726d2 100644 --- a/geoportal-commons/pom.xml +++ b/geoportal-commons/pom.xml @@ -4,7 +4,7 @@ geoportal-harvester com.esri.geoportal - 2.7.0 + 2.7.1 geoportal-commons pom diff --git a/pom.xml b/pom.xml index dc4a1234..c8eed2da 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.esri.geoportal geoportal-harvester - 2.7.0 + 2.7.1 pom Esri :: Geoportal Server :: Harvester Top-level project for all Harvester modules. From 1217ebce7a22ebe24d626fafd3bd13d561a4d2ee Mon Sep 17 00:00:00 2001 From: cont_anki Date: Wed, 1 Nov 2023 12:15:06 +0530 Subject: [PATCH 13/22] Revert "Revert "update terminate function to count actual deletes"" This reverts commit 500c9ffc4b1045cf3104ea8c36f303fb89206fa5. --- .../esri/geoportal/harvester/gpt/GptBroker.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java index 0597553f..484f70b4 100644 --- a/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java +++ b/geoportal-connectors/geoportal-harvester-gpt/src/main/java/com/esri/geoportal/harvester/gpt/GptBroker.java @@ -47,7 +47,9 @@ import com.esri.geoportal.harvester.api.ex.DataProcessorException; import com.esri.geoportal.harvester.api.specs.OutputBroker; import com.esri.geoportal.harvester.api.specs.OutputConnector; +import java.util.ArrayList; import java.util.Arrays; +import java.util.ListIterator; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -122,10 +124,16 @@ public void onError(DataException ex) { public void terminate() { try { if (client != null && definition.getCleanup() && !preventCleanup) { + int deleted = 0; + for (String id : existing) { - client.delete(id); - } - LOG.info(String.format("%d records has been removed during cleanup.", existing.size())); + PublishResponse deleteResult = client.delete(id); + if ((deleteResult != null) && (deleteResult.getStatus() != null)) { + deleted += 1; + } + } + + LOG.info(String.format("%d records has been removed during cleanup.", deleted)); } } catch (URISyntaxException | IOException ex) { LOG.error(String.format("Error terminating broker."), ex); From 3773fc2a660182039bb1e1e21963f9753eb2d356 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Fri, 3 Nov 2023 11:17:39 +0530 Subject: [PATCH 14/22] #204 Added oAuth Token for AGOL Sublayer metadata --- .../harvester/rest/ConnectorController.java | 5 ++ .../src/main/webapp/hrv/ui/brokers/Broker.js | 75 ++++++++++++++----- .../webapp/hrv/ui/brokers/BrokerEditorPane.js | 4 + .../src/main/webapp/hrv/ui/main/UIRenderer.js | 56 ++++++++++++-- 4 files changed, 116 insertions(+), 24 deletions(-) diff --git a/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java b/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java index 4dd5e442..32b82f0c 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java +++ b/geoportal-application/geoportal-harvester-war/src/main/java/com/esri/geoportal/harvester/rest/ConnectorController.java @@ -18,6 +18,9 @@ import static com.esri.geoportal.commons.utils.CrlfUtils.formatForLog; import com.esri.geoportal.harvester.api.defs.UITemplate; import com.esri.geoportal.harvester.engine.services.Engine; +import com.esri.geoportal.harvester.engine.services.TemplatesService; +import java.util.List; +import java.util.Locale; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -181,6 +184,8 @@ public UITemplate[] listInboundConnectors() { @RequestMapping(value = "/rest/harvester/connectors/outbound", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) public UITemplate[] listOutboundConnectors() { LOG.debug(String.format("GET /rest/harvester/connectors/outbound")); + TemplatesService srv1 = engine.getTemplatesService(); + List temp2= srv1.getOutboundConnectorTemplates(Locale.US); return engine.getTemplatesService().getOutboundConnectorTemplates(LocaleContextHolder.getLocale()).toArray(new UITemplate[0]); } diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js index 6fe09ebe..cda52818 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js @@ -28,14 +28,18 @@ define(["dojo/_base/declare", "dijit/Dialog", "dijit/ConfirmDialog", "hrv/rest/Brokers", - "hrv/ui/brokers/BrokerEditorPane" + "hrv/ui/brokers/BrokerEditorPane", + "esri/IdentityManager", + "esri/arcgis/OAuthInfo", + "esri/arcgis/Portal" ], function(declare, _WidgetBase,_TemplatedMixin,_WidgetsInTemplateMixin, i18n,template, lang,string,topic,on,json, Dialog,ConfirmDialog, - BrokersREST,BrokerEditorPane + BrokersREST,BrokerEditorPane, + esriId, OAuthInfo, arcgisPortal ){ return declare([_WidgetBase, _TemplatedMixin, _WidgetsInTemplateMixin],{ @@ -71,23 +75,56 @@ define(["dojo/_base/declare", // listen to "submit" button click this.own(on(brokerEditorPane,"submit",lang.hitch(this, function(evt){ - var brokerDefinition = evt.brokerDefinition; - - // use API to update broker - BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( - lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ - topic.publish("msg"); // clear any former errors - this.brokerEditorDialog.destroy(); - this.brokerEditorPane.destroy(); - this.self.load(); - }), - lang.hitch(this,function(error){ - console.debug(error); - topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); - }) - ); - }))); - + var brokerDefinition = evt.brokerDefinition; + var brokerDefinitionProp = evt.brokerDefinition.properties; + var portalUrl = brokerDefinitionProp["agp-host-url"] ; + + if(brokerDefinitionProp["agp-oauth"]=== "true"){ + esriId.getCredential(portalUrl,{oAuthPopupConfirmation:false}).then( + lang.hitch(this,function(){ + var portal = new arcgisPortal.Portal(portalUrl); + portal.signIn().then(function(portalUser){ + + var token = portalUser.credential.token; + brokerDefinitionProp["agp-token"]= token; + + // use API to update broker + BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( + lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ + topic.publish("msg"); // clear any former errors + this.brokerEditorDialog.destroy(); + this.brokerEditorPane.destroy(); + //TODO reload Broker + // this.self.load(); + }), + lang.hitch(this,function(error){ + console.debug(error); + topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); + }) + ); + }).otherwise(function(error){ + console.warn("Error occurred while signing in:",error); + }); + })); + } + else + { + brokerDefinitionProp["agp-token"]= ""; + // use API to update broker + BrokersREST.update(brokerDefinition.uuid,json.stringify(brokerDefinition)).then( + lang.hitch({brokerEditorPane: brokerEditorPane, brokerEditorDialog: brokerEditorDialog, self: this},function(){ + topic.publish("msg"); // clear any former errors + this.brokerEditorDialog.destroy(); + this.brokerEditorPane.destroy(); + this.self.load(); + }), + lang.hitch(this,function(error){ + console.debug(error); + topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); + })); + } + }))); + brokerEditorDialog.show(); }, diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js index 888d804e..b6bd2569 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/BrokerEditorPane.js @@ -108,6 +108,10 @@ define(["dojo/_base/declare", this.updateArgumentsForm(this.connectorTemplates[type].arguments); }, + onOAuth:function() + { + alert("hello"); + }, _onSubmit: function() { if (this.formWidget.validate()) { var values = this.formWidget.getValues(); diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js index a60e3ea2..fed1444f 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/main/UIRenderer.js @@ -26,17 +26,19 @@ define(["dojo/_base/declare", "dijit/form/Select", "dijit/form/ValidationTextBox", "dijit/form/CheckBox", + "dijit/form/TextBox", "dijit/form/TimeTextBox", "dijit/form/RadioButton", "dijit/form/NumberTextBox", "dijit/form/Textarea", "dijit/form/Form", + "dijit/form/Button", "dojox/html/entities", "hrv/utils/TextScrambler" ], function(declare,i18n, lang,array,domConstruct,domAttr,html,number, - Select,ValidationTextBox,CheckBox,TimeTextBox,RadioButton,NumberTextBox,Textarea,Form, + Select,ValidationTextBox,CheckBox,TextBox,TimeTextBox,RadioButton,NumberTextBox,Textarea,Form,Button, entities,TextScrambler ){ @@ -61,7 +63,8 @@ define(["dojo/_base/declare", renderArgument: function(rootNode,arg) { var argNode = domConstruct.create("div",{class: "h-editor-line"},rootNode); - var titleNode = domConstruct.create("span",{innerHTML: arg.label+":", class: "h-editor-argname"},argNode); + if(arg.label!=="hidden") + var titleNode = domConstruct.create("span",{innerHTML: arg.label+":", class: "h-editor-argname"},argNode); var placeholderWrapper = domConstruct.create("span",{class: "h-editor-argctrl"},argNode); var placeholderNode = domConstruct.create("span",null,placeholderWrapper); @@ -79,6 +82,8 @@ define(["dojo/_base/declare", case "temporal": return this.renderTime(placeholderNode,arg); case "periodical": return this.renderPeriod(placeholderNode,arg); case "integer": return this.renderInteger(placeholderNode, arg); + case "button": return this.renderButton(placeholderNode, arg); + case "hidden": return this.renderHidden(placeholderNode, arg); default: console.error("Unsupported argument type:", arg.type); return { @@ -126,16 +131,16 @@ define(["dojo/_base/declare", renderText: function(placeholderNode,arg) { var input = new Textarea({ name: arg.name, - required: arg.required + required: arg.required }).placeAt(placeholderNode); input.name = arg.name; - if (arg.defaultValue!=null) { + if (arg.defaultValue!==null) { input.set("value", arg.defaultValue); } input.startup(); return { init: function(values) { - input.set("value", values[arg.name]!=null? values[arg.name]: arg.defaultValue); + input.set("value", values[arg.name]!==null? values[arg.name]: arg.defaultValue); }, read: function(values) { values[arg.name] = input.get("value"); @@ -145,6 +150,47 @@ define(["dojo/_base/declare", } }; }, + + renderButton: function (placeholderNode, arg) { + var input = new Button({ + name: arg.name, + required: arg.required + }).placeAt(placeholderNode); + + input.set("label", arg.label); + input.startup(); + + return { + init: function (values) { + }, + read: function (values) { + }, + destroy: function () { + input.destroy(); + } + } + }, + renderHidden: function (placeholderNode, arg) { + var input = new TextBox({ + name: arg.name, + type: "hidden" + }); + + input.set("value", ""); + input.startup(); + + return { + init: function (values) { + input.set("value", Number(values[arg.name])); + }, + read: function (values) { + values[arg.name] = input.get("value"); + }, + destroy: function () { + input.destroy(); + } + }; + }, renderInteger: function (placeholderNode, arg) { var input = new NumberTextBox({ From d64f2a5be831848151fdeb13304bf1625248536b Mon Sep 17 00:00:00 2001 From: cont_anki Date: Fri, 3 Nov 2023 12:36:11 +0530 Subject: [PATCH 15/22] #204 Added oAuth Token, fixed cors error --- .../src/main/webapp/hrv/ui/brokers/Broker.js | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js index cda52818..29f3bf53 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/brokers/Broker.js @@ -29,9 +29,9 @@ define(["dojo/_base/declare", "dijit/ConfirmDialog", "hrv/rest/Brokers", "hrv/ui/brokers/BrokerEditorPane", - "esri/IdentityManager", - "esri/arcgis/OAuthInfo", - "esri/arcgis/Portal" + "esri/IdentityManager", + "esri/arcgis/Portal", + "esri/config", ], function(declare, _WidgetBase,_TemplatedMixin,_WidgetsInTemplateMixin, @@ -39,7 +39,7 @@ define(["dojo/_base/declare", lang,string,topic,on,json, Dialog,ConfirmDialog, BrokersREST,BrokerEditorPane, - esriId, OAuthInfo, arcgisPortal + esriId, arcgisPortal,esriConfig ){ return declare([_WidgetBase, _TemplatedMixin, _WidgetsInTemplateMixin],{ @@ -81,11 +81,8 @@ define(["dojo/_base/declare", if(brokerDefinitionProp["agp-oauth"]=== "true"){ esriId.getCredential(portalUrl,{oAuthPopupConfirmation:false}).then( - lang.hitch(this,function(){ - var portal = new arcgisPortal.Portal(portalUrl); - portal.signIn().then(function(portalUser){ - - var token = portalUser.credential.token; + lang.hitch(this,function(credential){ + var token = credential.token; brokerDefinitionProp["agp-token"]= token; // use API to update broker @@ -102,10 +99,7 @@ define(["dojo/_base/declare", topic.publish("msg", new Error(this.i18n.brokers.errors.creating)); }) ); - }).otherwise(function(error){ - console.warn("Error occurred while signing in:",error); - }); - })); + })); } else { From 91dc0e7568ac4b179a9f0e84aa03e0e6bbdda086 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Fri, 3 Nov 2023 12:38:24 +0530 Subject: [PATCH 16/22] #204 get featureLayer token using oAuthToken --- .../harvester/api/defs/UITemplate.java | 45 ++++++++++++++++++- .../commons/agp/client/AgpClient.java | 26 ++++++++++- .../geoportal/harvester/agp/AgpConstants.java | 2 + .../harvester/agp/AgpOutputBroker.java | 22 ++++++--- .../agp/AgpOutputBrokerDefinitionAdaptor.java | 12 +++++ .../harvester/agp/AgpOutputConnector.java | 5 ++- .../src/main/resources/AgpResource.properties | 5 ++- 7 files changed, 103 insertions(+), 14 deletions(-) diff --git a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java index c8343bdf..7ac4d169 100644 --- a/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java +++ b/geoportal-SDK/geoportal-harvester-api/src/main/java/com/esri/geoportal/harvester/api/defs/UITemplate.java @@ -81,7 +81,10 @@ public static enum ArgumentType { /** temporal type */ temporal, /** periodical */ - periodical + periodical, + button, + hidden + } /** @@ -224,7 +227,7 @@ public String toString() { } } - /** + /** * String argument. */ public static class StringArgument extends ArgumentBase { @@ -272,6 +275,44 @@ public ArgumentType getType() { } } + /** + * Button argument. + */ + public static class ButtonArgument extends ArgumentBase { + + /** + * Creates instance of the argument. + * @param name type + * @param label label + * @param required true if argument is required + */ + public ButtonArgument(String name, String label, boolean required) { + super(name, label, required); + } + @Override + public ArgumentType getType() { + return ArgumentType.button; + } + } + + /** + * Button argument. + */ + public static class HiddenArgument extends ArgumentBase { + /** + * Creates instance of the argument. + * @param name type + * @param label label + */ + public HiddenArgument(String name, String label) { + super(name, label); + } + @Override + public ArgumentType getType() { + return ArgumentType.hidden; + } + } + /** * String argument. */ diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java index 115582d9..c0acbffd 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java @@ -29,6 +29,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -316,7 +318,7 @@ public String writeItemMetadata(String itemId, String metadataXML, String token) * @throws URISyntaxException if invalid URL * @throws IOException if operation fails */ - public boolean writeSubLayerMetadata(String resourceURL, File fileToUpload, String token) throws IOException, URISyntaxException { + public boolean writeSubLayerMetadata(String resourceURL, String fileToUpload, String token) throws IOException, URISyntaxException { URIBuilder builder = new URIBuilder(resourceURL); HttpPost req = new HttpPost(builder.build()); @@ -325,9 +327,10 @@ public boolean writeSubLayerMetadata(String resourceURL, File fileToUpload, Stri params.put("token", token); params.put("metadataUploadFormat", "xml"); params.put("overwrite", "true"); + params.put("metadata",fileToUpload); try { - HttpEntity entity = createEntity(params, fileToUpload); + HttpEntity entity = createEntity(params); req.setEntity(entity); try (CloseableHttpResponse httpResponse = httpClient.execute(req); InputStream contentStream = httpResponse.getEntity().getContent();) { @@ -560,6 +563,25 @@ public TokenResponse generateToken(int minutes) throws URISyntaxException, IOExc return execute(req,TokenResponse.class); } + public TokenResponse generateToken(int minutes, String serverUrl,String token) throws URISyntaxException, IOException { + HttpPost req = new HttpPost(generateTokenUri()); + + HashMap params = new HashMap<>(); + params.put("f", "json"); + if (credentials != null) { + params.put("username", StringUtils.trimToEmpty(credentials.getUserName())); + params.put("password", StringUtils.trimToEmpty(credentials.getPassword())); + } + params.put("client", "requestip"); + params.put("expiration", Integer.toString(minutes)); + params.put("serverUrl", serverUrl); + params.put("token",token ); + + req.setEntity(createEntity(params)); + + return execute(req,TokenResponse.class); + } + private Map makeStdParams(String title, String description, ItemType itemType, URL thumbnailUrl, Double [] extent, String [] typeKeywords, String [] tags, String token) { HashMap params = new HashMap<>(); params.put("f", "json"); diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java index dc5beb09..1ce87676 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpConstants.java @@ -25,4 +25,6 @@ public static final String P_MAX_REDIRECTS = "agp-max-redirects"; public static final String P_UPLOAD = "agp-upload"; public static final String P_MARKDOWN2HTML = "agp-markdown2html"; + public static final String P_OAUTH = "agp-oauth"; + public static final String P_TOKEN = "agp-token"; } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 9121e350..41eb8731 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -198,6 +198,8 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } String sThumbnailUrl = StringUtils.trimToNull(getAttributeValue(attributes, WKAConstants.WKA_THUMBNAIL_URL, null)); String resourceUrl = getAttributeValue(attributes, WKAConstants.WKA_RESOURCE_URL, null); + + //resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; // clean up resource URL resourceUrl = resourceUrl.replace("http:", "https:") .replace(":80/", "/"); @@ -279,11 +281,15 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { try { - // generate token + // generate token if (token == null) { - token = generateToken(); + token = definition.getOAuthToken(); + //oAuth token was not generated, hence generate token from user password. In this case subLayer metadata update will fail + if(token == null || token.isBlank()) + { + token = generateToken(60); + } } - // check if item exists ItemEntry itemEntry = searchForItem(resourceUrl); @@ -412,11 +418,11 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { return PublishingStatus.UPDATED; } else { - // the metadata is apparently for a sublayer - + // the metadata is apparently for a sublayer // DO SOMETHING ELSE + String featureServerToken = generateToken(60,"https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/admin/services/group_of_layers/FeatureServer",token); String metadataUpdateURI = resourceUrl + "/metadata/update/"; - boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, metadataFile, token); + boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); return PublishingStatus.SKIPPED; @@ -715,6 +721,10 @@ private DeleteResponse deleteItem(String id, String owner, String folderId, Stri private String generateToken(int minutes) throws URISyntaxException, IOException { return client.generateToken(minutes).token; } + private String generateToken(int minutes,String serverUrl,String token) throws URISyntaxException, IOException { + return client.generateToken(minutes,serverUrl,token).token; + } + private String generateToken() throws URISyntaxException, IOException { return client.generateToken(60).token; diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java index 680a7050..189afd69 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java @@ -42,6 +42,7 @@ private Integer maxRedirects; private boolean uploadFiles; private boolean markdown2html; + private String oAuthToken; /** @@ -71,6 +72,7 @@ public AgpOutputBrokerDefinitionAdaptor(EntityDefinition def) throws InvalidDefi maxRedirects = NumberUtils.toInt(get(P_MAX_REDIRECTS), DEFAULT_MAX_REDIRECTS); uploadFiles = Boolean.parseBoolean(get(P_UPLOAD)); markdown2html = Boolean.parseBoolean(get(P_MARKDOWN2HTML)); + oAuthToken = get(P_TOKEN); } } @@ -82,6 +84,7 @@ public void override(Map params) { consume(params, P_MAX_REDIRECTS); consume(params, P_UPLOAD); consume(params, P_MARKDOWN2HTML); + consume(params, P_TOKEN); credAdaptor.override(params); } @@ -194,4 +197,13 @@ public void setMarkdown2HTML(boolean markdown2html) { this.markdown2html = markdown2html; set(P_MARKDOWN2HTML, Boolean.toString(markdown2html)); } + + public String getOAuthToken() + { + return oAuthToken; + } + public void setOAuthToken(String oAuthToken) { + this.oAuthToken = oAuthToken; + set(P_TOKEN, oAuthToken); + } } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java index e2e541fe..a74c4592 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputConnector.java @@ -87,7 +87,7 @@ public String getHint() { return bundle.getString("agp.hint"); } }); - args.add(new UITemplate.StringArgument(P_FOLDER_ID, bundle.getString("agp.folderId"), false)); + args.add(new UITemplate.StringArgument(P_FOLDER_ID, bundle.getString("agp.folderId"), false)); args.add(new UITemplate.StringArgument(P_CRED_USERNAME, bundle.getString("agp.username"), true)); args.add(new UITemplate.StringArgument(P_CRED_PASSWORD, bundle.getString("agp.password"), true) { public boolean isPassword() { @@ -98,7 +98,8 @@ public boolean isPassword() { args.add(new UITemplate.BooleanArgument(P_FOLDER_CLEANUP, bundle.getString("agp.cleanup"))); args.add(new UITemplate.BooleanArgument(P_UPLOAD, bundle.getString("agp.upload"), true, true)); args.add(new UITemplate.BooleanArgument(P_MARKDOWN2HTML, bundle.getString("agp.markdown2html"), true, true)); - + args.add(new UITemplate.BooleanArgument(P_OAUTH, bundle.getString("agp.oauth"), false,false)); + args.add(new UITemplate.HiddenArgument(P_TOKEN, "hidden")); return new UITemplate(getType(), bundle.getString("agp"), args); } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties index c025d790..53e9043b 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/resources/AgpResource.properties @@ -16,9 +16,10 @@ agp = Portal for ArcGIS agp.url = URL agp.folderId = Folder agp.username = User name -agp.password = user password +agp.password = User password agp.cleanup = Perform cleanup agp.max.redirects = Maximum redirects agp.hint = https://www.arcgis.com agp.upload = Upload files -agp.markdown2html = Markdown to HTML \ No newline at end of file +agp.markdown2html = Markdown to HTML +agp.oauth = Sign In to AGOL \ No newline at end of file From 98c2baadd839f5ab31b88260ccb9f14147ab9f53 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Mon, 20 Nov 2023 12:48:53 +0530 Subject: [PATCH 17/22] #204 fix status update --- .../com/esri/geoportal/harvester/agp/AgpOutputBroker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 41eb8731..5ac50705 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -199,7 +199,7 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { String sThumbnailUrl = StringUtils.trimToNull(getAttributeValue(attributes, WKAConstants.WKA_THUMBNAIL_URL, null)); String resourceUrl = getAttributeValue(attributes, WKAConstants.WKA_RESOURCE_URL, null); - //resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; + resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; // clean up resource URL resourceUrl = resourceUrl.replace("http:", "https:") .replace(":80/", "/"); @@ -425,7 +425,7 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); - return PublishingStatus.SKIPPED; + return PublishingStatus.UPDATED; } } } catch (MalformedURLException ex) { From 2e44b95b41591eedc064431bc5bd4c0def213db3 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Mon, 20 Nov 2023 12:51:14 +0530 Subject: [PATCH 18/22] #204 Comment test url --- .../java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 5ac50705..c10a429c 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -199,7 +199,8 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { String sThumbnailUrl = StringUtils.trimToNull(getAttributeValue(attributes, WKAConstants.WKA_THUMBNAIL_URL, null)); String resourceUrl = getAttributeValue(attributes, WKAConstants.WKA_RESOURCE_URL, null); - resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; + //Hardcoded url just for dev testing + //resourceUrl="https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/services/group_of_layers/FeatureServer/1"; // clean up resource URL resourceUrl = resourceUrl.replace("http:", "https:") .replace(":80/", "/"); From 7cee1e60a494d7a468f112e3ef18662dc4895049 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Mon, 20 Nov 2023 09:48:41 -0800 Subject: [PATCH 19/22] layer harvest don't hard code test url. added 'useOAuth' to broker definition, set updated/skipped when done --- .../geoportal/harvester/agp/AgpOutputBroker.java | 13 +++++++++---- .../agp/AgpOutputBrokerDefinitionAdaptor.java | 6 ++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index c10a429c..96288ec5 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -283,7 +283,7 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { try { // generate token - if (token == null) { + if ((definition.useOAuth()) || (token == null)) { token = definition.getOAuthToken(); //oAuth token was not generated, hence generate token from user password. In this case subLayer metadata update will fail if(token == null || token.isBlank()) @@ -421,12 +421,17 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } else { // the metadata is apparently for a sublayer // DO SOMETHING ELSE - String featureServerToken = generateToken(60,"https://services.arcgis.com/RhGiohBHzSBKt1MS/arcgis/rest/admin/services/group_of_layers/FeatureServer",token); + String parentUrl = resourceUrl.substring(0,resourceUrl.lastIndexOf("/")); + String featureServerToken = generateToken(60, parentUrl,token); String metadataUpdateURI = resourceUrl + "/metadata/update/"; boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); - - return PublishingStatus.UPDATED; + + if (wasUpdated) { + return PublishingStatus.UPDATED; + } else { + return PublishingStatus.SKIPPED; + } } } } catch (MalformedURLException ex) { diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java index 189afd69..eb3ef6d3 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBrokerDefinitionAdaptor.java @@ -42,6 +42,7 @@ private Integer maxRedirects; private boolean uploadFiles; private boolean markdown2html; + private boolean useOAuth; private String oAuthToken; @@ -72,6 +73,7 @@ public AgpOutputBrokerDefinitionAdaptor(EntityDefinition def) throws InvalidDefi maxRedirects = NumberUtils.toInt(get(P_MAX_REDIRECTS), DEFAULT_MAX_REDIRECTS); uploadFiles = Boolean.parseBoolean(get(P_UPLOAD)); markdown2html = Boolean.parseBoolean(get(P_MARKDOWN2HTML)); + useOAuth = Boolean.parseBoolean(get(P_OAUTH)); oAuthToken = get(P_TOKEN); } } @@ -198,6 +200,10 @@ public void setMarkdown2HTML(boolean markdown2html) { set(P_MARKDOWN2HTML, Boolean.toString(markdown2html)); } + public boolean useOAuth() { + return useOAuth; + } + public String getOAuthToken() { return oAuthToken; From b08337044f9e417369212e866c64cfe6e806bfe8 Mon Sep 17 00:00:00 2001 From: cont_anki Date: Thu, 30 Nov 2023 16:22:47 +0530 Subject: [PATCH 20/22] Changed system.out to Log statements --- .../com/esri/geoportal/commons/agp/client/AgpClient.java | 6 +++--- .../com/esri/geoportal/harvester/agp/AgpOutputBroker.java | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java index c0acbffd..95b1afb6 100644 --- a/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java +++ b/geoportal-commons/geoportal-commons-agp-client/src/main/java/com/esri/geoportal/commons/agp/client/AgpClient.java @@ -338,14 +338,14 @@ public boolean writeSubLayerMetadata(String resourceURL, String fileToUpload, St throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } String responseContent = IOUtils.toString(contentStream, "UTF-8"); - System.out.println(responseContent); + LOG.debug(" writeSubLayerMetadata "+responseContent); if (responseContent.contains("error")) { return false; } } } catch (IOException ex) { - System.out.println(ex.getMessage()); + LOG.error("Error writeSubLayerMetadata " +ex.getMessage(),ex); return false; } @@ -770,7 +770,7 @@ private String execute(HttpUriRequest req, Integer redirectDepth) throws IOExcep return execute(newReq, ++redirectDepth); } catch (IOException | URISyntaxException e) { - LOG.debug("Error executing request", e); + LOG.error("Error executing request", e); throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } } diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 96288ec5..379b4021 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -425,6 +425,7 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { String featureServerToken = generateToken(60, parentUrl,token); String metadataUpdateURI = resourceUrl + "/metadata/update/"; boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); + LOG.debug("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); if (wasUpdated) { From 1d916d5e61f5c7739b11b088697be15217a848e7 Mon Sep 17 00:00:00 2001 From: Marten Hogeweg Date: Thu, 7 Dec 2023 16:54:43 -0800 Subject: [PATCH 21/22] update AgpOutputBroker as sub-layer harvest was failing --- .../harvester/agp/AgpOutputBroker.java | 128 +++++++----------- 1 file changed, 49 insertions(+), 79 deletions(-) diff --git a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java index 96288ec5..7c9aa7b6 100644 --- a/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java +++ b/geoportal-connectors/geoportal-harvester-agp-publisher/src/main/java/com/esri/geoportal/harvester/agp/AgpOutputBroker.java @@ -295,77 +295,43 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { ItemEntry itemEntry = searchForItem(resourceUrl); if (itemEntry == null) { - // add item if doesn't exist - // add item with dummy 'simple' metadata - // no longer needed, but keep for testing purposes - /* - String dummyMetadata = "" - + "" - + " " - + " 2022-07-21" - + " 12:26:34.68" - + " 2022-07-21" - + " 12:59:19.59" - + " editor:esri.dijit.metadata.editor" - + " 1.0" - + " ISO 19139 Metadata Implementation Specification GML3.2" - + " ISO19139" - + " false" - + " " - + " " - + " " + resourceUrl + "" - + " 2016-02-1916:48:06.84" - + " " - + " " - + " " + title + "" - + " " - + " " + sanitize(description) + "" - + " " - + ""; - - Path dummyMetadataPath = Files.createTempFile(null, null); - Files.write(dummyMetadataPath, dummyMetadata.getBytes(StandardCharsets.UTF_8)); - File dummyMetadataFile = dummyMetadataPath.toFile(); - */ - - ItemResponse response = addItem( - title, - description, - new URL(resourceUrl), - sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, - itemType, - extractEnvelope(bbox), - typeKeywords, - null, - metadataFile, - token - ); - - // remove the dummy metadata file - // no longer needed, but keep for testing purposes - // dummyMetadataFile.delete(); - - if (response == null || !response.success) { - String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; - throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); - } else { - System.out.print("addItem -> " + response.toString()); - - // Now upload full metadata - String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); - System.out.print("METADATA -> " + metadataAdded); - } - - client.share(definition.getCredentials().getUserName(), definition.getFolderId(), response.id, true, true, null, token); + // neither the potential sub layer, nor the parent layer exist. Add a new portal item + + ItemResponse response = addItem( + title, + description, + new URL(resourceUrl), + sThumbnailUrl != null ? new URL(sThumbnailUrl) : null, + itemType, + extractEnvelope(bbox), + typeKeywords, + null, + metadataFile, + token + ); - return PublishingStatus.CREATED; + if (response == null || !response.success) { + String error = response != null && response.error != null && response.error.message != null ? response.error.message : null; + throw new DataOutputException(this, ref, String.format("Error adding item: %s%s", ref, error != null ? "; " + error : "")); + } else { + System.out.print("addItem -> " + response.toString()); + + // Now upload full metadata + String metadataAdded = client.writeItemMetadata(response.id, arcgisMetadata, token); + System.out.print("METADATA -> " + metadataAdded); + } - } else { // if (itemEntry.owner.equals(definition.getCredentials().getUserName())) { - // if the item is not owned by the registered account, try to update - // assuming the item is in a shared update group. + client.share(definition.getCredentials().getUserName(), definition.getFolderId(), response.id, true, true, null, token); + + return PublishingStatus.CREATED; + + } else { + // there is an item registered for this resourceUrl, try to update + // if the item is not owned by the registered account + // assume the item is in a shared update group. // if this fails and the registered account cannot update the existing item - // then consider this item 'skipped' + // then this item will be 'skipped' itemEntry = client.readItem(itemEntry.id, token); if (itemEntry == null) { @@ -420,18 +386,7 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { return PublishingStatus.UPDATED; } else { // the metadata is apparently for a sublayer - // DO SOMETHING ELSE - String parentUrl = resourceUrl.substring(0,resourceUrl.lastIndexOf("/")); - String featureServerToken = generateToken(60, parentUrl,token); - String metadataUpdateURI = resourceUrl + "/metadata/update/"; - boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); - System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); - - if (wasUpdated) { - return PublishingStatus.UPDATED; - } else { - return PublishingStatus.SKIPPED; - } + return updateSubLayerMetadata(resourceUrl, arcgisMetadata); } } } catch (MalformedURLException ex) { @@ -451,6 +406,21 @@ public PublishingStatus publish(DataReference ref) throws DataOutputException { } + private PublishingStatus updateSubLayerMetadata(String resourceUrl, String arcgisMetadata) throws URISyntaxException, IOException { + String parentUrl = resourceUrl.substring(0,resourceUrl.lastIndexOf("/")); + String featureServerToken = generateToken(60, parentUrl,token); + String metadataUpdateURI = resourceUrl + "/metadata/update/"; + boolean wasUpdated = client.writeSubLayerMetadata(metadataUpdateURI, arcgisMetadata, featureServerToken); + System.out.println("update metadata at " + metadataUpdateURI + " was a succes: " + wasUpdated); + + if (wasUpdated) { + return PublishingStatus.UPDATED; + } else { + return PublishingStatus.SKIPPED; + } + } + + private ItemType createItemType(String resourceUrl) { resourceUrl = StringUtils.trimToEmpty(resourceUrl); From 06f75003c5e61271e2e40c098f951c204b43a02b Mon Sep 17 00:00:00 2001 From: cont_anki Date: Tue, 12 Dec 2023 11:03:00 +0530 Subject: [PATCH 22/22] Fixed security updates --- geoportal-application/geoportal-harvester-war/pom.xml | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/geoportal-application/geoportal-harvester-war/pom.xml b/geoportal-application/geoportal-harvester-war/pom.xml index 59aea817..00aa9474 100644 --- a/geoportal-application/geoportal-harvester-war/pom.xml +++ b/geoportal-application/geoportal-harvester-war/pom.xml @@ -13,7 +13,7 @@ ${project.build.directory}/endorsed - 5.3.24 + 5.3.27 5.3.10.RELEASE @@ -67,7 +67,7 @@ com.h2database h2 - 2.1.210 + 2.2.220 diff --git a/pom.xml b/pom.xml index c8eed2da..80be9e70 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.4.1 + 2.13.4.2 com.esri.geometry