Skip to content

Commit

Permalink
Merge pull request #168 from integratedmodelling/IM-359-Digital-earth…
Browse files Browse the repository at this point in the history 
…-resource-from-STAC-catalogue-cannot-be-imported

Im 359 digital earth resource from stac catalogue cannot be imported
  • Loading branch information
@inigo-cobian
inigo-cobian committed May 14, 2024
2 parents a065961 + cfc40a8 commit 13e94ab
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 17 deletions.
14 changes: 11 additions & 3 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package org.integratedmodelling.klab.stac;

import org.integratedmodelling.klab.exceptions.KlabResourceAccessException;

import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.Unirest;
import kong.unirest.json.JSONObject;

Expand All @@ -21,8 +25,9 @@ private static JSONObject readAssets(JSONObject items) {
* @param catalogUrl endpoint of the catalog
* @param collectionId id of the collection
* @return The asset list as a JSON
* @throws KlabResourceAccessException
*/
public static JSONObject readAssets(String catalogUrl, String collectionId) {
public static JSONObject readAssets(String catalogUrl, String collectionId) throws KlabResourceAccessException {
JSONObject assets;
JSONObject collectionData = Unirest.get(catalogUrl + "/collections/" + collectionId)
.asJson().getBody().getObject();
Expand All @@ -32,8 +37,11 @@ public static JSONObject readAssets(String catalogUrl, String collectionId) {
if (collectionData.has("item_assets")) {
assets = STACCollectionParser.readItemAssets(collectionData);
} else {
JSONObject itemsData = Unirest.get(catalogUrl + "/collections/" + collectionId + "/items")
.asJson().getBody().getObject();
HttpResponse<JsonNode> response = Unirest.get(catalogUrl + "/collections/" + collectionId + "/items").asJson();
if (!response.isSuccess()) {
throw new KlabResourceAccessException("Cannot read items at " + catalogUrl + "/collections/" + collectionId + "/items");
}
JSONObject itemsData = response.getBody().getObject();
assets = STACCollectionParser.readAssets(itemsData);
}
return assets;
Expand Down
32 changes: 21 additions & 11 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.integratedmodelling.klab.components.time.extents.TimeInstant;
import org.integratedmodelling.klab.exceptions.KlabContextualizationException;
import org.integratedmodelling.klab.exceptions.KlabIllegalStateException;
import org.integratedmodelling.klab.exceptions.KlabInternalErrorException;
import org.integratedmodelling.klab.ogc.STACAdapter;
import org.integratedmodelling.klab.raster.files.RasterEncoder;
import org.integratedmodelling.klab.scale.Scale;
Expand Down Expand Up @@ -98,26 +99,36 @@ private Time refitTime(Time contextTime, Time resourceTime) {
throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted.");
}

private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics) {
private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor monitor) {
if (targetSemantics == null) {
monitor.debug("Using average as merge mode");
return HMRaster.MergeMode.AVG;
}
switch(targetSemantics.getArtifactType()) {
case CONCEPT:
case BOOLEAN:
monitor.debug("Using substitute as merge mode");
return HMRaster.MergeMode.SUBSTITUTE;
case NUMBER:
return Observables.INSTANCE.isExtensive(targetSemantics) ? HMRaster.MergeMode.SUM : HMRaster.MergeMode.SUBSTITUTE;
if (Observables.INSTANCE.isExtensive(targetSemantics)) {
monitor.debug("Using sum as merge mode");
return HMRaster.MergeMode.SUM;
}
monitor.debug("Using substitute as merge mode");
return HMRaster.MergeMode.SUBSTITUTE;
default:
monitor.debug("Defaulting to average as merge mode");
return HMRaster.MergeMode.AVG;
}
}

private void sortByDate(List<HMStacItem> items) {
private void sortByDate(List<HMStacItem> items, IMonitor monitor) {
if (items.stream().anyMatch(i -> i.getTimestamp() == null)) {
throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date.");
}
items.sort((i1, i2) -> i1.getTimestamp().compareTo(i2.getTimestamp()));
monitor.debug(
"Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + items.get(items.size() - 1).getTimestamp() + "]");
}

@Override
Expand All @@ -126,7 +137,7 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation
? ((Observation) scope.getTargetArtifact()).getObservable()
: null;
HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics);
HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor());

String catalogUrl = resource.getParameters().get("catalogUrl", String.class);
String collectionId = resource.getParameters().get("collectionId", String.class);
Expand Down Expand Up @@ -168,14 +179,14 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
List<HMStacItem> items = collection.setGeometryFilter(poly)
.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds()))
.searchItems();

if (mergeMode == HMRaster.MergeMode.SUBSTITUTE) {
sortByDate(items);
}

if (items.isEmpty()) {
throw new KlabIllegalStateException("No STAC items found for this context.");
}
scope.getMonitor().debug("Found " + items.size() + " STAC items.");

if (mergeMode == HMRaster.MergeMode.SUBSTITUTE) {
sortByDate(items, scope.getMonitor());
}

LogProgressMonitor lpm = new LogProgressMonitor();
IGrid grid = space.getGrid();
Expand All @@ -199,9 +210,8 @@ public void getEncodedData(IResource resource, Map<String, String> urnParameters
String assetId = resource.getParameters().get("asset", String.class);
HMRaster outRaster = HMStacCollection.readRasterBandOnRegion(regionTransformed, assetId, items, allowTransform, mergeMode, lpm);
coverage = outRaster.buildCoverage();
scope.getMonitor().info("Coverage: " + coverage);
} catch (Exception e) {
scope.getMonitor().error("Cannot create STAC file. " + e.getMessage());
throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage());
}

encoder.encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope);
Expand Down
6 changes: 4 additions & 2 deletions adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.integratedmodelling.klab.api.observations.IObservation;
import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor;
import org.integratedmodelling.klab.exceptions.KlabIOException;
import org.integratedmodelling.klab.exceptions.KlabUnsupportedFeatureException;
import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException;
import org.integratedmodelling.klab.utils.Parameters;
import org.integratedmodelling.klab.utils.Triple;

Expand Down Expand Up @@ -92,7 +92,9 @@ public Collection<Builder> importResources(String importLocation, IProject proje
String[] locationElements = STACUtils.extractCatalogAndCollection(importLocation);

if (locationElements.length != 2) {
throw new KlabUnsupportedFeatureException("Bulk import from a catalog is not supported.");
monitor.error("It is not possible to bulk import form the URL " + importLocation + "."
+ "Check if the resource is a proper STAC collection.");
throw new KlabIllegalArgumentException("Unexpected STAC import location.");
}
try {
monitor.info("Beginning STAC collection import from " + importLocation);
Expand Down
3 changes: 2 additions & 1 deletion ...s/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACImporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.integratedmodelling.klab.api.knowledge.IProject;
import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor;
import org.integratedmodelling.klab.exceptions.KlabIOException;
import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException;
import org.integratedmodelling.klab.exceptions.KlabUnsupportedFeatureException;
import org.integratedmodelling.klab.stac.STACImporter;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -55,7 +56,7 @@ public void importResource_failCannotImportCatalog() {
Mockito.when(identity.getId()).thenReturn("hares");
Authentication.INSTANCE.registerIdentity(identity);

Assertions.assertThrows(KlabUnsupportedFeatureException.class, () -> {
Assertions.assertThrows(KlabIllegalArgumentException.class, () -> {
Collection<Builder> ret = importer.importResources(importLocation, project, params, monitor);
});
}
Expand Down

0 comments on commit 13e94ab

Please sign in to comment.