diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java index 3be8c4a501..7dacca18b6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java @@ -10,6 +10,11 @@ public static Set readAssetNames(JSONObject assets) { } public static JSONObject getAsset(JSONObject assetMap, String assetId) { - return assetMap.getJSONObject(assetId); + /* + * Not failing if the assetId is not found, the reason being that in + * some collections the items have different assets, and the Asset naming related conventions that MSFT + * Planetary follows is not necessarily followed by evryone! + */ + return assetMap.optJSONObject(assetId); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index c351c324c1..6dd160b9f0 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -1,8 +1,8 @@ package org.integratedmodelling.klab.stac; import java.time.Instant; -import java.util.List; -import java.util.Optional; +import java.util.*; +import java.util.function.Predicate; import org.integratedmodelling.klab.api.data.IGeometry; import org.integratedmodelling.klab.common.Geometry; @@ -11,11 +11,10 @@ import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; -import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema.Items; - import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; public class STACCollectionParser { @@ -35,7 +34,7 @@ public static String readCollectionId(JSONObject collection) { */ public static IGeometry readGeometry(JSONObject collection) { GeometryBuilder gBuilder = Geometry.builder(); - + JSONObject extent = collection.getJSONObject("extent"); List bbox = extent.getJSONObject("spatial").getJSONArray("bbox").getJSONArray(0).toList(); gBuilder.space().boundingBox(Double.valueOf(bbox.get(0).toString()), Double.valueOf(bbox.get(1).toString()), @@ -56,18 +55,46 @@ public static IGeometry readGeometry(JSONObject collection) { return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid"); } - /** - * Reads the assets of a STAC collection and returns them as a JSON. - * @param collection as a JSON - * @return The asset list as a JSON - * @throws KlabResourceAccessException - */ - public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObject collection) throws KlabResourceAccessException { + private static JSONObject findAsset(JSONObject assets, String assetId, Predicate predicate) { + if (assets == null) { + return null; + } + + if (assetId != null) { + JSONObject asset = assets.optJSONObject(assetId); + if (asset == null) { + return null; + } + JSONObject result = new JSONObject(); + result.put(assetId, asset); + return result; + } + + if (predicate != null) { + return assets.keySet().stream().map(key -> { + JSONObject asset = assets.optJSONObject(key); + if (asset == null || !predicate.test(asset)) { + return null; + } + JSONObject result = new JSONObject(); + result.put(key, asset); + return result; + }).filter(Objects::nonNull).findFirst().orElse(null); + } + + return null; + } + + private static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId, + Predicate predicate) throws KlabResourceAccessException { + if ((assetId == null) == (predicate == null)) { + throw new KlabResourceAccessException("Exactly one of assetId or predicate must be provided"); + } String collectionId = collection.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") + Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") ? STACUtils.getLinkTo(catalogData, "search") : STACUtils.getLinkTo(collection, "search"); @@ -75,7 +102,10 @@ public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObje if (searchEndpoint.isEmpty()) { // Check the assets if (collection.has("assets")) { - return collection.getJSONObject("assets"); + JSONObject assetInfo = findAsset(collection.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } } // Try to get the assets from a link that has type `item` Optional itemHref = STACUtils.getLinkTo(collection, "item"); @@ -85,27 +115,74 @@ public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObje String itemUrl = itemHref.get().startsWith(".") ? collectionUrl.replace("collection.json", "") + itemHref.get().replace("./", "") : itemHref.get(); - // TODO get assets from the item JSONObject itemData = STACUtils.requestMetadata(itemUrl, "feature"); - if (itemData.has("assets")) { - return itemData.getJSONObject("assets"); + JSONObject assetInfo = findAsset(itemData.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; } throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); } - // TODO Move the query to another place. - String parameters = "?collections=" + collectionId + "&limit=1"; - HttpResponse response = Unirest.get(searchEndpoint.get() + parameters).asJson(); + JSONObject searchPayload = new JSONObject().put("limit", 100) + .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) + .put("collections", new JSONArray().put(collectionId)); + + HttpResponse response = Unirest.post(searchEndpoint.get()).header("Content-Type", "application/json") + .body(searchPayload).asJson(); if (!response.isSuccess()) { - throw new KlabResourceAccessException(); //TODO set message + throw new KlabResourceAccessException("Unable to import collection, Search failed"); } JSONObject searchResponse = response.getBody().getObject(); - if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException(); // TODO set message there is no feature + JSONArray features = searchResponse.optJSONArray("features"); + if (features == null || features.length() == 0) { + throw new KlabResourceAccessException("No features were found in the collection to be imported"); } - return searchResponse.getJSONArray("features").getJSONObject(0).getJSONObject("assets"); + for(int i = 0; i < features.length(); i++) { + JSONObject feature = features.optJSONObject(i); + if (feature == null) { + continue; + } + + JSONObject assetInfo = findAsset(feature.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } + } + if (assetId != null) { + throw new KlabResourceAccessException("No asset with ID \"" + assetId + "\" was found in the collection"); + } + + throw new KlabResourceAccessException("No asset matching the predicate was found in the collection"); + } + + /** + * Reads an asset from a STAC collection by asset key. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param assetId asset key to find inside each feature's assets object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) + throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, assetId, null); + } + + /** + * Reads the first asset in a STAC collection whose asset JSON object matches the predicate. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param predicate predicate evaluated against each asset JSON object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, + Predicate predicate) throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, null, predicate); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9c331a850a..0570919d6f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -2,32 +2,40 @@ import java.io.IOException; import java.io.OutputStream; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Date; -import java.util.function.Predicate; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; -import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.api.data.FeatureSource; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.processing.Operations; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.libs.modules.HMRaster.HMRasterWritableBuilder; import org.hortonmachine.gears.libs.modules.HMRaster.MergeMode; import org.hortonmachine.gears.libs.monitor.LogProgressMonitor; -import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.klab.Authentication; import org.integratedmodelling.klab.Observables; import org.integratedmodelling.klab.api.data.IGeometry; -import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.IGeometry.Dimension.Type; +import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder; import org.integratedmodelling.klab.api.data.adapters.IResourceEncoder; import org.integratedmodelling.klab.api.knowledge.ICodelist; @@ -46,10 +54,10 @@ import org.integratedmodelling.klab.components.time.extents.TimeInstant; import org.integratedmodelling.klab.exceptions.KlabContextualizationException; import org.integratedmodelling.klab.exceptions.KlabIOException; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabIllegalStateException; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; -import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.ogc.STACAdapter; import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; @@ -57,24 +65,16 @@ import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; +import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; -import org.geotools.coverage.processing.Operations; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.geotools.api.feature.simple.SimpleFeature; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.geometry.Position; -import org.geotools.referencing.CRS; -import org.geotools.api.referencing.FactoryException; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; -import java.time.*; -import java.time.format.DateTimeFormatter; import kong.unirest.json.JSONObject; @@ -117,7 +117,8 @@ public ICodelist categorize(IResource resource, String attribute, IMonitor monit private Time refitTime(Time contextTime, Time resourceTime) { if (resourceTime.getCoveredExtent() < contextTime.getCoveredExtent()) { - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } if (contextTime.getStart().isBefore(resourceTime.getStart())) { ITimeInstant newEnd = TimeInstant.create(resourceTime.getStart().getMilliseconds() + contextTime.getLength()); @@ -127,7 +128,8 @@ private Time refitTime(Time contextTime, Time resourceTime) { ITimeInstant newStart = TimeInstant.create(resourceTime.getEnd().getMilliseconds() - contextTime.getLength()); return Time.create(newStart.getMilliseconds(), resourceTime.getEnd().getMilliseconds()); } - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } /** @@ -157,7 +159,7 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor case NUMBER: if (Observables.INSTANCE.isExtensive(targetSemantics)) { monitor.debug("Using sum as merge mode"); - return HMRaster.MergeMode.SUM; + return HMRaster.MergeMode.SUM; } monitor.debug("Using substitute as merge mode"); return HMRaster.MergeMode.SUBSTITUTE; @@ -167,13 +169,16 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor } } + /* + Helper to Sort Items (of type HMStacItem) based on their timestamp + */ private void sortByDate(List items, IMonitor monitor) { if (items.stream().anyMatch(i -> i.getTimestamp() == null)) { throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date."); } items.sort((i1, i2) -> i1.getTimestamp().compareTo(i2.getTimestamp())); - monitor.debug( - "Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + items.get(items.size() - 1).getTimestamp() + "]"); + monitor.debug("Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + + items.get(items.size() - 1).getTimestamp() + "]"); } private Client buildS3Client(String bucketRegion) throws IOException { @@ -184,15 +189,13 @@ private Client buildS3Client(String bucketRegion) throws IOException { } catch (Exception e) { throw new KlabIOException("Error defining S3 credenetials. " + e.getMessage()); } - return Client.s3() - .regionFromEnvironment() // TODO get region from other sources if needed - .credentials(credentials) - .build(); + return Client.s3().regionFromEnvironment() // TODO get region from other sources if needed + .credentials(credentials).build(); } private boolean isDateWithinRange(Time rangeTime, Date date) { Date start = new Date(rangeTime.getStart().getMilliseconds()); - Date end = new Date(rangeTime.getEnd().getMilliseconds()); + Date end = new Date(rangeTime.getEnd().getMilliseconds()); return date.after(start) && date.before(end); } @@ -201,158 +204,183 @@ public void getEncodedData(IResource resource, Map urnParameters IContextualizationScope scope) { String COGURL = null; - Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space) - .findFirst().orElseThrow(); + Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space).findFirst().orElseThrow(); IEnvelope envelope = space.getEnvelope(); List bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + // Only for Backward Compatiability + // A new COG Adapter would be added if (resource.getParameters().get("cog") != null) { COGURL = resource.getParameters().get("cog", String.class); scope.getMonitor().info("Getting requested extent from the COG Asset from url" + COGURL); GridCoverage2D coverage = COGAssetExtension.getCOGWindowCoverage(bbox, COGURL); - - String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters().get( - org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, - String.class); - - Projection crs = Projection.create(rcrs); - org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), - bbox.get(1), bbox.get(2), bbox.get(3)); - - HMRaster raster = HMRaster.fromGridCoverage(coverage); - HMRaster outRaster = new HMRasterWritableBuilder().setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, - (int) space.shape()[0], - (int) space.shape()[1])).setCrs(crs.getCoordinateReferenceSystem()) - .setNoValue(raster.getNovalue()) - .build(); - - GridCoverage2D adjCoverage = null; - try { - outRaster.mapRaster(null, raster, null); - adjCoverage = outRaster.buildCoverage(); - } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); - } - - encoder = new RasterEncoder(); + + String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters() + .get(org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, String.class); + + Projection crs = Projection.create(rcrs); + org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), bbox.get(1), + bbox.get(2), bbox.get(3)); + + HMRaster raster = HMRaster.fromGridCoverage(coverage); + HMRaster outRaster = new HMRasterWritableBuilder() + .setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, (int) space.shape()[0], (int) space.shape()[1])) + .setCrs(crs.getCoordinateReferenceSystem()).setNoValue(raster.getNovalue()).build(); + + GridCoverage2D adjCoverage = null; + try { + outRaster.mapRaster(null, raster, null); + adjCoverage = outRaster.buildCoverage(); + } catch (Exception e) { + throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); + } + + encoder = new RasterEncoder(); ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, adjCoverage, geometry, builder, scope); return; } - + String collectionUrl = resource.getParameters().get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = collectionData.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - String assetId = resource.getParameters().get("asset", String.class); Integer bandIndex = resource.getParameters().get("band", Integer.class); - + String assetId = resource.getParameters().get("asset", String.class); boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); - // This is part of a WIP that will be removed in the future - boolean isIIASA = catalogUrl.contains("iiasa.blob"); - - Time time = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time) - .findFirst().orElseThrow(); + final boolean allowTransform = true; + Time ctxTime = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow(); Time resourceTime = (Time) Scale.create(resource.getGeometry()).getDimension(Type.TIME); - if (isIIASA) { + Time effectiveTime = ctxTime; + if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null + && resourceTime.getCoveredExtent() > 0) { + + effectiveTime = validateTemporalDimension(ctxTime, resourceTime); + } + + // This is part of a WIP that will be removed in the future + if (catalogUrl.contains("iiasa.blob")) { FeatureSource source; try { source = STACIIASAExtension.getFeatures(collectionData, bbox); } catch (IOException e) { - throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); + throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); } encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } - // These are the static STAC catalogs + /* + Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue + */ + + Predicate assetPredicate = null; + if (assetId != null) { + assetPredicate = new Predicate(){ + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would + // be there, adding support for customised + // predicates + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + for(var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this + // case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } + return false; + } + }; + } else if (resource.getParameters().get("jsonSelector", String.class) != null) { + // based on the JSON Expression on JSONSelector and JSONValue + try { + assetPredicate = getAssetPredicate(resource); + } catch (Exception e) { + throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); + } + + } + + // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); - Time time2 = time; //TODO make the time and query time different features = features.stream().filter(f -> { Geometry fGeometry = (Geometry) f.getDefaultGeometry(); return fGeometry.intersects(space.getShape().getJTSGeometry()); }).toList(); - features = features.stream().filter(f -> isFeatureInTimeRange(time2, f)).toList(); - if (features.isEmpty()) { - throw new KlabResourceNotFoundException("There are no items in this context for the collection " + collectionId); - } CoordinateReferenceSystem crs = features.get(0).getFeatureType().getCoordinateReferenceSystem(); if (crs == null) { crs = CrsUtilities.getCrsFromSrid(4326); // We go to the standard } - + var time2 = effectiveTime; // TODO merge with similar code from below IGrid grid = space.getGrid(); - RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); // end //TODO - List items = features.stream().map(f -> { try { return HMStacItem.fromSimpleFeature(f); } catch (Exception e) { scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); + return null; } - return null; - }).filter(i -> i != null).toList(); + }).filter(Objects::nonNull) + .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) + .toList(); GridCoverage2D coverage = null; try { - // TODO see if we can access to the same readRasterBandOnRegion without using a collection + // TODO see if we can access to the same readRasterBandOnRegion without using a + // collection LogProgressMonitor lpm = new LogProgressMonitor(); - HMStacManager manager = new HMStacManager(catalogUrl, lpm); - HMStacCollection collection = null; - try { - manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e1) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + try (HMStacManager manager = new HMStacManager(catalogUrl, lpm)) { + HMStacCollection collection = null; + try { + manager.open(); + collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + } catch (Exception e1) { + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + } + + if (collection == null) { + scope.getMonitor().error( + "Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + } + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, + MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } - - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - ArrayNode bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); + throw new KlabResourceAccessException( + "Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); } encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); return; } @@ -362,33 +390,47 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou try { manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl + ". Reason :" + e.getMessage()); - } - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } + if (collection == null) { + scope.getMonitor() + .error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + manager.close(); + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail + // fast + } - IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation - ? ((Observation) scope.getTargetArtifact()).getObservable() - : null; - HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation + ? ((Observation) scope.getTargetArtifact()).getObservable() + : null; + HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); + collection.setGeometryFilter(poly); + // collection.setTimestampFilter(new Date(start.getMilliseconds()), new + // Date(end.getMilliseconds())); --> Filter later :) - Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); - Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - + GridCoverage2D coverage = null; - if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { - time = validateTemporalDimension(time, resourceTime); - } - ITimeInstant start = time.getStart(); - ITimeInstant end = time.getEnd(); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) + // Allow transform ensures the process to finish, but I would not bet on the resulting + // data + if (assetPredicate == null) { + // NO JSONSelector and JSONValue found, NO assetID was passed as well + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), + effectiveTime.getEnd()); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); + return; + } - GridCoverage2D coverage = null; - try { List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -402,8 +444,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); @@ -415,110 +456,105 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou Client s3Client = buildS3Client(bucketRegion); collection.setS3Client(s3Client); } + var time = effectiveTime; + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC + items = items.stream() + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); + + if (items.size() == 0) { + manager.close(); + throw new KlabIllegalStateException( + "No STAC items found covering the entire time duration of the context requested"); + } else { + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + } - // Allow transform ensures the process to finish, but I would not bet on the resulting - // data. - final boolean allowTransform = true; - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } - return false; - } - }).collect(Collectors.toList()); - - if (items.size() == 0) { - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); - } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); - } - - - Set EPSGAtAssets = - items.stream() - .flatMap(item -> item.getAssets().stream()) - .filter(p) - .map(HMStacAsset::getEpsg) - .collect(Collectors.toUnmodifiableSet()); - - - if (EPSGAtAssets.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); - } + // Once the support for customized predicate is added, we can apply for features as well - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); - if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); - } - - manager.close(); + var pred = assetPredicate; + Set EPSGAtAssets = items.stream() + .flatMap(item -> item.getAssets().stream().filter(pred).findFirst() + .map(asset -> asset.getEpsg() != null ? asset.getEpsg() : item.getEpsg()).stream()) + .collect(Collectors.toUnmodifiableSet()); + + if (EPSGAtAssets.size() > 1) { + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." + + "The transformation process could affect the data."); + } + + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, + MergeMode.SUBSTITUTE, lpm); + if (outRaster == null) { + scope.getMonitor().error("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + throw new KlabIllegalStateException("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + } + coverage = outRaster.buildCoverage(); + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } + manager.close(); + encoder = new RasterEncoder(); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { - e.printStackTrace(); + e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } - encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } - private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { - Date datetime = (Date) f.getAttribute("datetime"); - if (datetime != null) { - if (isDateWithinRange(time2, datetime)) { - return true; + private Predicate getAssetPredicate(IResource resource) { + String assetId = resource.getParameters().get("asset", String.class); + if (assetId != null) { + return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); + } + String jsonSelector = resource.getParameters().get("jsonSelector", String.class); + String jsonValue = resource.getParameters().get("jsonValue", String.class); + if (jsonSelector != null && !jsonSelector.isBlank() && jsonValue != null) { + try { + return STACPathExpression.STACAssetPredicate.fromHMStacAsset(jsonSelector, jsonValue); + } catch (IllegalArgumentException e) { + throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector); } + } else { + throw new KlabIllegalArgumentException("Either asset or both jsonSelector and jsonValue must be provided"); } + } - Date itemStart = (Date) f.getAttribute("start_datetime"); - if (itemStart == null) { - return false; - } - Date itemEnd = (Date) f.getAttribute("end_datetime"); - if (itemEnd == null) { - return itemStart.toInstant().getEpochSecond() <= time2.getStart().getMilliseconds(); + /* + To check if an Item (of type HMStacItem) is within a time range + */ + private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok } - if (isDateWithinRange(time2, itemStart) || isDateWithinRange(time2, itemEnd)) { - return true; + + try { + + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + return startMillis >= itemStart && endMillis <= itemEnd; + + } catch (Exception e) { + e.printStackTrace(); + return false; } - return false; } - private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { - List links = collectionData.getJSONArray("links").toList().stream().filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("item")).toList(); - List urlOfLinks = links.stream().map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); + private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, + String collectionId) { + List links = collectionData.getJSONArray("links").toList().stream() + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("item")).toList(); + List urlOfLinks = links.stream() + .map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); return urlOfLinks.stream().map(i -> { try { return STACUtils.getItemAsFeature(i); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f5093..3dfab91737 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -9,10 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import org.integratedmodelling.kim.api.IParameters; -import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Resources; import org.integratedmodelling.klab.api.data.ILocator; import org.integratedmodelling.klab.api.data.IResource; @@ -44,22 +42,20 @@ public boolean acceptsMultiple() { return false; } + private boolean hasAssetSelector(IParameters parameters) { + return (parameters.contains("asset") || (parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + } + private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) - throws MalformedURLException { + throws Exception { String collectionUrl = parameters.get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = STACCollectionParser.readCollectionId(collectionData); parameters.put("collectionId", collectionId); - String regex = null; - if (parameters.contains("regex")) { - regex = parameters.get(Resources.REGEX_ENTRY, String.class); - parameters.remove(Resources.REGEX_ENTRY); - } - boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); - if (!parameters.contains("asset") && !isBulkImport) { + if (!hasAssetSelector(parameters) && !isBulkImport) { Builder builder = buildResource(parameters, project, monitor, collectionId); if (builder != null) { ret.add(builder); @@ -68,19 +64,82 @@ private void importCollection(List ret, IParameters parameters, } return; } - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + + String assetId = parameters.get("asset", String.class); + String resourceUrn = collectionId; + JSONObject assetData = null; + /* + This is only for the part, to check the Asset is under an S3 Bucket + */ + if (assetId != null) { + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + assetData = assetNode.getJSONObject(assetId); + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + } + resourceUrn = collectionId + "-" + assetId; + } + + //TODO: Check if assetId is null, and instead a jsonSelector and Value is passed! + //else if (parameters.get("jsonSelector", String.class) != null) { + // String selector = parameters.get("jsonSelector", String.class); + // String val = parameters.get("jsonValue", String.class); + // assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, null, STACPathExpression.STACAssetPredicate.fromKongJsonObject(selector, val)); + //} + + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + + Builder builder = buildResource(parameters, project, monitor, resourceUrn); + if (builder != null) { + ret.add(builder); + } else { + monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); + } + + // Think of a way to do the REGEX matching. Possibly not worth it to support + // this for something as diverse as STAC + + // String regex = null; + if (parameters.contains("regex")) { + // regex = parameters.get(Resources.REGEX_ENTRY, String.class); + parameters.remove(Resources.REGEX_ENTRY); + } + + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { if (regex != null && !assetId.matches(regex)) { Logging.INSTANCE.info("Asset " + assetId + " doesn't match REGEX, skipped"); continue; } - + JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); - if (!STACAssetParser.isSupportedMediaType(assetData)) { - Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); - continue; + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); + continue; + } } + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; String href = assetData.getString("href"); @@ -89,7 +148,7 @@ private void importCollection(List ret, IParameters parameters, String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } - + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); @@ -97,12 +156,14 @@ private void importCollection(List ret, IParameters parameters, monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } } + + */ } - private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { - Builder builder = validator.validate( - Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(parameters.get("collection", String.class)), - parameters, monitor); + private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) + throws MalformedURLException { + Builder builder = validator.validate(Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), + new URL(parameters.get("collection", String.class)), parameters, monitor); if (builder == null) { return null; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java new file mode 100644 index 0000000000..dfdfdc7264 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -0,0 +1,405 @@ +package org.integratedmodelling.klab.stac; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.TextNode; + +import kong.unirest.json.JSONObject; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; + +public final class STACPathExpression { + + private final List path; + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands.name"; + public static final String MEDIA_TYPE_BANDS_NAME = "type"; + + private STACPathExpression(List path) { + this.path = path; + } + + public static STACPathExpression parse(String jsonPath) { + if (jsonPath == null || jsonPath.isBlank()) { + throw new IllegalArgumentException("JSON path cannot be empty"); + } + return new STACPathExpression(parsePath(jsonPath)); + } + + public boolean matches(JsonNode root, String expectedValue) { + List resolvedNodes = resolve(root); + + for(JsonNode node : resolvedNodes) { + if (jsonValueEquals(node, expectedValue)) { + return true; + } + } + + return false; + } + + public List resolve(JsonNode root) { + List currentNodes = new ArrayList<>(); + currentNodes.add(root); + + for(PathPart part : path) { + List nextNodes = new ArrayList<>(); + + for(JsonNode current : currentNodes) { + if (current == null || current.isNull() || current.isMissingNode()) { + continue; + } + + resolvePart(current, part, nextNodes); + } + + currentNodes = nextNodes; + + if (currentNodes.isEmpty()) { + break; + } + } + + return currentNodes; + } + + private static void resolvePart(JsonNode current, PathPart part, List nextNodes) { + if (current.isArray()) { + resolveFromArray(current, part, nextNodes); + } else if (current.isObject()) { + resolveFromObject(current, part, nextNodes); + } + } + + private static void resolveFromArray(JsonNode arrayNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + JsonNode indexedNode = arrayNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + resolvePart(indexedNode, new PathPart(part.fieldName(), ArrayMode.NONE, null), nextNodes); + } + + } else { + for(JsonNode element : arrayNode) { + if (isUsable(element)) { + resolvePart(element, part, nextNodes); + } + } + } + } + + private static void resolveFromObject(JsonNode objectNode, PathPart part, List nextNodes) { + JsonNode directField = objectNode.get(part.fieldName()); + + if (isUsable(directField)) { + addResolvedField(directField, part, nextNodes); + } else { + objectNode.fields().forEachRemaining(entry -> { + String mapKey = entry.getKey(); + JsonNode mapValue = entry.getValue(); + + if (!isUsable(mapValue) || !mapValue.isObject()) { + return; + } + + if ("id".equalsIgnoreCase(part.fieldName())) { + nextNodes.add(TextNode.valueOf(mapKey)); + return; + } + + JsonNode nestedField = mapValue.get(part.fieldName()); + + if (isUsable(nestedField)) { + addResolvedField(nestedField, part, nextNodes); + } + }); + + } + } + + private static void addResolvedField(JsonNode fieldNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + nextNodes.add(indexedNode); + } + } + } else if (fieldNode.isArray()) { + for(JsonNode element : fieldNode) { + if (isUsable(element)) { + nextNodes.add(element); + } + } + } else { + nextNodes.add(fieldNode); + } + } + + private static boolean isUsable(JsonNode node) { + return node != null && !node.isNull() && !node.isMissingNode(); + } + + private static List parsePath(String jsonPath) { + String[] tokens = jsonPath.split("\\."); + + List parts = new ArrayList<>(); + + for(String token : tokens) { + String trimmed = token.trim(); + + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Invalid empty path element"); + } + + parts.add(PathPart.parse(trimmed)); + } + + return parts; + } + + private static boolean valueEquals(Object actualValue, String expectedValue) { + if (actualValue == null) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue instanceof Number number) { + return numberEquals(number, expectedValue); + } + + if (actualValue instanceof Boolean bool) { + return Boolean.toString(bool).equalsIgnoreCase(expectedValue); + } + + return Objects.equals(String.valueOf(actualValue), expectedValue); + } + + private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { + if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { + return expectedValue == null; + } + + if (expectedValue == null || !actualValue.isValueNode()) { + return false; + } + + if (actualValue.isNumber()) { + return numberEquals(actualValue.decimalValue(), expectedValue); + } + + if (actualValue.isBoolean()) { + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); + } + + return false; + } + + private static boolean numberEquals(Number actualValue, String expectedValue) { + try { + BigDecimal actual = new BigDecimal(actualValue.toString()); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } + + public record PathPart(String fieldName, ArrayMode arrayMode, Integer arrayIndex) { + + public static PathPart parse(String token) { + int bracketStart = token.indexOf('['); + + if (bracketStart < 0) { + return new PathPart(token, ArrayMode.NONE, null); + } + + int bracketEnd = token.indexOf(']', bracketStart); + + if (bracketEnd < 0) { + throw new IllegalArgumentException("Invalid array syntax in path element: " + token); + } + + if (bracketEnd != token.length() - 1) { + throw new IllegalArgumentException("Unexpected characters after array syntax in path element: " + token); + } + + String fieldName = token.substring(0, bracketStart).trim(); + String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); + + if (fieldName.isEmpty()) { + throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); + } + + int index; + try { + index = Integer.parseInt(indexText); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid array index in path element: " + token, e); + } + if (index < 0) { + throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); + } + return new PathPart(fieldName, ArrayMode.INDEX, index); + } + } + + public enum ArrayMode { + NONE, INDEX + } + + public static final class STACAssetPredicate { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private STACAssetPredicate() { + } + + public static Predicate fromJsonPath(String jsonPath, String expectedValue, + Function jsonNodeExtractor) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return object -> { + if (object == null) { + return false; + } + + JsonNode node = jsonNodeExtractor.apply(object); + + if (node == null || node.isNull() || node.isMissingNode()) { + return false; + } + + return expression.matches(node, expectedValue); + }; + } + + public static Predicate fromHMStacAsset(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, asset -> asset == null ? null : asset.getAssetNode()); + } + + public static Predicate fromJsonNode(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, node -> node); + } + + public static Predicate fromKongJsonObject(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); + } + + private static JsonNode toJsonNode(JSONObject jsonObject) { + if (jsonObject == null) { + return null; + } + + try { + return OBJECT_MAPPER.readTree(jsonObject.toString()); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Cannot convert JSONObject to JsonNode", e); + } + } + + public static Predicate fromHMStacAssetAttribute(String attributeName, String expectedValue) { + AssetAttribute attribute = AssetAttribute.fromName(attributeName); + + return asset -> { + if (asset == null) { + return false; + } + + Object actualValue = attribute.read(asset); + + return valueEquals(actualValue, expectedValue); + }; + } + + public static Predicate fromHMStacAssetId(String expectedValue) { + return fromHMStacAssetAttribute("id", expectedValue); + } + } + + public enum AssetAttribute { + + ID("id") { + @Override + Object read(HMStacAsset asset) { + return asset.getId(); + } + }, + + TITLE("title") { + @Override + Object read(HMStacAsset asset) { + return asset.getTitle(); + } + }, + + TYPE("type") { + @Override + Object read(HMStacAsset asset) { + return asset.getType(); + } + }, + + VALID("valid") { + @Override + Object read(HMStacAsset asset) { + return asset.isValid(); + } + }, + + EPSG("epsg") { + @Override + Object read(HMStacAsset asset) { + return asset.getEpsg(); + } + }, + + NON_VALID_REASON("nonValidReason") { + @Override + Object read(HMStacAsset asset) { + return asset.getNonValidReason(); + } + }; + + private final String name; + + AssetAttribute(String name) { + this.name = name; + } + + abstract Object read(HMStacAsset asset); + + public static AssetAttribute fromName(String name) { + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("Asset attribute name cannot be empty"); + } + + for(AssetAttribute attribute : values()) { + if (attribute.name.equalsIgnoreCase(name.trim())) { + return attribute; + } + } + + throw new IllegalArgumentException("Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason"); + } + } + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java index 99302419d1..0239e6f73f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACUtils.java @@ -32,13 +32,14 @@ public static String readKeywords(JSONObject json) { return null; } List keywords = json.getJSONArray("keywords").toList(); - return keywords.isEmpty() ? null : - keywords.stream().collect(Collectors.joining(",")); + return keywords.isEmpty() ? null : keywords.stream().collect(Collectors.joining(",")); } - final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", "properties.sci:doi", "item_assets.sci:doi"); + final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", + "properties.sci:doi", "item_assets.sci:doi"); public static String readDOI(JSONObject json) { - Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)).findFirst(); + Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)) + .findFirst(); return doi.isPresent() ? doi.get() : null; } @@ -61,13 +62,13 @@ public static String readDOIAuthors(String doi) { */ public static boolean containsLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .anyMatch(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)); + .anyMatch(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)); } public static Optional getLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)) - .map(link -> ((JSONObject)link).getString("href")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)) + .map(link -> ((JSONObject) link).getString("href")).findFirst(); } public static JSONObject requestMetadata(String collectionUrl, String type) { @@ -87,7 +88,7 @@ public static String readLicense(JSONObject collection) { return null; } JSONArray links = collection.getJSONArray("links"); - for (int i = 0; i < links.length(); i++) { + for(int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (!link.has("rel") || !link.getString("rel").equals("license")) { continue; @@ -121,11 +122,12 @@ public static Type inferValueType(String key) { public static String getCatalogUrl(String collectionUrl, String collectionId, JSONObject collectionData) { // The URL of the catalog is the root if (!collectionData.has("links")) { - throw new KlabResourceAccessException("STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); + throw new KlabResourceAccessException( + "STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); } JSONArray links = collectionData.getJSONArray("links"); Optional rootLink = links.toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("root")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("root")).findFirst(); if (rootLink.isEmpty()) { throw new KlabResourceAccessException("STAC collection is missing a relationship to the root catalog"); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac05..d02bc66870 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import org.integratedmodelling.kim.api.IParameters; import org.integratedmodelling.klab.api.data.IGeometry; @@ -20,6 +21,7 @@ import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor; import org.integratedmodelling.klab.data.resources.Resource; import org.integratedmodelling.klab.data.resources.ResourceBuilder; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnimplementedException; import org.integratedmodelling.klab.rest.CodelistReference; import org.integratedmodelling.klab.rest.MappingReference; @@ -43,55 +45,92 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni String collectionUrl = userData.get("collection", String.class); String collectionId = userData.get("collectionId", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); - if (collectionId == null) { + if (collectionId == null) { collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } - IGeometry geometry = STACCollectionParser.readGeometry(collectionData); - Builder builder = new ResourceBuilder(urn) - .withParameters(userData) - .withGeometry(geometry) - .withType(Type.OBJECT); + + IGeometry geometry = STACCollectionParser.readGeometry(collectionData); + Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); - // The default URL of the resource is the collection endpoint. May be overwritten. + // The default URL of the resource is the collection endpoint. May be overwritten. builder.withMetadata(IMetadata.DC_URL, collectionUrl); + JSONObject assetNode; + if (userData.contains("asset")) { - String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); - } - builder.addCodeList(codelist); + String requestedAssetId = userData.get("asset", String.class); + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); + + } else if (userData.contains("jsonSelector")) { + if (!userData.contains("jsonValue")) { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); } - if (type != null) { - builder.withType(type); + + Predicate predicate = STACPathExpression.STACAssetPredicate + .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); + + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); + + } else { + // Just import Features + monitor.info("import STAC Collection for Features"); + readMetadata(collectionData, builder); + return builder; + } + + String assetId = assetNode.keys().next(); + JSONObject asset = assetNode.getJSONObject(assetId); + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); } + builder.addCodeList(codelist); } - + if (type != null) { + builder.withType(type); + } + generateCodeList(builder, assetId, asset); + if (userData.contains("cog")) { - if (userData.get("cog") != null) { - builder.withType(Type.NUMBER); - } + if (userData.get("cog") != null) { + builder.withType(Type.NUMBER); + } } readMetadata(collectionData, builder); return builder; } + private void generateCodeList(Builder builder, String assetId, JSONObject asset) { + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + } + private Type readRasterDataType(JSONObject asset) { if (!asset.has("raster:bands")) { return null; } - + if (asset.getJSONArray("raster:bands").isEmpty() || !asset.getJSONArray("raster:bands").getJSONObject(0).has("data_type")) { // We assume that most rasters are numeric. When in doubt, we set the default to Number @@ -99,7 +138,8 @@ private Type readRasterDataType(JSONObject asset) { } String type = asset.getJSONArray("raster:bands").getJSONObject(0).getString("data_type"); // https://github.com/stac-extensions/raster?tab=readme-ov-file#data-types - final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", "float16", "float32", "float64"); + final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", + "float16", "float32", "float64"); if (NUMERIC_DATA_TYPES.contains(type)) { return Type.NUMBER; } @@ -121,8 +161,8 @@ private CodelistReference populateCodelist(String assetId, Map v MappingReference direct = new MappingReference(); MappingReference inverse = new MappingReference(); vals.entrySet().forEach(code -> { - direct.getMappings().add(new Pair<>(code.getKey(), (String)code.getValue())); - codelist.getCodeDescriptions().put(code.getKey(), (String)code.getValue()); + direct.getMappings().add(new Pair<>(code.getKey(), (String) code.getValue())); + codelist.getCodeDescriptions().put(code.getKey(), (String) code.getValue()); }); Type type = STACUtils.inferValueType(vals.entrySet().stream().findFirst().get().getKey()); codelist.setType(type); @@ -132,7 +172,8 @@ private CodelistReference populateCodelist(String assetId, Map v } private void readMetadata(final JSONObject json, Builder builder) { - // We could check the doi only if the Scientific Notation extension is provided, but we can try anyway + // We could check the doi only if the Scientific Notation extension is provided, but we can + // try anyway String doi = STACUtils.readDOI(json); if (doi != null && !doi.isBlank()) { builder.withMetadata(IMetadata.DC_URL, doi); @@ -199,7 +240,7 @@ public Collection getAllFilesForResource(File file) { } @Override - public Map describeResource(IResource resource) { + public Map< ? extends String, ? extends Object> describeResource(IResource resource) { // TODO Auto-generated method stub return null; } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java new file mode 100644 index 0000000000..aa668f5663 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -0,0 +1,119 @@ +package org.integratedmodelling.klab.stac.extensions; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.geotools.api.data.FeatureSource; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; +import org.integratedmodelling.klab.stac.STACUtils; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.hortonmachine.gears.io.stac.HMStacItem; + +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.integratedmodelling.klab.api.observations.scale.IScale; +import org.integratedmodelling.klab.api.observations.scale.space.IEnvelope; +import org.integratedmodelling.klab.api.observations.scale.space.IGrid; +import org.integratedmodelling.klab.api.observations.scale.time.ITimeInstant; +import org.integratedmodelling.klab.stac.STACUtils; +import java.time.format.DateTimeFormatter; +import java.time.*; + +public class STACFeatureExtension { + public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { + + String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") + .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); + + List featureList = new ArrayList<>(); + + JSONArray bboxArray = new JSONArray(); + for (Double v : bbox) { + bboxArray.put(v); + } + + + JSONObject searchPayload = new JSONObject() + .put("limit", 100) + .put("bbox", bboxArray) + .put("collections", new JSONArray().put(collectionId)); + + while (searchEndpoint != null) { + + HttpResponse response = Unirest + .post(searchEndpoint) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); + + JSONObject body = response.getBody().getObject(); + JSONArray features = body.getJSONArray("features"); + + Iterator featureIterator = features.iterator(); + + while (featureIterator.hasNext()) { + try { + JSONObject feature = (JSONObject) featureIterator.next(); + SimpleFeature feat = GeoJSONReader.parseFeature(feature.toString()); + HMStacItem item = HMStacItem.fromSimpleFeature(feat); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + if (item.getStartTimestamp() != null && item.getEndTimestamp() != null) { + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { + featureList.add(feat); + } + } else { + featureList.add(feat); + } + + + } catch (Exception e) { + e.printStackTrace(); + } + } + searchEndpoint = null; + if (body.has("links")) { + JSONArray links = body.getJSONArray("links"); + + for (Object obj : links) { + JSONObject link = (JSONObject) obj; + + if ("next".equalsIgnoreCase(link.optString("rel"))) { + String searchEndpointNew = link.getString("href"); + if (searchEndpointNew.equals(searchEndpoint)) { + searchEndpoint = null; + } + break; + } + } + } + } + if (featureList.isEmpty()) { + throw new Exception("No features found for the given parameters"); + } + + SimpleFeatureType type = featureList.get(0).getType(); + MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); + dataStore.addFeatures(featureList); + return dataStore.getFeatureSource(type.getTypeName()); + } +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d69..899adae92a 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -8,19 +8,26 @@ { final text 'collection' - "The URL pointing to the STAC collection file that contains the resource dataset." + "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." + The band for a multi-band raster. Default it will pick up the first one." default 0 - optional text 'cog' - "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" + optional text 'jsonSelector' + "JSON selector" + default "" + + optional text 'jsonValue' + "JSON value" default "" + + optional text 'cog' + "Relevant only for Resources served as Cloud Optimised GeoTiff, with Public Access" optional enum 'bandmixer' "Relevant only for raster resources.\n diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl index 24e2f0a112..dad0ee4666 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl @@ -35,6 +35,6 @@ optional enum 'axisOrder' "Select the order of the XY axis: 'lat_lon' (default), or 'lon_lat'." - default "lat_lon" + default lat_lon values lat_lon, lon_lat } \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 0000000000..11c75358ea --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,407 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +class STACPathExpressionTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static JsonNode stacItemNode() throws Exception { + String json = """ + { + "assets": { + "CHELSA_bio12_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_bio12_1981-2010_V.2.1.tif", + "title": "Climate Annual Precipitation (CHELSA bio12)", + "description": "Annual precipitation data from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "proj:bbox": [ + -180.0, + -90.0, + 180.0, + 90.0 + ], + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "bio12", + "description": "Annual precipitation" + } + ], + "file:size": 2608605060, + "raster:bands": [ + { + "nodata": -99999, + "data_type": "int32" + } + ] + }, + "CHELSA_gdd5_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gdd5_1981-2010_V.2.1.tif", + "title": "Climate Growing Degree Days > 5°C (CHELSA gdd5)", + "description": "Growing degree days (T > 5°C) from the CHELSA Climate dataset for the period 1981-2010", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gdd5", + "description": "Growing degree days with temperature > 5°C" + } + ], + "file:size": 3520234749, + "raster:bands": [ + { + "nodata": 2147483647, + "data_type": "int32" + } + ] + }, + "CHELSA_gsp_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gsp_1981-2010_V.2.1.tif", + "title": "Climate Precipitation in Growing Season (CHELSA gsp)", + "description": "Precipitation during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:3857", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gsp", + "description": "Precipitation in growing season" + } + ], + "file:size": 2447532198, + "raster:bands": [ + { + "nodata": 2147483648, + "data_type": "float32" + } + ] + }, + "CHELSA_gst_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gst_1981-2010_V.2.1.tif", + "title": "Climate Mean Temperature in Growing Season (CHELSA gst)", + "description": "Mean temperature during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gst", + "description": "Mean temperature in growing season" + } + ], + "file:size": 308275230, + "raster:bands": [ + { + "nodata": 65535, + "data_type": "int32" + } + ] + } + } + } + """; + + return OBJECT_MAPPER.readTree(json); + } + + private static JsonNode assetNode(String assetId) throws Exception { + return stacItemNode().get("assets").get(assetId); + } + + private static HMStacAsset bio12Asset() throws Exception { + return new HMStacAsset( + "CHELSA_bio12_1981-2010_V.2.1", + assetNode("CHELSA_bio12_1981-2010_V.2.1") + ); + } + + private static HMStacAsset gstAsset() throws Exception { + return new HMStacAsset( + "CHELSA_gst_1981-2010_V.2.1", + assetNode("CHELSA_gst_1981-2010_V.2.1") + ); + } + + @Test + void stacPathExpressionMatchesAssetByVirtualIdFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.id"); + + assertTrue(expression.matches(root, "CHELSA_gst_1981-2010_V.2.1")); + assertTrue(expression.matches(root, "CHELSA_bio12_1981-2010_V.2.1")); + assertFalse(expression.matches(root, "CHELSA_unknown")); + } + + @Test + void stacPathExpressionResolvesAllAssetIdsFromAssetsObjectMapKeys() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.id").resolve(root); + + assertEquals(4, resolved.size()); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_bio12_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gdd5_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gsp_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gst_1981-2010_V.2.1"))); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingImplicitArrayExpansion() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands.name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gdd5")); + assertTrue(expression.matches(root, "gsp")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingExplicitArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands[0].name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesAssetMediaTypeFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.type"); + + assertTrue(expression.matches(root, "image/tiff")); + assertFalse(expression.matches(root, "application/json")); + } + + @Test + void stacPathExpressionMatchesRasterBandDataTypeUsingArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.raster:bands[0].data_type"); + + assertTrue(expression.matches(root, "int32")); + assertTrue(expression.matches(root, "float32")); + assertFalse(expression.matches(root, "uint8")); + } + + @Test + void stacPathExpressionMatchesNumericValuesUsingNumericComparison() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.file:size"); + + assertTrue(expression.matches(root, "2608605060")); + assertTrue(expression.matches(root, "308275230.0")); + assertFalse(expression.matches(root, "123")); + } + + @Test + void hmStacAssetPredicateMatchesJsonExpressionAgainstAssetNode() throws Exception { + HMStacAsset asset = bio12Asset(); + + Predicate bandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate indexedBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands[0].name", "bio12"); + + Predicate typePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("type", "image/tiff"); + + Predicate rasterDataTypePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("raster:bands[0].data_type", "int32"); + + assertTrue(bandPredicate.test(asset)); + assertTrue(indexedBandPredicate.test(asset)); + assertTrue(typePredicate.test(asset)); + assertTrue(rasterDataTypePredicate.test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "gst") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("type", "application/json") + .test(asset)); + } + + @Test + void hmStacAssetPredicateMatchesJavaAssetAttributes() throws Exception { + HMStacAsset asset = bio12Asset(); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Climate Annual Precipitation (CHELSA bio12)") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "image/tiff") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326.0") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("valid", Boolean.toString(asset.isValid())) + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Wrong title") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "application/json") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "3857") + .test(asset)); + } + + @Test + void hmStacAssetAttributePredicateDistinguishesDifferentAssets() throws Exception { + HMStacAsset bio12 = bio12Asset(); + HMStacAsset gst = gstAsset(); + + Predicate bio12IdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1"); + + Predicate gstIdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1"); + + Predicate bio12BandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate gstBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "gst"); + + assertTrue(bio12IdPredicate.test(bio12)); + assertFalse(bio12IdPredicate.test(gst)); + + assertTrue(gstIdPredicate.test(gst)); + assertFalse(gstIdPredicate.test(bio12)); + + assertTrue(bio12BandPredicate.test(bio12)); + assertFalse(bio12BandPredicate.test(gst)); + + assertTrue(gstBandPredicate.test(gst)); + assertFalse(gstBandPredicate.test(bio12)); + } + + @Test + void predicatesReturnFalseForNullHMStacAsset() { + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "bio12") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + } + + @Test + void stacPathExpressionReturnsEmptyResultForMissingPath() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.missing.attribute").resolve(root); + + assertTrue(resolved.isEmpty()); + } + + @Test + void parseRejectsOldWildcardArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[*].name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void oldGreaterThanSeparatorDoesNotMatchBecauseDotIsNowTheSeparator() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets>eo:bands>name"); + + assertFalse(expression.matches(root, "bio12")); + } + + @Test + void parseRejectsEmptyPath() { + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(null)); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse("")); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(" ")); + } + + @Test + void parseRejectsNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[-1].name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void parseRejectsUnsupportedHMStacAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute("unsupported", "value") + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } +} \ No newline at end of file