From 1c7ec8f60c7f22175b83e17cf27bb52fb54ce9ac Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 4 May 2026 14:29:25 +0200 Subject: [PATCH 01/28] Ensure close manager --- .../klab/stac/STACEncoder.java | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9c331a850..d2bb60d42 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -311,40 +311,41 @@ public void getEncodedData(IResource resource, Map urnParameters try { // TODO see if we can access to the same readRasterBandOnRegion without using a collection LogProgressMonitor lpm = new LogProgressMonitor(); - HMStacManager manager = new HMStacManager(catalogUrl, lpm); - HMStacCollection collection = null; - try { - manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e1) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } - - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - ArrayNode bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); + try (HMStacManager manager = new HMStacManager(catalogUrl, lpm)) { + HMStacCollection collection = null; + try { + manager.open(); + collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + } catch (Exception e1) { + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + } + + if (collection == null) { + scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + } + + var p = new Predicate() { + + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + ArrayNode bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } + return false; + } + }; + + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + } if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } @@ -363,6 +364,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); } catch (Exception e) { + try { + manager.close(); + } catch (Exception e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl + ". Reason :" + e.getMessage()); } @@ -461,6 +468,7 @@ public boolean test(HMStacItem item) { }).collect(Collectors.toList()); if (items.size() == 0) { + manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); From 331e52f7ffddb2c2a9afb31f495879dfedd4a41b Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 4 May 2026 14:35:04 +0200 Subject: [PATCH 02/28] Better generated xtext code --- .../kactors/ui/KactorsUiModule.java | 4 +- .../kdl/ui/KdlUiModule.java | 4 +- .../kim/ui/KimUiModule.java | 4 +- .../kim/formatting2/KimFormatter.java | 2 + .../kim/validation/KimValidator.java | 52 +++++++++---------- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java b/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java index 4a329480b..7537652c7 100644 --- a/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java +++ b/kactors/org.integratedmodelling.kactors.ui/xtend-gen/org/integratedmodelling/kactors/ui/KactorsUiModule.java @@ -30,7 +30,7 @@ public Class bindIHighlightingConfiguratio return KactorsHighlightingConfiguration.class; } - public KactorsUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KactorsUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java b/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java index 9d12f273a..26b2c6bf3 100644 --- a/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java +++ b/kdl/org.integratedmodelling.kdl.ui/xtend-gen/org/integratedmodelling/kdl/ui/KdlUiModule.java @@ -30,7 +30,7 @@ public Class bindAbstractAntlrTo return KdlSyntaxHighlighter.class; } - public KdlUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KdlUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java b/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java index 059a6f44a..0ede2c465 100644 --- a/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java +++ b/kim/org.integratedmodelling.kim.ui/xtend-gen/org/integratedmodelling/kim/ui/KimUiModule.java @@ -100,7 +100,7 @@ public Class bindIEObjectDocumentationP return KimDocumentationProvider.class; } - public KimUiModule(final AbstractUIPlugin arg0) { - super(arg0); + public KimUiModule(final AbstractUIPlugin plugin) { + super(plugin); } } diff --git a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java index 88dea3ee1..2958fb5f5 100644 --- a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java +++ b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/formatting2/KimFormatter.java @@ -12,6 +12,7 @@ import org.eclipse.xtext.formatting2.IFormattableDocument; import org.eclipse.xtext.resource.XtextResource; import org.eclipse.xtext.xbase.lib.Extension; +import org.eclipse.xtext.xbase.lib.XbaseGenerated; import org.integratedmodelling.kim.kim.Concept; import org.integratedmodelling.kim.kim.ConceptDeclaration; import org.integratedmodelling.kim.kim.ConceptStatement; @@ -60,6 +61,7 @@ protected void _format(final ConceptDeclaration conceptDeclaration, @Extension f document.format(conceptDeclaration.getContext()); } + @XbaseGenerated public void format(final Object conceptDeclaration, final IFormattableDocument document) { if (conceptDeclaration instanceof XtextResource) { _format((XtextResource)conceptDeclaration, document); diff --git a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java index 5e927d1d4..8ad16aee2 100644 --- a/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java +++ b/kim/org.integratedmodelling.kim/xtend-gen/org/integratedmodelling/kim/validation/KimValidator.java @@ -4,12 +4,12 @@ */ package org.integratedmodelling.kim.validation; -import com.google.common.base.Objects; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.Objects; import java.util.Set; import java.util.logging.Level; import org.eclipse.emf.common.util.EList; @@ -443,20 +443,20 @@ public void checkModelStatement(final ModelStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_STATEMENT__MODEL, KimValidator.REASONING_PROBLEM); } @@ -533,14 +533,14 @@ public void checkModelDefinition(final ModelBodyStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); @@ -548,7 +548,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__OBSERVABLES, obsIdx, KimValidator.REASONING_PROBLEM); @@ -668,14 +668,14 @@ public void checkModelDefinition(final ModelBodyStatement model) { int _level = ref.getLevel(); boolean _matched = false; int _intValue = Level.SEVERE.intValue(); - if (Objects.equal(_level, _intValue)) { + if (Objects.equals(_level, _intValue)) { _matched=true; this.error(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); } if (!_matched) { int _intValue_1 = Level.WARNING.intValue(); - if (Objects.equal(_level, _intValue_1)) { + if (Objects.equals(_level, _intValue_1)) { _matched=true; this.warning(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); @@ -683,7 +683,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } if (!_matched) { int _intValue_2 = Level.INFO.intValue(); - if (Objects.equal(_level, _intValue_2)) { + if (Objects.equals(_level, _intValue_2)) { _matched=true; this.info(ref.getMessage(), KimPackage.Literals.MODEL_BODY_STATEMENT__DEPENDENCIES, i, KimValidator.REASONING_PROBLEM); @@ -775,7 +775,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { for (final Classifier classifier : _classifiers) { { KimConcept decl = Kim.INSTANCE.declareConcept(classifier.getDeclaration()); - boolean _equals = Objects.equal(type, Integer.valueOf(0)); + boolean _equals = Objects.equals(type, Integer.valueOf(0)); if (_equals) { type = decl.getType(); } else { @@ -853,10 +853,10 @@ public void checkModelDefinition(final ModelBodyStatement model) { String _id = arg.getId(); boolean _tripleNotEquals_4 = (_id != null); if (_tripleNotEquals_4) { - if (((!Objects.equal(arg.getId(), "?")) && (!Objects.equal(arg.getId(), "*")))) { + if (((!Objects.equals(arg.getId(), "?")) && (!Objects.equals(arg.getId(), "*")))) { boolean found = false; for (final KimObservable dependency : dependencies) { - if (((dependency.getName() != null) && Objects.equal(dependency.getName(), arg))) { + if (((dependency.getName() != null) && Objects.equals(dependency.getName(), arg))) { found = true; } } @@ -864,7 +864,7 @@ public void checkModelDefinition(final ModelBodyStatement model) { } } else { String _id_1 = arg.getId(); - boolean _equals = Objects.equal(_id_1, "?"); + boolean _equals = Objects.equals(_id_1, "?"); if (_equals) { if (checkFound) { this.error("Only one \'?\' is allowed in the argument list, to mark the result column", @@ -1118,18 +1118,18 @@ public void checkModelDefinition(final ModelBodyStatement model) { public void notify(final KimNotification notification, final EObject object, final EStructuralFeature cls) { final Level _switchValue = notification.level; boolean _matched = false; - if (Objects.equal(_switchValue, Level.SEVERE)) { + if (Objects.equals(_switchValue, Level.SEVERE)) { _matched=true; this.error(notification.message, object, cls); } if (!_matched) { - if (Objects.equal(_switchValue, Level.WARNING)) { + if (Objects.equals(_switchValue, Level.WARNING)) { _matched=true; this.warning(notification.message, object, cls); } } if (!_matched) { - if (Objects.equal(_switchValue, Level.INFO)) { + if (Objects.equals(_switchValue, Level.INFO)) { _matched=true; this.info(notification.message, object, cls); } @@ -1208,7 +1208,7 @@ public KimAcknowledgement checkObservation(final ObserveStatementBody observatio for (final KimNotification notification : _addAction) { { this.notify(notification, observation, KimPackage.Literals.OBSERVE_STATEMENT_BODY__ACTIONS, i); - boolean _equals = Objects.equal(notification.level, Level.SEVERE); + boolean _equals = Objects.equals(notification.level, Level.SEVERE); if (_equals) { ok = false; } @@ -2366,7 +2366,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep EnumSet corectype = Kim.INSTANCE.getType(((ConceptStatement) _eContainer).getConcept(), null); EnumSet a = Kim.intersection(type, IKimConcept.DECLARABLE_TYPES); EnumSet b = Kim.intersection(corectype, IKimConcept.DECLARABLE_TYPES); - if ((((a.size() != 1) || (b.size() != 1)) || (!Objects.equal(((IKimConcept.Type[])Conversions.unwrapArray(a, IKimConcept.Type.class))[0], ((Object[])Conversions.unwrapArray(b, Object.class))[0])))) { + if ((((a.size() != 1) || (b.size() != 1)) || (!Objects.equals(((IKimConcept.Type[])Conversions.unwrapArray(a, IKimConcept.Type.class))[0], ((Object[])Conversions.unwrapArray(b, Object.class))[0])))) { this.error("Core concept is not compatible with the stated type", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS); error = true; @@ -2428,7 +2428,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep if (_not_1) { KimConcept declaration = Kim.INSTANCE.declareConcept(p); if (((declaration != null) && (!declaration.getType().isEmpty()))) { - if (((i == 0) || Objects.equal(concept.getConnectors().get((i - 1)), ","))) { + if (((i == 0) || Objects.equals(concept.getConnectors().get((i - 1)), ","))) { KimConceptStatement.ParentConceptImpl group = new KimConceptStatement.ParentConceptImpl(); group.getConcepts().add(declaration); declaredParents.add(group); @@ -2458,12 +2458,12 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep _xifexpression_1 = _xifexpression_2; } BinarySemanticOperator connector = _xifexpression_1; - if ((Objects.equal(connector, BinarySemanticOperator.FOLLOWS) && + if ((Objects.equals(connector, BinarySemanticOperator.FOLLOWS) && (!declaration.getType().contains(IKimConcept.Type.EVENT)))) { this.error("The consequentiality (\'follows\') operator is only allowed between events", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS, i); error = true; } - if (((group_1.getConnector() != BinarySemanticOperator.NONE) && (!Objects.equal(group_1.getConnector(), connector)))) { + if (((group_1.getConnector() != BinarySemanticOperator.NONE) && (!Objects.equals(group_1.getConnector(), connector)))) { this.error( "Cannot mix union (\'or\'), intersection (\'and\') and consequentiality (\'follows\') operators", concept, KimPackage.Literals.CONCEPT_STATEMENT_BODY__PARENTS, i); error = true; @@ -2592,7 +2592,7 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep { KimConcept iden = Kim.INSTANCE.declareConcept(identity); String _type = requirement.getType(); - boolean _equals = Objects.equal(_type, "identity"); + boolean _equals = Objects.equals(_type, "identity"); if (_equals) { boolean _contains = iden.getType().contains(IKimConcept.Type.IDENTITY); boolean _not_1 = (!_contains); @@ -3026,18 +3026,18 @@ public KimConceptStatement validateConceptBody(final ConceptStatementBody concep public void notify(final KimNotification notification, final EObject object, final EStructuralFeature cls, final int index) { final Level _switchValue = notification.level; boolean _matched = false; - if (Objects.equal(_switchValue, Level.SEVERE)) { + if (Objects.equals(_switchValue, Level.SEVERE)) { _matched=true; this.error(notification.message, object, cls, index); } if (!_matched) { - if (Objects.equal(_switchValue, Level.WARNING)) { + if (Objects.equals(_switchValue, Level.WARNING)) { _matched=true; this.warning(notification.message, object, cls, index); } } if (!_matched) { - if (Objects.equal(_switchValue, Level.INFO)) { + if (Objects.equals(_switchValue, Level.INFO)) { _matched=true; this.info(notification.message, object, cls, index); } From 094383269021e8dda9b56c07cf0ade1d4731b51d Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 5 May 2026 12:50:51 +0200 Subject: [PATCH 03/28] Add first version of predicate for assets using path expressions --- .../klab/stac/STACEncoder.java | 140 ++++------ .../klab/stac/STACPathExpression.java | 227 ++++++++++++++++ .../ogc/stac/test/STACPathExpressionTest.java | 254 ++++++++++++++++++ 3 files changed, 535 insertions(+), 86 deletions(-) create mode 100644 adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index d2bb60d42..7eece7172 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -56,6 +56,7 @@ import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; +import org.integratedmodelling.klab.stac.STACPathExpression.STACAssetPredicate; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; @@ -70,6 +71,7 @@ import org.geotools.api.referencing.FactoryException; import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; @@ -79,8 +81,8 @@ import kong.unirest.json.JSONObject; public class STACEncoder implements IResourceEncoder { - - /** + + /** * The raster or vector encoder that does the actual work after we get our coverage from the service. */ IResourceEncoder encoder; @@ -324,25 +326,7 @@ public void getEncodedData(IResource resource, Map urnParameters scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); } - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - ArrayNode bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; - + Predicate p = STACAssetPredicate.from("eo:bands[*]>name", assetId); HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } @@ -426,74 +410,58 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // Allow transform ensures the process to finish, but I would not bet on the resulting // data. final boolean allowTransform = true; - var p = new Predicate() { - - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } - return false; - } - }; + Predicate p = STACAssetPredicate.from(STACPathExpression.PREDICATE_EO_BANDS_NAME, assetId); - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } - return false; - } - }).collect(Collectors.toList()); + // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC + items = items.stream().filter(new Predicate() { + + @Override + public boolean test(HMStacItem item) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } + return false; + } + }).collect(Collectors.toList()); - if (items.size() == 0) { - manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); - } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); - } - - - Set EPSGAtAssets = - items.stream() - .flatMap(item -> item.getAssets().stream()) - .filter(p) - .map(HMStacAsset::getEpsg) - .collect(Collectors.toUnmodifiableSet()); - - - if (EPSGAtAssets.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); - } + if (items.size() == 0) { + manager.close(); + throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); + } else { + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + } + + + Set EPSGAtAssets = + items.stream() + .flatMap(item -> item.getAssets().stream()) + .filter(p) + .map(HMStacAsset::getEpsg) + .collect(Collectors.toUnmodifiableSet()); + + + if (EPSGAtAssets.size() > 1) { + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); + } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); - if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); - } - - manager.close(); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + if (bandIndex != null) { // Which means theat it's a Multi Band COG + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + } + + manager.close(); } catch (Exception e) { e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java new file mode 100644 index 000000000..3daa068ab --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -0,0 +1,227 @@ +package org.integratedmodelling.klab.stac; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; + +public final class STACPathExpression { + + private final List path; + + private STACPathExpression(List path) { + this.path = path; + } + + public static STACPathExpression parse(String jsonPath) { + if (jsonPath == null || jsonPath.isBlank()) { + throw new IllegalArgumentException("JSON path cannot be empty"); + } + + return new STACPathExpression(parsePath(jsonPath)); + } + + public boolean matches(JsonNode root, String expectedValue) { + List resolvedNodes = resolve(root); + + for (JsonNode node : resolvedNodes) { + if (jsonValueEquals(node, expectedValue)) { + return true; + } + } + + return false; + } + + public List resolve(JsonNode root) { + List currentNodes = new ArrayList<>(); + currentNodes.add(root); + + for (PathPart part : path) { + List nextNodes = new ArrayList<>(); + + for (JsonNode current : currentNodes) { + if (current == null || current.isNull() || current.isMissingNode()) { + continue; + } + + JsonNode fieldNode = current.get(part.fieldName()); + + if (fieldNode == null || fieldNode.isNull() || fieldNode.isMissingNode()) { + continue; + } + + if (part.arrayMode() == ArrayMode.NONE) { + nextNodes.add(fieldNode); + } else if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (indexedNode != null && !indexedNode.isNull() && !indexedNode.isMissingNode()) { + nextNodes.add(indexedNode); + } + } + } else if (part.arrayMode() == ArrayMode.WILDCARD) { + if (fieldNode.isArray()) { + for (JsonNode arrayElement : fieldNode) { + if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { + nextNodes.add(arrayElement); + } + } + } + } + } + + currentNodes = nextNodes; + + if (currentNodes.isEmpty()) { + break; + } + } + + return currentNodes; + } + + private static List parsePath(String jsonPath) { + String[] tokens = jsonPath.split(">"); + + List parts = new ArrayList<>(); + + for (String token : tokens) { + String trimmed = token.trim(); + + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Invalid empty path element"); + } + + parts.add(PathPart.parse(trimmed)); + } + + return parts; + } + + private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { + if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue.isNumber()) { + return numberEquals(actualValue, expectedValue); + } + + if (actualValue.isBoolean()) { + return Boolean.toString(actualValue.booleanValue()) + .equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); + } + + return Objects.equals(actualValue.asText(), expectedValue); + } + + private static boolean numberEquals(JsonNode actualValue, String expectedValue) { + try { + BigDecimal actual = actualValue.decimalValue(); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } + + public record PathPart( + String fieldName, + ArrayMode arrayMode, + Integer arrayIndex + ) { + + public static PathPart parse(String token) { + int bracketStart = token.indexOf('['); + + if (bracketStart < 0) { + return new PathPart(token, ArrayMode.NONE, null); + } + + int bracketEnd = token.indexOf(']', bracketStart); + + if (bracketEnd < 0) { + throw new IllegalArgumentException( + "Invalid array syntax in path element: " + token + ); + } + + if (bracketEnd != token.length() - 1) { + throw new IllegalArgumentException( + "Unexpected characters after array syntax in path element: " + token + ); + } + + String fieldName = token.substring(0, bracketStart).trim(); + String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); + + if (fieldName.isEmpty()) { + throw new IllegalArgumentException( + "Field name cannot be empty in path element: " + token + ); + } + + if ("*".equals(indexText)) { + return new PathPart(fieldName, ArrayMode.WILDCARD, null); + } + + int index; + + try { + index = Integer.parseInt(indexText); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid array index in path element: " + token, + e + ); + } + + if (index < 0) { + throw new IllegalArgumentException( + "Array index cannot be negative in path element: " + token + ); + } + + return new PathPart(fieldName, ArrayMode.INDEX, index); + } + } + + public enum ArrayMode { + NONE, + INDEX, + WILDCARD + } + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + + public final class STACAssetPredicate { + + private STACAssetPredicate() { + } + + public static Predicate from(String jsonPath, String expectedValue) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return asset -> expression.matches(asset.getAssetNode(), expectedValue); + } + } + + + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 000000000..f66774e9a --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,254 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class STACPathExpressionTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonNode testJson() throws Exception { + String json = """ + { + "node1": { + "node2": [ + { + "name": "something", + "count": 10, + "active": true + }, + { + "name": "other", + "count": 20, + "active": false + } + ] + }, + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + }, + { + "name": "B08", + "common_name": "nir" + } + ], + "properties": { + "cloud_cover": 12.5, + "enabled": true + } + } + """; + + return MAPPER.readTree(json); + } + + @Test + void shouldResolveIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertTrue(expression.matches(root, "something")); + assertFalse(expression.matches(root, "other")); + } + + @Test + void shouldResolveSecondIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[1]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("other", resolved.get(0).asText()); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "something")); + } + + @Test + void shouldResolveWildcardArrayElements() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(2, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertEquals("other", resolved.get(1).asText()); + + assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "missing")); + } + + @Test + void shouldMatchEoBandsUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(3, resolved.size()); + assertEquals("B01", resolved.get(0).asText()); + assertEquals("B04", resolved.get(1).asText()); + assertEquals("B08", resolved.get(2).asText()); + + assertTrue(expression.matches(root, "B01")); + assertTrue(expression.matches(root, "B04")); + assertTrue(expression.matches(root, "B08")); + assertFalse(expression.matches(root, "B99")); + } + + @Test + void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>common_name"); + + assertTrue(expression.matches(root, "coastal")); + assertTrue(expression.matches(root, "red")); + assertTrue(expression.matches(root, "nir")); + assertFalse(expression.matches(root, "green")); + } + + @Test + void shouldMatchNumbers() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>count"); + + assertTrue(expression.matches(root, "10")); + assertTrue(expression.matches(root, "10.0")); + assertFalse(expression.matches(root, "20")); + assertFalse(expression.matches(root, "not-a-number")); + } + + @Test + void shouldMatchBooleans() throws Exception { + JsonNode root = testJson(); + + STACPathExpression firstExpression = + STACPathExpression.parse("node1>node2[0]>active"); + + STACPathExpression secondExpression = + STACPathExpression.parse("node1>node2[1]>active"); + + assertTrue(firstExpression.matches(root, "true")); + assertTrue(firstExpression.matches(root, "TRUE")); + assertFalse(firstExpression.matches(root, "false")); + + assertTrue(secondExpression.matches(root, "false")); + assertTrue(secondExpression.matches(root, "FALSE")); + assertFalse(secondExpression.matches(root, "true")); + } + + @Test + void shouldMatchTopLevelProperties() throws Exception { + JsonNode root = testJson(); + + STACPathExpression cloudCoverExpression = + STACPathExpression.parse("properties>cloud_cover"); + + STACPathExpression enabledExpression = + STACPathExpression.parse("properties>enabled"); + + assertTrue(cloudCoverExpression.matches(root, "12.5")); + assertTrue(cloudCoverExpression.matches(root, "12.50")); + assertFalse(cloudCoverExpression.matches(root, "13")); + + assertTrue(enabledExpression.matches(root, "true")); + assertFalse(enabledExpression.matches(root, "false")); + } + + @Test + void shouldReturnEmptyListForMissingPath() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>missing"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "anything")); + } + + @Test + void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("properties[*]>cloud_cover"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "12.5")); + } + + @Test + void shouldRejectInvalidWildcardSyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[abc]>name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void shouldRejectNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[-1]>name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void shouldRejectUnexpectedCharactersAfterArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[0]extra>name") + ); + + assertTrue(exception.getMessage().contains("Unexpected characters")); + } + + @Test + void shouldRejectEmptyPathElement() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>>name") + ); + + assertTrue(exception.getMessage().contains("Invalid empty path element")); + } +} \ No newline at end of file From f48b09d0d269aa78a895a2fc124b55dcc1103e29 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 5 May 2026 14:23:36 +0200 Subject: [PATCH 04/28] Add predicate using HMAsset attributes --- .../klab/stac/STACPathExpression.java | 137 ++++- .../STACPathExpressionAndAttributeTest.java | 545 ++++++++++++++++++ .../ogc/stac/test/STACPathExpressionTest.java | 254 -------- 3 files changed, 679 insertions(+), 257 deletions(-) create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java delete mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 3daa068ab..6bdd725ef 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -13,6 +13,8 @@ public final class STACPathExpression { private final List path; + + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; private STACPathExpression(List path) { this.path = path; @@ -104,6 +106,37 @@ private static List parsePath(String jsonPath) { return parts; } + + private static boolean valueEquals(Object actualValue, String expectedValue) { + if (actualValue == null) { + return expectedValue == null; + } + + if (expectedValue == null) { + return false; + } + + if (actualValue instanceof Number number) { + return numberEquals(number, expectedValue); + } + + if (actualValue instanceof Boolean bool) { + return Boolean.toString(bool).equalsIgnoreCase(expectedValue); + } + + return Objects.equals(String.valueOf(actualValue), expectedValue); + } + + private static boolean numberEquals(Number actualValue, String expectedValue) { + try { + BigDecimal actual = new BigDecimal(actualValue.toString()); + BigDecimal expected = new BigDecimal(expectedValue); + + return actual.compareTo(expected) == 0; + } catch (NumberFormatException e) { + return false; + } + } private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { @@ -208,12 +241,42 @@ public enum ArrayMode { WILDCARD } - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; - public final class STACAssetPredicate { private STACAssetPredicate() { } + + public static Predicate fromJsonPath( + String jsonPath, + String expectedValue + ) { + STACPathExpression expression = STACPathExpression.parse(jsonPath); + + return asset -> { + if (asset == null || asset.getAssetNode() == null) { + return false; + } + + return expression.matches(asset.getAssetNode(), expectedValue); + }; + } + + public static Predicate fromAssetAttribute( + String attributeName, + String expectedValue + ) { + AssetAttribute attribute = AssetAttribute.fromName(attributeName); + + return asset -> { + if (asset == null) { + return false; + } + + Object actualValue = attribute.read(asset); + + return valueEquals(actualValue, expectedValue); + }; + } public static Predicate from(String jsonPath, String expectedValue) { STACPathExpression expression = STACPathExpression.parse(jsonPath); @@ -222,6 +285,74 @@ public static Predicate from(String jsonPath, String expectedValue) } } - + public enum AssetAttribute { + + ID("id") { + @Override + Object read(HMStacAsset asset) { + return asset.getId(); + } + }, + + TITLE("title") { + @Override + Object read(HMStacAsset asset) { + return asset.getTitle(); + } + }, + + TYPE("type") { + @Override + Object read(HMStacAsset asset) { + return asset.getType(); + } + }, + + VALID("valid") { + @Override + Object read(HMStacAsset asset) { + return asset.isValid(); + } + }, + + EPSG("epsg") { + @Override + Object read(HMStacAsset asset) { + return asset.getEpsg(); + } + }, + + NON_VALID_REASON("nonValidReason") { + @Override + Object read(HMStacAsset asset) { + return asset.getNonValidReason(); + } + }; + + private final String name; + + AssetAttribute(String name) { + this.name = name; + } + + abstract Object read(HMStacAsset asset); + + public static AssetAttribute fromName(String name) { + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("Asset attribute name cannot be empty"); + } + + for (AssetAttribute attribute : values()) { + if (attribute.name.equalsIgnoreCase(name.trim())) { + return attribute; + } + } + + throw new IllegalArgumentException( + "Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason" + ); + } + } } \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java new file mode 100644 index 000000000..d24b147f3 --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java @@ -0,0 +1,545 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.*; + +class STACPathExpressionAndAttributeTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonNode testJson() throws Exception { + String json = """ + { + "node1": { + "node2": [ + { + "name": "something", + "count": 10, + "active": true + }, + { + "name": "other", + "count": 20, + "active": false + } + ] + }, + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + }, + { + "name": "B08", + "common_name": "nir" + } + ], + "properties": { + "cloud_cover": 12.5, + "enabled": true + } + } + """; + + return MAPPER.readTree(json); + } + + @Test + void shouldResolveIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertTrue(expression.matches(root, "something")); + assertFalse(expression.matches(root, "other")); + } + + @Test + void shouldResolveSecondIndexedArrayElement() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[1]>name"); + + List resolved = expression.resolve(root); + + assertEquals(1, resolved.size()); + assertEquals("other", resolved.get(0).asText()); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "something")); + } + + @Test + void shouldResolveWildcardArrayElements() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(2, resolved.size()); + assertEquals("something", resolved.get(0).asText()); + assertEquals("other", resolved.get(1).asText()); + + assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "other")); + assertFalse(expression.matches(root, "missing")); + } + + @Test + void shouldMatchEoBandsUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>name"); + + List resolved = expression.resolve(root); + + assertEquals(3, resolved.size()); + assertEquals("B01", resolved.get(0).asText()); + assertEquals("B04", resolved.get(1).asText()); + assertEquals("B08", resolved.get(2).asText()); + + assertTrue(expression.matches(root, "B01")); + assertTrue(expression.matches(root, "B04")); + assertTrue(expression.matches(root, "B08")); + assertFalse(expression.matches(root, "B99")); + } + + @Test + void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("eo:bands[*]>common_name"); + + assertTrue(expression.matches(root, "coastal")); + assertTrue(expression.matches(root, "red")); + assertTrue(expression.matches(root, "nir")); + assertFalse(expression.matches(root, "green")); + } + + @Test + void shouldMatchNumbers() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[0]>count"); + + assertTrue(expression.matches(root, "10")); + assertTrue(expression.matches(root, "10.0")); + assertFalse(expression.matches(root, "20")); + assertFalse(expression.matches(root, "not-a-number")); + } + + @Test + void shouldMatchBooleans() throws Exception { + JsonNode root = testJson(); + + STACPathExpression firstExpression = + STACPathExpression.parse("node1>node2[0]>active"); + + STACPathExpression secondExpression = + STACPathExpression.parse("node1>node2[1]>active"); + + assertTrue(firstExpression.matches(root, "true")); + assertTrue(firstExpression.matches(root, "TRUE")); + assertFalse(firstExpression.matches(root, "false")); + + assertTrue(secondExpression.matches(root, "false")); + assertTrue(secondExpression.matches(root, "FALSE")); + assertFalse(secondExpression.matches(root, "true")); + } + + @Test + void shouldMatchTopLevelProperties() throws Exception { + JsonNode root = testJson(); + + STACPathExpression cloudCoverExpression = + STACPathExpression.parse("properties>cloud_cover"); + + STACPathExpression enabledExpression = + STACPathExpression.parse("properties>enabled"); + + assertTrue(cloudCoverExpression.matches(root, "12.5")); + assertTrue(cloudCoverExpression.matches(root, "12.50")); + assertFalse(cloudCoverExpression.matches(root, "13")); + + assertTrue(enabledExpression.matches(root, "true")); + assertFalse(enabledExpression.matches(root, "false")); + } + + @Test + void shouldReturnEmptyListForMissingPath() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("node1>node2[*]>missing"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "anything")); + } + + @Test + void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { + JsonNode root = testJson(); + + STACPathExpression expression = + STACPathExpression.parse("properties[*]>cloud_cover"); + + List resolved = expression.resolve(root); + + assertTrue(resolved.isEmpty()); + assertFalse(expression.matches(root, "12.5")); + } + + @Test + void shouldRejectInvalidWildcardSyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[abc]>name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void shouldRejectNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[-1]>name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void shouldRejectUnexpectedCharactersAfterArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>node2[0]extra>name") + ); + + assertTrue(exception.getMessage().contains("Unexpected characters")); + } + + @Test + void shouldRejectEmptyPathElement() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("node1>>name") + ); + + assertTrue(exception.getMessage().contains("Invalid empty path element")); + } + + // Test by attributes + + @Test + void shouldMatchAssetByIdAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetByDifferentIdAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B08"); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByTitleAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Surface reflectance band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "title", + "Surface reflectance band 4" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByTypeAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "type", + "image/tiff; application=geotiff" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByEpsgAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByEpsgAttributeUsingDecimalEquivalent() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632.0"); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetByDifferentEpsgAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "proj:code": "EPSG:32632" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "4326"); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetByValidAttribute() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff" + } + """); + + HMStacAsset asset = new HMStacAsset("B04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("valid", "true"); + + /* + * One caveat: HMStacAsset computes isValid through HMStacAssetHandlers.getHandler(this). + * Depending on your test classpath and supported MIME types, asset.isValid() + * may be either true or false. That is why the validity test uses: + * assertEquals(asset.isValid(), predicate.test(asset)); + */ + assertEquals(asset.isValid(), predicate.test(asset)); + } + + // JSON-path-through-asset tests + @Test + void shouldMatchAssetUsingJsonPathPredicate() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B04" + ); + + assertTrue(predicate.test(asset)); + } + + @Test + void shouldNotMatchAssetUsingJsonPathPredicateWhenValueIsMissing() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal" + }, + { + "name": "B04", + "common_name": "red" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B08" + ); + + assertFalse(predicate.test(asset)); + } + + @Test + void shouldMatchAssetUsingDefaultJsonPathPredicateFactory() throws Exception { + JsonNode assetNode = MAPPER.readTree(""" + { + "title": "Band 4", + "type": "image/tiff; application=geotiff", + "eo:bands": [ + { + "name": "B04" + } + ] + } + """); + + HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); + + Predicate predicate = + STACPathExpression.STACAssetPredicate.from( + "eo:bands[*]>name", + "B04" + ); + + assertTrue(predicate.test(asset)); + } + + // negative/null-safety tests: + @Test + void shouldReturnFalseWhenAssetIsNullForAttributePredicate() { + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); + + assertFalse(predicate.test(null)); + } + + @Test + void shouldReturnFalseWhenAssetIsNullForJsonPathPredicate() { + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromJsonPath( + "eo:bands[*]>name", + "B04" + ); + + assertFalse(predicate.test(null)); + } + + @Test + void shouldRejectUnsupportedAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( + "unsupportedAttribute", + "value" + ) + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } + + @Test + void shouldRejectEmptyAssetAttributeName() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( + " ", + "value" + ) + ); + + assertTrue(exception.getMessage().contains("Asset attribute name cannot be empty")); + } + + +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java deleted file mode 100644 index f66774e9a..000000000 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java +++ /dev/null @@ -1,254 +0,0 @@ -package org.integratedmodelling.klab.ogc.stac.test; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.integratedmodelling.klab.stac.STACPathExpression; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -class STACPathExpressionTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static JsonNode testJson() throws Exception { - String json = """ - { - "node1": { - "node2": [ - { - "name": "something", - "count": 10, - "active": true - }, - { - "name": "other", - "count": 20, - "active": false - } - ] - }, - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - }, - { - "name": "B08", - "common_name": "nir" - } - ], - "properties": { - "cloud_cover": 12.5, - "enabled": true - } - } - """; - - return MAPPER.readTree(json); - } - - @Test - void shouldResolveIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "something")); - assertFalse(expression.matches(root, "other")); - } - - @Test - void shouldResolveSecondIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[1]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("other", resolved.get(0).asText()); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "something")); - } - - @Test - void shouldResolveWildcardArrayElements() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(2, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertEquals("other", resolved.get(1).asText()); - - assertTrue(expression.matches(root, "something")); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "missing")); - } - - @Test - void shouldMatchEoBandsUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(3, resolved.size()); - assertEquals("B01", resolved.get(0).asText()); - assertEquals("B04", resolved.get(1).asText()); - assertEquals("B08", resolved.get(2).asText()); - - assertTrue(expression.matches(root, "B01")); - assertTrue(expression.matches(root, "B04")); - assertTrue(expression.matches(root, "B08")); - assertFalse(expression.matches(root, "B99")); - } - - @Test - void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>common_name"); - - assertTrue(expression.matches(root, "coastal")); - assertTrue(expression.matches(root, "red")); - assertTrue(expression.matches(root, "nir")); - assertFalse(expression.matches(root, "green")); - } - - @Test - void shouldMatchNumbers() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>count"); - - assertTrue(expression.matches(root, "10")); - assertTrue(expression.matches(root, "10.0")); - assertFalse(expression.matches(root, "20")); - assertFalse(expression.matches(root, "not-a-number")); - } - - @Test - void shouldMatchBooleans() throws Exception { - JsonNode root = testJson(); - - STACPathExpression firstExpression = - STACPathExpression.parse("node1>node2[0]>active"); - - STACPathExpression secondExpression = - STACPathExpression.parse("node1>node2[1]>active"); - - assertTrue(firstExpression.matches(root, "true")); - assertTrue(firstExpression.matches(root, "TRUE")); - assertFalse(firstExpression.matches(root, "false")); - - assertTrue(secondExpression.matches(root, "false")); - assertTrue(secondExpression.matches(root, "FALSE")); - assertFalse(secondExpression.matches(root, "true")); - } - - @Test - void shouldMatchTopLevelProperties() throws Exception { - JsonNode root = testJson(); - - STACPathExpression cloudCoverExpression = - STACPathExpression.parse("properties>cloud_cover"); - - STACPathExpression enabledExpression = - STACPathExpression.parse("properties>enabled"); - - assertTrue(cloudCoverExpression.matches(root, "12.5")); - assertTrue(cloudCoverExpression.matches(root, "12.50")); - assertFalse(cloudCoverExpression.matches(root, "13")); - - assertTrue(enabledExpression.matches(root, "true")); - assertFalse(enabledExpression.matches(root, "false")); - } - - @Test - void shouldReturnEmptyListForMissingPath() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>missing"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "anything")); - } - - @Test - void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("properties[*]>cloud_cover"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "12.5")); - } - - @Test - void shouldRejectInvalidWildcardSyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[abc]>name") - ); - - assertTrue(exception.getMessage().contains("Invalid array index")); - } - - @Test - void shouldRejectNegativeArrayIndex() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[-1]>name") - ); - - assertTrue(exception.getMessage().contains("Array index cannot be negative")); - } - - @Test - void shouldRejectUnexpectedCharactersAfterArraySyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[0]extra>name") - ); - - assertTrue(exception.getMessage().contains("Unexpected characters")); - } - - @Test - void shouldRejectEmptyPathElement() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>>name") - ); - - assertTrue(exception.getMessage().contains("Invalid empty path element")); - } -} \ No newline at end of file From 689eb676e676cf378c134fe686b7a6016fc49c32 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 8 May 2026 13:23:34 +0200 Subject: [PATCH 05/28] Working with import and encode, need test it --- .../klab/stac/STACEncoder.java | 108 +++++++----- .../klab/stac/STACImporter.java | 6 +- .../klab/stac/STACPathExpression.java | 162 +++++++++--------- .../klab/stac/STACValidator.java | 47 +++-- .../main/resources/ogc/prototypes/stac.kdl | 10 +- .../STACPathExpressionAndAttributeTest.java | 118 +++++++++---- .../KlabIllegalArgumentException.java | 19 +- 7 files changed, 289 insertions(+), 181 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 7eece7172..1e38894e4 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -2,32 +2,39 @@ import java.io.IOException; import java.io.OutputStream; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Date; -import java.util.function.Predicate; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; -import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.api.data.FeatureSource; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.geotools.api.referencing.crs.CoordinateReferenceSystem; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.processing.Operations; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; import org.hortonmachine.gears.libs.modules.HMRaster.HMRasterWritableBuilder; import org.hortonmachine.gears.libs.modules.HMRaster.MergeMode; import org.hortonmachine.gears.libs.monitor.LogProgressMonitor; -import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.RegionMap; +import org.hortonmachine.gears.utils.crs.CrsUtilities; import org.hortonmachine.gears.utils.geometry.GeometryUtilities; import org.integratedmodelling.klab.Authentication; import org.integratedmodelling.klab.Observables; import org.integratedmodelling.klab.api.data.IGeometry; -import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.IGeometry.Dimension.Type; +import org.integratedmodelling.klab.api.data.IResource; import org.integratedmodelling.klab.api.data.adapters.IKlabData.Builder; import org.integratedmodelling.klab.api.data.adapters.IResourceEncoder; import org.integratedmodelling.klab.api.knowledge.ICodelist; @@ -46,6 +53,7 @@ import org.integratedmodelling.klab.components.time.extents.TimeInstant; import org.integratedmodelling.klab.exceptions.KlabContextualizationException; import org.integratedmodelling.klab.exceptions.KlabIOException; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabIllegalStateException; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; @@ -56,27 +64,16 @@ import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; -import org.integratedmodelling.klab.stac.STACPathExpression.STACAssetPredicate; +import org.integratedmodelling.klab.stac.STACPathExpression.AssetAttribute; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; -import org.geotools.coverage.processing.Operations; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; -import org.geotools.api.feature.simple.SimpleFeature; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.geometry.Position; -import org.geotools.referencing.CRS; -import org.geotools.api.referencing.FactoryException; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; -import java.time.*; -import java.time.format.DateTimeFormatter; import kong.unirest.json.JSONObject; @@ -246,7 +243,6 @@ public void getEncodedData(IResource resource, Map urnParameters String collectionId = collectionData.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - String assetId = resource.getParameters().get("asset", String.class); Integer bandIndex = resource.getParameters().get("band", Integer.class); boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); @@ -325,9 +321,14 @@ public void getEncodedData(IResource resource, Map urnParameters if (collection == null) { scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); } - - Predicate p = STACAssetPredicate.from("eo:bands[*]>name", assetId); - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, true, MergeMode.SUBSTITUTE, lpm); + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); } if (bandIndex != null) { // Which means theat it's a Multi Band COG @@ -410,27 +411,20 @@ public void getEncodedData(IResource resource, Map urnParameters // Allow transform ensures the process to finish, but I would not bet on the resulting // data. final boolean allowTransform = true; - Predicate p = STACAssetPredicate.from(STACPathExpression.PREDICATE_EO_BANDS_NAME, assetId); - - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC + items = items.stream().filter(new Predicate(){ + + @Override + public boolean test(HMStacItem item) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } return false; } }).collect(Collectors.toList()); @@ -442,11 +436,17 @@ public boolean test(HMStacItem item) { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } Set EPSGAtAssets = items.stream() .flatMap(item -> item.getAssets().stream()) - .filter(p) + .filter(predicate) .map(HMStacAsset::getEpsg) .collect(Collectors.toUnmodifiableSet()); @@ -455,7 +455,7 @@ public boolean test(HMStacItem item) { scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + ". The transformation process could affect the data."); } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, p, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); @@ -469,6 +469,24 @@ public boolean test(HMStacItem item) { encoder = new RasterEncoder(); ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } + + private Predicate getAssetPredicate(IResource resource){ + String assetId = resource.getParameters().get("asset", String.class); + if (assetId != null) { + return STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute(AssetAttribute.ID.name(), assetId); + } + String jsonSelector = resource.getParameters().get("jsonSelector", String.class); + String jsonValue = resource.getParameters().get("jsonValue", String.class); + if (jsonSelector != null && !jsonSelector.isBlank() && jsonValue != null) { + try { + return STACPathExpression.STACAssetPredicate.fromHMStacAsset(jsonSelector, jsonValue); + } catch (IllegalArgumentException e) { + throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector, e); + } + } else { + throw new KlabIllegalArgumentException("Search parameters didn't exists"); + } + } private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { Date datetime = (Date) f.getAttribute("datetime"); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f509..10e324eb5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -43,6 +43,10 @@ public boolean acceptsMultiple() { // TODO Auto-generated method stub return false; } + + private boolean hasAssetSelector(IParameters parameters) { + return (parameters.contains("asset")||(parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) throws MalformedURLException { @@ -59,7 +63,7 @@ private void importCollection(List ret, IParameters parameters, boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); - if (!parameters.contains("asset") && !isBulkImport) { + if (!hasAssetSelector(parameters) && !isBulkImport) { Builder builder = buildResource(parameters, project, monitor, collectionId); if (builder != null) { ret.add(builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 6bdd725ef..c34270bf6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -1,20 +1,31 @@ package org.integratedmodelling.klab.stac; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import kong.unirest.json.JSONObject; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Function; import java.util.function.Predicate; import org.hortonmachine.gears.io.stac.HMStacAsset; public final class STACPathExpression { - private final List path; - - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + private final List path; + + /* + * This works: eo:bands[*]>name + * This probably does not behave as a user might expect: eo:bands>name + * because eo:bands resolves to an array node, and then the next path part tries to read field name directly from an array. + * That returns nothing. That is acceptable, but it should be documented: arrays require either [n] or [*]. + */ + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; private STACPathExpression(List path) { this.path = path; @@ -24,14 +35,13 @@ public static STACPathExpression parse(String jsonPath) { if (jsonPath == null || jsonPath.isBlank()) { throw new IllegalArgumentException("JSON path cannot be empty"); } - return new STACPathExpression(parsePath(jsonPath)); } public boolean matches(JsonNode root, String expectedValue) { List resolvedNodes = resolve(root); - for (JsonNode node : resolvedNodes) { + for(JsonNode node : resolvedNodes) { if (jsonValueEquals(node, expectedValue)) { return true; } @@ -44,10 +54,10 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for (PathPart part : path) { + for(PathPart part : path) { List nextNodes = new ArrayList<>(); - for (JsonNode current : currentNodes) { + for(JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } @@ -70,7 +80,7 @@ public List resolve(JsonNode root) { } } else if (part.arrayMode() == ArrayMode.WILDCARD) { if (fieldNode.isArray()) { - for (JsonNode arrayElement : fieldNode) { + for(JsonNode arrayElement : fieldNode) { if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { nextNodes.add(arrayElement); } @@ -94,7 +104,7 @@ private static List parsePath(String jsonPath) { List parts = new ArrayList<>(); - for (String token : tokens) { + for(String token : tokens) { String trimmed = token.trim(); if (trimmed.isEmpty()) { @@ -106,7 +116,7 @@ private static List parsePath(String jsonPath) { return parts; } - + private static boolean valueEquals(Object actualValue, String expectedValue) { if (actualValue == null) { return expectedValue == null; @@ -127,17 +137,6 @@ private static boolean valueEquals(Object actualValue, String expectedValue) { return Objects.equals(String.valueOf(actualValue), expectedValue); } - private static boolean numberEquals(Number actualValue, String expectedValue) { - try { - BigDecimal actual = new BigDecimal(actualValue.toString()); - BigDecimal expected = new BigDecimal(expectedValue); - - return actual.compareTo(expected) == 0; - } catch (NumberFormatException e) { - return false; - } - } - private static boolean jsonValueEquals(JsonNode actualValue, String expectedValue) { if (actualValue == null || actualValue.isNull() || actualValue.isMissingNode()) { return expectedValue == null; @@ -148,24 +147,19 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isNumber()) { - return numberEquals(actualValue, expectedValue); + return numberEquals(actualValue.decimalValue(), expectedValue); } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()) - .equalsIgnoreCase(expectedValue); - } - - if (actualValue.isTextual()) { - return Objects.equals(actualValue.asText(), expectedValue); + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); } return Objects.equals(actualValue.asText(), expectedValue); } - private static boolean numberEquals(JsonNode actualValue, String expectedValue) { + private static boolean numberEquals(Number actualValue, String expectedValue) { try { - BigDecimal actual = actualValue.decimalValue(); + BigDecimal actual = new BigDecimal(actualValue.toString()); BigDecimal expected = new BigDecimal(expectedValue); return actual.compareTo(expected) == 0; @@ -173,12 +167,8 @@ private static boolean numberEquals(JsonNode actualValue, String expectedValue) return false; } } - - public record PathPart( - String fieldName, - ArrayMode arrayMode, - Integer arrayIndex - ) { + + public record PathPart(String fieldName, ArrayMode arrayMode, Integer arrayIndex) { public static PathPart parse(String token) { int bracketStart = token.indexOf('['); @@ -190,24 +180,18 @@ public static PathPart parse(String token) { int bracketEnd = token.indexOf(']', bracketStart); if (bracketEnd < 0) { - throw new IllegalArgumentException( - "Invalid array syntax in path element: " + token - ); + throw new IllegalArgumentException("Invalid array syntax in path element: " + token); } if (bracketEnd != token.length() - 1) { - throw new IllegalArgumentException( - "Unexpected characters after array syntax in path element: " + token - ); + throw new IllegalArgumentException("Unexpected characters after array syntax in path element: " + token); } String fieldName = token.substring(0, bracketStart).trim(); String indexText = token.substring(bracketStart + 1, bracketEnd).trim(); if (fieldName.isEmpty()) { - throw new IllegalArgumentException( - "Field name cannot be empty in path element: " + token - ); + throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); } if ("*".equals(indexText)) { @@ -219,52 +203,72 @@ public static PathPart parse(String token) { try { index = Integer.parseInt(indexText); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "Invalid array index in path element: " + token, - e - ); + throw new IllegalArgumentException("Invalid array index in path element: " + token, e); } if (index < 0) { - throw new IllegalArgumentException( - "Array index cannot be negative in path element: " + token - ); + throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); } return new PathPart(fieldName, ArrayMode.INDEX, index); } } - + public enum ArrayMode { - NONE, - INDEX, - WILDCARD + NONE, INDEX, WILDCARD } - - public final class STACAssetPredicate { + + public static final class STACAssetPredicate { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private STACAssetPredicate() { } - - public static Predicate fromJsonPath( - String jsonPath, - String expectedValue - ) { + + public static Predicate fromJsonPath(String jsonPath, String expectedValue, + Function jsonNodeExtractor) { STACPathExpression expression = STACPathExpression.parse(jsonPath); - return asset -> { - if (asset == null || asset.getAssetNode() == null) { + return object -> { + if (object == null) { + return false; + } + + JsonNode node = jsonNodeExtractor.apply(object); + + if (node == null || node.isNull() || node.isMissingNode()) { return false; } - return expression.matches(asset.getAssetNode(), expectedValue); + return expression.matches(node, expectedValue); }; } - public static Predicate fromAssetAttribute( - String attributeName, - String expectedValue - ) { + public static Predicate fromHMStacAsset(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, asset -> asset == null ? null : asset.getAssetNode()); + } + + public static Predicate fromJsonNode(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, node -> node); + } + + public static Predicate fromKongJsonObject(String jsonPath, String expectedValue) { + return fromJsonPath(jsonPath, expectedValue, STACAssetPredicate::toJsonNode); + } + + private static JsonNode toJsonNode(JSONObject jsonObject) { + if (jsonObject == null) { + return null; + } + + try { + return OBJECT_MAPPER.readTree(jsonObject.toString()); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Cannot convert JSONObject to JsonNode", e); + } + } + + public static Predicate fromHMStacAssetAttribute(String attributeName, String expectedValue) { AssetAttribute attribute = AssetAttribute.fromName(attributeName); return asset -> { @@ -277,14 +281,8 @@ public static Predicate fromAssetAttribute( return valueEquals(actualValue, expectedValue); }; } - - public static Predicate from(String jsonPath, String expectedValue) { - STACPathExpression expression = STACPathExpression.parse(jsonPath); - - return asset -> expression.matches(asset.getAssetNode(), expectedValue); - } } - + public enum AssetAttribute { ID("id") { @@ -342,17 +340,15 @@ public static AssetAttribute fromName(String name) { throw new IllegalArgumentException("Asset attribute name cannot be empty"); } - for (AssetAttribute attribute : values()) { + for(AssetAttribute attribute : values()) { if (attribute.name.equalsIgnoreCase(name.trim())) { return attribute; } } - throw new IllegalArgumentException( - "Unsupported HMStacAsset attribute: " + name - + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason" - ); + throw new IllegalArgumentException("Unsupported HMStacAsset attribute: " + name + + ". Supported attributes are: id, title, type, valid, epsg, nonValidReason"); } } - + } \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac0..61a9ddcf4 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import org.integratedmodelling.kim.api.IParameters; import org.integratedmodelling.klab.api.data.IGeometry; @@ -20,6 +21,7 @@ import org.integratedmodelling.klab.api.runtime.monitoring.IMonitor; import org.integratedmodelling.klab.data.resources.Resource; import org.integratedmodelling.klab.data.resources.ResourceBuilder; +import org.integratedmodelling.klab.exceptions.KlabIllegalArgumentException; import org.integratedmodelling.klab.exceptions.KlabUnimplementedException; import org.integratedmodelling.klab.rest.CodelistReference; import org.integratedmodelling.klab.rest.MappingReference; @@ -62,18 +64,25 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); + generateCodeList(builder, assetId, asset); + } else if (userData.contains("jsonSelector")) { + if (userData.contains("jsonValue")) { + JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + Predicate predicate = + STACPathExpression.STACAssetPredicate.fromKongJsonObject( + userData.get("jsonSelector", String.class), + userData.get("jsonValue", String.class) + ); + + for (String assetId : assets.keySet()) { + JSONObject asset = assets.getJSONObject(assetId); + + if (predicate.test(asset)) { + generateCodeList(builder, assetId, asset); + } } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); + } else { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue have to be filled"); } } @@ -87,6 +96,22 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni return builder; } + private void generateCodeList(Builder builder, String assetId, JSONObject asset) { + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + } + private Type readRasterDataType(JSONObject asset) { if (!asset.has("raster:bands")) { return null; diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d6..5ccdd7c9a 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -15,9 +15,17 @@ optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." + The band index number for a multi-band raster. Default is 0." default 0 + optional text 'jsonSelector' + "JSON selector" + default "" + + optional text 'jsonValue' + "JSON value" + default "" + optional text 'cog' "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" default "" diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java index d24b147f3..8f840105d 100644 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java @@ -18,40 +18,86 @@ class STACPathExpressionAndAttributeTest { private static JsonNode testJson() throws Exception { String json = """ - { - "node1": { - "node2": [ - { - "name": "something", - "count": 10, - "active": true - }, - { - "name": "other", - "count": 20, - "active": false - } - ] - }, - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - }, - { - "name": "B08", - "common_name": "nir" - } - ], - "properties": { - "cloud_cover": 12.5, - "enabled": true - } - } + "{assets": { + "peat_thickness": { + "href": "https://s3.waw4-1.cloudferro.com/ecdc-waw4-1-ekqouvq3otv8hmw0njzuvo0g4dy0ys8r985n7dggjis3erkpn5o/ECDC/Soil/Soil_V2_global_20010101_20211231_peat_thickness.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ], + "Method to produce": "Spatial distribution is based on the prediction using the Quantile Random Forest algorithm, informed by land surface data (soil, climate, organisms, and topography), to develop regional models over six regions for peat thickness. Peat thickness models are based on approximately 27,000 data points. Highest accuracy observed in African peatlands.", + "Link to resource": [ + "https://zenodo.org/records/14183473" + ], + "proj:code": "EPSG:4326", + "proj:geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 180, + -60.007461 + ], + [ + 180, + 88.007948 + ], + [ + -180, + 88.007948 + ], + [ + -180, + -60.007461 + ], + [ + 180, + -60.007461 + ] + ] + ] + }, + "proj:bbox": [ + -180, + -60.007461, + 180, + 88.007948 + ], + "proj:shape": [ + 16477, + 40076 + ], + "proj:transform": [ + 0.008983152841201715, + 0, + -180.004416632, + 0, + -0.008983152841172543, + 88.007948385, + 0, + 0, + 1 + ], + "eo:bands": [ + { + "name": "peat_thickness" + } + ], + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32", + "statistics": { + "minimum": 1.9351852, + "maximum": 1006.69586 + }, + "unit": "cm", + "scale": 1, + "offset": 0 + } + ] + } + }} """; return MAPPER.readTree(json); @@ -62,13 +108,13 @@ void shouldResolveIndexedArrayElement() throws Exception { JsonNode root = testJson(); STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>name"); + STACPathExpression.parse("eo:bands[*]>name"); List resolved = expression.resolve(root); assertEquals(1, resolved.size()); assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "something")); + assertTrue(expression.matches(root, "peat_thickness")); assertFalse(expression.matches(root, "other")); } diff --git a/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java b/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java index cff1a099b..a9a4b028f 100644 --- a/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java +++ b/api/org.integratedmodelling.klab.api/src/org/integratedmodelling/klab/exceptions/KlabIllegalArgumentException.java @@ -39,17 +39,28 @@ public KlabIllegalArgumentException() { /** * Instantiates a new klab illegal status exception. * - * @param arg0 the arg 0 + * @param message the message */ - public KlabIllegalArgumentException(String arg0) { - super(arg0); + public KlabIllegalArgumentException(String message) { + super(message); // TODO Auto-generated constructor stub } + + /** + * Instantiates a new klab illegal status exception. + * + * @param message the message + * @param e the exception + */ + public KlabIllegalArgumentException(String message, Throwable e) { + super(message, e); + // TODO Auto-generated constructor stub + } /** * Instantiates a new klab illegal status exception. * - * @param e the e + * @param e the exception */ public KlabIllegalArgumentException(Throwable e) { super(e); From 06d5d511815ac5f1b03f41dfe94277c6cac31739 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Fri, 8 May 2026 13:42:30 +0200 Subject: [PATCH 06/28] Small improvements, still not test --- .../klab/stac/STACEncoder.java | 5 +++-- .../klab/stac/STACPathExpression.java | 15 ++++++++++++--- .../klab/stac/STACValidator.java | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 1e38894e4..3284d5517 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -473,7 +473,7 @@ public boolean test(HMStacItem item) { private Predicate getAssetPredicate(IResource resource){ String assetId = resource.getParameters().get("asset", String.class); if (assetId != null) { - return STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute(AssetAttribute.ID.name(), assetId); + return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); } String jsonSelector = resource.getParameters().get("jsonSelector", String.class); String jsonValue = resource.getParameters().get("jsonValue", String.class); @@ -484,7 +484,8 @@ private Predicate getAssetPredicate(IResource resource){ throw new KlabIllegalArgumentException("Invalid STAC asset JSON selector: " + jsonSelector, e); } } else { - throw new KlabIllegalArgumentException("Search parameters didn't exists"); + throw new KlabIllegalArgumentException( + "Either asset or both jsonSelector and jsonValue must be provided"); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index c34270bf6..7799ef556 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -142,7 +142,7 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu return expectedValue == null; } - if (expectedValue == null) { + if (expectedValue == null || !actualValue.isValueNode()) { return false; } @@ -151,10 +151,15 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); + return Boolean.toString(actualValue.booleanValue()) + .equalsIgnoreCase(expectedValue); + } + + if (actualValue.isTextual()) { + return Objects.equals(actualValue.asText(), expectedValue); } - return Objects.equals(actualValue.asText(), expectedValue); + return false; } private static boolean numberEquals(Number actualValue, String expectedValue) { @@ -281,6 +286,10 @@ public static Predicate fromHMStacAssetAttribute(String attributeNa return valueEquals(actualValue, expectedValue); }; } + + public static Predicate fromHMStacAssetId(String expectedValue) { + return fromHMStacAssetAttribute("id", expectedValue); + } } public enum AssetAttribute { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 61a9ddcf4..041c5243e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -82,7 +82,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni } } } else { - throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue have to be filled"); + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); } } From 8653a22f3aab7887f16a3e5fde2144e96c7e6d74 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Sat, 9 May 2026 11:43:55 +0200 Subject: [PATCH 07/28] adds support to get the features from stac, and to rasterize based on presence --- .../klab/stac/STACEncoder.java | 83 +++++++----- .../klab/stac/STACValidator.java | 35 ++--- .../stac/extensions/STACFeatureExtension.java | 121 ++++++++++++++++++ .../main/resources/ogc/prototypes/stac.kdl | 10 +- .../src/main/resources/ogc/prototypes/wfs.kdl | 2 +- 5 files changed, 196 insertions(+), 55 deletions(-) create mode 100644 adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9c331a850..e227cf93e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -12,6 +12,7 @@ import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.api.data.FeatureSource; import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; import org.hortonmachine.gears.io.stac.HMStacItem; @@ -58,23 +59,31 @@ import org.integratedmodelling.klab.scale.Scale; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; +import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; import org.geotools.coverage.processing.Operations; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; import org.geotools.api.feature.simple.SimpleFeature; import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.api.geometry.Position; -import org.geotools.referencing.CRS; -import org.geotools.api.referencing.FactoryException; +import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaOptions.Item; + import java.time.*; import java.time.format.DateTimeFormatter; +import org.locationtech.jts.operation.union.UnaryUnionOp; +import org.geotools.feature.simple.SimpleFeatureBuilder; +import org.geotools.feature.simple.SimpleFeatureTypeBuilder; +import org.geotools.data.DataUtilities; +import java.util.ArrayList; import kong.unirest.json.JSONObject; @@ -246,7 +255,6 @@ public void getEncodedData(IResource resource, Map urnParameters JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); String assetId = resource.getParameters().get("asset", String.class); Integer bandIndex = resource.getParameters().get("band", Integer.class); - boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); // This is part of a WIP that will be removed in the future boolean isIIASA = catalogUrl.contains("iiasa.blob"); @@ -266,8 +274,8 @@ public void getEncodedData(IResource resource, Map urnParameters ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } - - // These are the static STAC catalogs + + // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); Time time2 = time; //TODO make the time and query time different @@ -355,7 +363,8 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); return; } - + + System.out.println("Found the Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -385,10 +394,26 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } ITimeInstant start = time.getStart(); ITimeInstant end = time.getEnd(); + + if (assetId == null) { + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); + } catch (Exception e) { + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + return; + } + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - + GridCoverage2D coverage = null; try { + System.out.println("Searcing STAC.."); List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -422,7 +447,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou var p = new Predicate() { @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { var bandsArray = (ArrayNode) bands; @@ -438,34 +463,30 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou }; // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(new Predicate() { - - @Override - public boolean test(HMStacItem item) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { return true; } - return false; - } - }).collect(Collectors.toList()); - + items = items.stream().filter(item -> { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; + }).collect(Collectors.toList()); if (items.size() == 0) { throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } + // Once the support for customized predicate is added, we can apply for features as well + Set EPSGAtAssets = items.stream() diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index a01646ac0..26219e6a5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -59,28 +59,29 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); + JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); + } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); } - } - + if (userData.contains("cog")) { if (userData.get("cog") != null) { builder.withType(Type.NUMBER); - } + } + } readMetadata(collectionData, builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java new file mode 100644 index 000000000..bc43afe08 --- /dev/null +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -0,0 +1,121 @@ +package org.integratedmodelling.klab.stac.extensions; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.geotools.api.data.FeatureSource; +import org.geotools.data.geojson.GeoJSONReader; +import org.geotools.data.memory.MemoryDataStore; +import org.integratedmodelling.klab.stac.STACUtils; +import org.geotools.api.feature.simple.SimpleFeature; +import org.geotools.api.feature.simple.SimpleFeatureType; +import org.hortonmachine.gears.io.stac.HMStacItem; + +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; +import org.integratedmodelling.klab.api.observations.scale.IScale; +import org.integratedmodelling.klab.api.observations.scale.space.IEnvelope; +import org.integratedmodelling.klab.api.observations.scale.space.IGrid; +import org.integratedmodelling.klab.api.observations.scale.time.ITimeInstant; +import org.integratedmodelling.klab.stac.STACUtils; +import java.time.format.DateTimeFormatter; +import java.time.*; + +public class STACFeatureExtension { + public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { + + System.out.println("Getting features from STAC!"); + String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") + .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); + + List featureList = new ArrayList<>(); + System.out.println(searchEndpoint); + + JSONArray bboxArray = new JSONArray(); + for (Double v : bbox) { + bboxArray.put(v); + } + + + JSONObject searchPayload = new JSONObject() + .put("limit", 100) + .put("bbox", bboxArray) + .put("collections", new JSONArray().put(collectionId)); + + while (searchEndpoint != null) { + + HttpResponse response = Unirest + .post(searchEndpoint) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); + + JSONObject body = response.getBody().getObject(); + System.out.println(body); + + JSONArray features = body.getJSONArray("features"); + System.out.println(features.length()); + + Iterator featureIterator = features.iterator(); + + while (featureIterator.hasNext()) { + try { + JSONObject feature = (JSONObject) featureIterator.next(); + SimpleFeature feat = GeoJSONReader.parseFeature(feature.toString()); + HMStacItem item = HMStacItem.fromSimpleFeature(feat); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + if (item.getStartTimestamp() != null && item.getEndTimestamp() != null) { + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + if (start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd) { + featureList.add(feat); + } + } else { + featureList.add(feat); + } + + + } catch (Exception e) { + e.printStackTrace(); + } + } + searchEndpoint = null; + if (body.has("links")) { + JSONArray links = body.getJSONArray("links"); + + for (Object obj : links) { + JSONObject link = (JSONObject) obj; + + if ("next".equalsIgnoreCase(link.optString("rel"))) { + String searchEndpointNew = link.getString("href"); + if (searchEndpointNew.equals(searchEndpoint)) { + searchEndpoint = null; + } + break; + } + } + } + } + System.out.println(featureList.size()); + SimpleFeatureType type = featureList.get(0).getType(); + MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); + dataStore.addFeatures(featureList); + return dataStore.getFeatureSource(type.getTypeName()); + } +} \ No newline at end of file diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index 0404a17d6..e1d8d2b7c 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -8,19 +8,17 @@ { final text 'collection' - "The URL pointing to the STAC collection file that contains the resource dataset." + "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left it blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. - The band for a multi-band raster. Default is 0." - default 0 + The band for a multi-band raster. Default it will pick up the first one." optional text 'cog' - "Relevant only for Resources served as Cloud Optimized GeoTiff, with Public Access" - default "" + "Relevant only for Resources served as Cloud Optimised GeoTiff, with Public Access" optional enum 'bandmixer' "Relevant only for raster resources.\n diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl index 24e2f0a11..dad0ee466 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/wfs.kdl @@ -35,6 +35,6 @@ optional enum 'axisOrder' "Select the order of the XY axis: 'lat_lon' (default), or 'lon_lat'." - default "lat_lon" + default lat_lon values lat_lon, lon_lat } \ No newline at end of file From 83e4476f54ab709b0f841479c9b608567bf35f15 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Sat, 9 May 2026 11:49:11 +0200 Subject: [PATCH 08/28] removes prints, and adds some sanity checks --- .../org/integratedmodelling/klab/stac/STACEncoder.java | 2 -- .../klab/stac/extensions/STACFeatureExtension.java | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index e227cf93e..6b0296b2d 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -364,7 +364,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou return; } - System.out.println("Found the Search Option!"); LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -413,7 +412,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou GridCoverage2D coverage = null; try { - System.out.println("Searcing STAC.."); List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java index bc43afe08..aa668f566 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/extensions/STACFeatureExtension.java @@ -30,12 +30,10 @@ public class STACFeatureExtension { public static FeatureSource getFeatures(JSONObject catalogData, String collectionId, List bbox, ITimeInstant start, ITimeInstant end) throws Exception { - System.out.println("Getting features from STAC!"); String searchEndpoint = STACUtils.getLinkTo(catalogData, "search") .orElseThrow(() -> new Exception("Search Link not found for the Catalog")); List featureList = new ArrayList<>(); - System.out.println(searchEndpoint); JSONArray bboxArray = new JSONArray(); for (Double v : bbox) { @@ -57,10 +55,7 @@ public static FeatureSource getFeatures(JSONOb .asJson(); JSONObject body = response.getBody().getObject(); - System.out.println(body); - JSONArray features = body.getJSONArray("features"); - System.out.println(features.length()); Iterator featureIterator = features.iterator(); @@ -112,7 +107,10 @@ public static FeatureSource getFeatures(JSONOb } } } - System.out.println(featureList.size()); + if (featureList.isEmpty()) { + throw new Exception("No features found for the given parameters"); + } + SimpleFeatureType type = featureList.get(0).getType(); MemoryDataStore dataStore = new org.geotools.data.memory.MemoryDataStore(type); dataStore.addFeatures(featureList); From 9a814630d7a058dd97cf2cb977c71b66b4c53b37 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 11:00:50 +0200 Subject: [PATCH 09/28] fixes stac validator and importer to allow import --- .../klab/stac/STACAssetMapParser.java | 7 ++- .../klab/stac/STACCollectionParser.java | 38 +++++++++++---- .../klab/stac/STACEncoder.java | 2 +- .../klab/stac/STACImporter.java | 48 +++++++++++++++++-- .../klab/stac/STACValidator.java | 4 +- 5 files changed, 83 insertions(+), 16 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java index 3be8c4a50..7dacca18b 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACAssetMapParser.java @@ -10,6 +10,11 @@ public static Set readAssetNames(JSONObject assets) { } public static JSONObject getAsset(JSONObject assetMap, String assetId) { - return assetMap.getJSONObject(assetId); + /* + * Not failing if the assetId is not found, the reason being that in + * some collections the items have different assets, and the Asset naming related conventions that MSFT + * Planetary follows is not necessarily followed by evryone! + */ + return assetMap.optJSONObject(assetId); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index c351c324c..a7bcb0915 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -16,6 +16,7 @@ import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; +import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; public class STACCollectionParser { @@ -62,7 +63,7 @@ public static IGeometry readGeometry(JSONObject collection) { * @return The asset list as a JSON * @throws KlabResourceAccessException */ - public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObject collection) throws KlabResourceAccessException { + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) throws KlabResourceAccessException { String collectionId = collection.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); @@ -91,21 +92,42 @@ public static JSONObject readAssetsFromCollection(String collectionUrl, JSONObje return itemData.getJSONObject("assets"); } throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } + } - // TODO Move the query to another place. - String parameters = "?collections=" + collectionId + "&limit=1"; - HttpResponse response = Unirest.get(searchEndpoint.get() + parameters).asJson(); + JSONObject searchPayload = new JSONObject() + .put("limit", 100) + .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) + .put("collections", new JSONArray().put(collectionId)); + + HttpResponse response = Unirest + .post(searchEndpoint.get()) + .header("Content-Type", "application/json") + .body(searchPayload) + .asJson(); if (!response.isSuccess()) { - throw new KlabResourceAccessException(); //TODO set message + throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message } JSONObject searchResponse = response.getBody().getObject(); if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException(); // TODO set message there is no feature + throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + } + + JSONArray features = searchResponse.getJSONArray("features"); + + for (int i = 0; i < features.length(); i++) { + JSONObject feature = features.getJSONObject(i); + + JSONObject assetInfo = feature + .getJSONObject("assets") + .optJSONObject(assetId); + + if (assetInfo != null) { + return assetInfo; + } } - return searchResponse.getJSONArray("features").getJSONObject(0).getJSONObject("assets"); + throw new KlabResourceAccessException("No Asset with ID: " + assetId + " was found in the collection"); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 6b0296b2d..4e7dd55f7 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -53,7 +53,7 @@ import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.ogc.STACAdapter; -import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; +import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index dc768f509..1f085b4ed 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -68,7 +68,42 @@ private void importCollection(List ret, IParameters parameters, } return; } - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); + + String assetId = parameters.get("asset"); + JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + } + + + parameters.put("asset", assetId); + String resourceUrn = collectionId + "-" + assetId; + Builder builder = buildResource(parameters, project, monitor, resourceUrn); + if (builder != null) { + ret.add(builder); + } else { + monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); + } + + // Think of a way to do the REGEX matching. Possibly not worth it to support + // this for something as diverse as STAC + + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { if (regex != null && !assetId.matches(regex)) { @@ -77,10 +112,13 @@ private void importCollection(List ret, IParameters parameters, } JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); - if (!STACAssetParser.isSupportedMediaType(assetData)) { - Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); - continue; + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + Logging.INSTANCE.info("Asset " + assetId + " doesn't have a supported media type, skipped"); + continue; + } } + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; String href = assetData.getString("href"); @@ -97,6 +135,8 @@ private void importCollection(List ret, IParameters parameters, monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } } + + */ } private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 26219e6a5..4b73caf0e 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -59,8 +59,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String assetId = userData.get("asset", String.class); - JSONObject assets = STACCollectionParser.readAssetsFromCollection(collectionUrl, collectionData); - JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); + JSONObject asset = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + //JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); Type type = readRasterDataType(asset); // Currently, only files:values is supported. If needed, the classification extension could be used too. From a27c8de7bc2fba92b7cad577e14cdf7ddaebd825 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 11:35:41 +0200 Subject: [PATCH 10/28] adds conversion to str --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 1f085b4ed..5b6e8c6c9 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -45,7 +45,7 @@ public boolean acceptsMultiple() { } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) - throws MalformedURLException { + throws Exception { String collectionUrl = parameters.get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = STACCollectionParser.readCollectionId(collectionData); @@ -69,7 +69,7 @@ private void importCollection(List ret, IParameters parameters, return; } - String assetId = parameters.get("asset"); + String assetId = (String) parameters.get("asset"); JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); /* From b453fd1e1b9ac7ed202563a1c70a997bd65ba48a Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Mon, 11 May 2026 11:53:38 +0200 Subject: [PATCH 11/28] Changes to assume assets is not an array but a list of json object --- .../klab/stac/STACPathExpression.java | 135 ++-- .../STACPathExpressionAndAttributeTest.java | 591 ------------------ .../ogc/stac/test/STACPathExpressionTest.java | 407 ++++++++++++ 3 files changed, 496 insertions(+), 637 deletions(-) delete mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java create mode 100644 adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 7799ef556..0a16ae462 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.TextNode; import kong.unirest.json.JSONObject; @@ -19,13 +20,8 @@ public final class STACPathExpression { private final List path; - /* - * This works: eo:bands[*]>name - * This probably does not behave as a user might expect: eo:bands>name - * because eo:bands resolves to an array node, and then the next path part tries to read field name directly from an array. - * That returns nothing. That is acceptable, but it should be documented: arrays require either [n] or [*]. - */ - public static final String PREDICATE_EO_BANDS_NAME = "eo:bands[*]>name"; + public static final String PREDICATE_EO_BANDS_NAME = "eo:bands.name"; + public static final String MEDIA_TYPE_BANDS_NAME = "type"; private STACPathExpression(List path) { this.path = path; @@ -54,39 +50,15 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for(PathPart part : path) { + for (PathPart part : path) { List nextNodes = new ArrayList<>(); - for(JsonNode current : currentNodes) { + for (JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } - JsonNode fieldNode = current.get(part.fieldName()); - - if (fieldNode == null || fieldNode.isNull() || fieldNode.isMissingNode()) { - continue; - } - - if (part.arrayMode() == ArrayMode.NONE) { - nextNodes.add(fieldNode); - } else if (part.arrayMode() == ArrayMode.INDEX) { - if (fieldNode.isArray()) { - JsonNode indexedNode = fieldNode.get(part.arrayIndex()); - - if (indexedNode != null && !indexedNode.isNull() && !indexedNode.isMissingNode()) { - nextNodes.add(indexedNode); - } - } - } else if (part.arrayMode() == ArrayMode.WILDCARD) { - if (fieldNode.isArray()) { - for(JsonNode arrayElement : fieldNode) { - if (arrayElement != null && !arrayElement.isNull() && !arrayElement.isMissingNode()) { - nextNodes.add(arrayElement); - } - } - } - } + resolvePart(current, part, nextNodes); } currentNodes = nextNodes; @@ -99,8 +71,87 @@ public List resolve(JsonNode root) { return currentNodes; } + private static void resolvePart(JsonNode current, PathPart part, List nextNodes) { + if (current.isArray()) { + resolveFromArray(current, part, nextNodes); + } else if (current.isObject()) { + resolveFromObject(current, part, nextNodes); + } + } + + private static void resolveFromArray(JsonNode arrayNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + JsonNode indexedNode = arrayNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + resolvePart(indexedNode, new PathPart(part.fieldName(), ArrayMode.NONE, null), nextNodes); + } + + + } else { + for (JsonNode element : arrayNode) { + if (isUsable(element)) { + resolvePart(element, part, nextNodes); + } + } + } + } + + private static void resolveFromObject(JsonNode objectNode, PathPart part, List nextNodes) { + JsonNode directField = objectNode.get(part.fieldName()); + + if (isUsable(directField)) { + addResolvedField(directField, part, nextNodes); + } else { + objectNode.fields().forEachRemaining(entry -> { + String mapKey = entry.getKey(); + JsonNode mapValue = entry.getValue(); + + if (!isUsable(mapValue) || !mapValue.isObject()) { + return; + } + + if ("id".equalsIgnoreCase(part.fieldName())) { + nextNodes.add(TextNode.valueOf(mapKey)); + return; + } + + JsonNode nestedField = mapValue.get(part.fieldName()); + + if (isUsable(nestedField)) { + addResolvedField(nestedField, part, nextNodes); + } + }); + + } + } + + private static void addResolvedField(JsonNode fieldNode, PathPart part, List nextNodes) { + if (part.arrayMode() == ArrayMode.INDEX) { + if (fieldNode.isArray()) { + JsonNode indexedNode = fieldNode.get(part.arrayIndex()); + + if (isUsable(indexedNode)) { + nextNodes.add(indexedNode); + } + } + } else if (fieldNode.isArray()) { + for (JsonNode element : fieldNode) { + if (isUsable(element)) { + nextNodes.add(element); + } + } + } else { + nextNodes.add(fieldNode); + } + } + + private static boolean isUsable(JsonNode node) { + return node != null && !node.isNull() && !node.isMissingNode(); + } + private static List parsePath(String jsonPath) { - String[] tokens = jsonPath.split(">"); + String[] tokens = jsonPath.split("\\."); List parts = new ArrayList<>(); @@ -151,8 +202,7 @@ private static boolean jsonValueEquals(JsonNode actualValue, String expectedValu } if (actualValue.isBoolean()) { - return Boolean.toString(actualValue.booleanValue()) - .equalsIgnoreCase(expectedValue); + return Boolean.toString(actualValue.booleanValue()).equalsIgnoreCase(expectedValue); } if (actualValue.isTextual()) { @@ -199,28 +249,21 @@ public static PathPart parse(String token) { throw new IllegalArgumentException("Field name cannot be empty in path element: " + token); } - if ("*".equals(indexText)) { - return new PathPart(fieldName, ArrayMode.WILDCARD, null); - } - int index; - try { index = Integer.parseInt(indexText); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid array index in path element: " + token, e); } - if (index < 0) { throw new IllegalArgumentException("Array index cannot be negative in path element: " + token); } - return new PathPart(fieldName, ArrayMode.INDEX, index); } } public enum ArrayMode { - NONE, INDEX, WILDCARD + NONE, INDEX } public static final class STACAssetPredicate { @@ -286,7 +329,7 @@ public static Predicate fromHMStacAssetAttribute(String attributeNa return valueEquals(actualValue, expectedValue); }; } - + public static Predicate fromHMStacAssetId(String expectedValue) { return fromHMStacAssetAttribute("id", expectedValue); } diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java deleted file mode 100644 index 8f840105d..000000000 --- a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionAndAttributeTest.java +++ /dev/null @@ -1,591 +0,0 @@ -package org.integratedmodelling.klab.ogc.stac.test; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.hortonmachine.gears.io.stac.HMStacAsset; -import org.integratedmodelling.klab.stac.STACPathExpression; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.function.Predicate; - -import static org.junit.jupiter.api.Assertions.*; - -class STACPathExpressionAndAttributeTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static JsonNode testJson() throws Exception { - String json = """ - "{assets": { - "peat_thickness": { - "href": "https://s3.waw4-1.cloudferro.com/ecdc-waw4-1-ekqouvq3otv8hmw0njzuvo0g4dy0ys8r985n7dggjis3erkpn5o/ECDC/Soil/Soil_V2_global_20010101_20211231_peat_thickness.tif", - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "roles": [ - "data" - ], - "Method to produce": "Spatial distribution is based on the prediction using the Quantile Random Forest algorithm, informed by land surface data (soil, climate, organisms, and topography), to develop regional models over six regions for peat thickness. Peat thickness models are based on approximately 27,000 data points. Highest accuracy observed in African peatlands.", - "Link to resource": [ - "https://zenodo.org/records/14183473" - ], - "proj:code": "EPSG:4326", - "proj:geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - 180, - -60.007461 - ], - [ - 180, - 88.007948 - ], - [ - -180, - 88.007948 - ], - [ - -180, - -60.007461 - ], - [ - 180, - -60.007461 - ] - ] - ] - }, - "proj:bbox": [ - -180, - -60.007461, - 180, - 88.007948 - ], - "proj:shape": [ - 16477, - 40076 - ], - "proj:transform": [ - 0.008983152841201715, - 0, - -180.004416632, - 0, - -0.008983152841172543, - 88.007948385, - 0, - 0, - 1 - ], - "eo:bands": [ - { - "name": "peat_thickness" - } - ], - "raster:bands": [ - { - "sampling": "area", - "data_type": "float32", - "statistics": { - "minimum": 1.9351852, - "maximum": 1006.69586 - }, - "unit": "cm", - "scale": 1, - "offset": 0 - } - ] - } - }} - """; - - return MAPPER.readTree(json); - } - - @Test - void shouldResolveIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertTrue(expression.matches(root, "peat_thickness")); - assertFalse(expression.matches(root, "other")); - } - - @Test - void shouldResolveSecondIndexedArrayElement() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[1]>name"); - - List resolved = expression.resolve(root); - - assertEquals(1, resolved.size()); - assertEquals("other", resolved.get(0).asText()); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "something")); - } - - @Test - void shouldResolveWildcardArrayElements() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(2, resolved.size()); - assertEquals("something", resolved.get(0).asText()); - assertEquals("other", resolved.get(1).asText()); - - assertTrue(expression.matches(root, "something")); - assertTrue(expression.matches(root, "other")); - assertFalse(expression.matches(root, "missing")); - } - - @Test - void shouldMatchEoBandsUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>name"); - - List resolved = expression.resolve(root); - - assertEquals(3, resolved.size()); - assertEquals("B01", resolved.get(0).asText()); - assertEquals("B04", resolved.get(1).asText()); - assertEquals("B08", resolved.get(2).asText()); - - assertTrue(expression.matches(root, "B01")); - assertTrue(expression.matches(root, "B04")); - assertTrue(expression.matches(root, "B08")); - assertFalse(expression.matches(root, "B99")); - } - - @Test - void shouldMatchEoBandCommonNameUsingWildcard() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("eo:bands[*]>common_name"); - - assertTrue(expression.matches(root, "coastal")); - assertTrue(expression.matches(root, "red")); - assertTrue(expression.matches(root, "nir")); - assertFalse(expression.matches(root, "green")); - } - - @Test - void shouldMatchNumbers() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[0]>count"); - - assertTrue(expression.matches(root, "10")); - assertTrue(expression.matches(root, "10.0")); - assertFalse(expression.matches(root, "20")); - assertFalse(expression.matches(root, "not-a-number")); - } - - @Test - void shouldMatchBooleans() throws Exception { - JsonNode root = testJson(); - - STACPathExpression firstExpression = - STACPathExpression.parse("node1>node2[0]>active"); - - STACPathExpression secondExpression = - STACPathExpression.parse("node1>node2[1]>active"); - - assertTrue(firstExpression.matches(root, "true")); - assertTrue(firstExpression.matches(root, "TRUE")); - assertFalse(firstExpression.matches(root, "false")); - - assertTrue(secondExpression.matches(root, "false")); - assertTrue(secondExpression.matches(root, "FALSE")); - assertFalse(secondExpression.matches(root, "true")); - } - - @Test - void shouldMatchTopLevelProperties() throws Exception { - JsonNode root = testJson(); - - STACPathExpression cloudCoverExpression = - STACPathExpression.parse("properties>cloud_cover"); - - STACPathExpression enabledExpression = - STACPathExpression.parse("properties>enabled"); - - assertTrue(cloudCoverExpression.matches(root, "12.5")); - assertTrue(cloudCoverExpression.matches(root, "12.50")); - assertFalse(cloudCoverExpression.matches(root, "13")); - - assertTrue(enabledExpression.matches(root, "true")); - assertFalse(enabledExpression.matches(root, "false")); - } - - @Test - void shouldReturnEmptyListForMissingPath() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("node1>node2[*]>missing"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "anything")); - } - - @Test - void shouldReturnEmptyListWhenWildcardIsAppliedToNonArray() throws Exception { - JsonNode root = testJson(); - - STACPathExpression expression = - STACPathExpression.parse("properties[*]>cloud_cover"); - - List resolved = expression.resolve(root); - - assertTrue(resolved.isEmpty()); - assertFalse(expression.matches(root, "12.5")); - } - - @Test - void shouldRejectInvalidWildcardSyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[abc]>name") - ); - - assertTrue(exception.getMessage().contains("Invalid array index")); - } - - @Test - void shouldRejectNegativeArrayIndex() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[-1]>name") - ); - - assertTrue(exception.getMessage().contains("Array index cannot be negative")); - } - - @Test - void shouldRejectUnexpectedCharactersAfterArraySyntax() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>node2[0]extra>name") - ); - - assertTrue(exception.getMessage().contains("Unexpected characters")); - } - - @Test - void shouldRejectEmptyPathElement() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.parse("node1>>name") - ); - - assertTrue(exception.getMessage().contains("Invalid empty path element")); - } - - // Test by attributes - - @Test - void shouldMatchAssetByIdAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetByDifferentIdAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B08"); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByTitleAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Surface reflectance band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "title", - "Surface reflectance band 4" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByTypeAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "type", - "image/tiff; application=geotiff" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByEpsgAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByEpsgAttributeUsingDecimalEquivalent() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "32632.0"); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetByDifferentEpsgAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "proj:code": "EPSG:32632" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("epsg", "4326"); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetByValidAttribute() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff" - } - """); - - HMStacAsset asset = new HMStacAsset("B04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("valid", "true"); - - /* - * One caveat: HMStacAsset computes isValid through HMStacAssetHandlers.getHandler(this). - * Depending on your test classpath and supported MIME types, asset.isValid() - * may be either true or false. That is why the validity test uses: - * assertEquals(asset.isValid(), predicate.test(asset)); - */ - assertEquals(asset.isValid(), predicate.test(asset)); - } - - // JSON-path-through-asset tests - @Test - void shouldMatchAssetUsingJsonPathPredicate() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B04" - ); - - assertTrue(predicate.test(asset)); - } - - @Test - void shouldNotMatchAssetUsingJsonPathPredicateWhenValueIsMissing() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B01", - "common_name": "coastal" - }, - { - "name": "B04", - "common_name": "red" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B08" - ); - - assertFalse(predicate.test(asset)); - } - - @Test - void shouldMatchAssetUsingDefaultJsonPathPredicateFactory() throws Exception { - JsonNode assetNode = MAPPER.readTree(""" - { - "title": "Band 4", - "type": "image/tiff; application=geotiff", - "eo:bands": [ - { - "name": "B04" - } - ] - } - """); - - HMStacAsset asset = new HMStacAsset("asset-b04", assetNode); - - Predicate predicate = - STACPathExpression.STACAssetPredicate.from( - "eo:bands[*]>name", - "B04" - ); - - assertTrue(predicate.test(asset)); - } - - // negative/null-safety tests: - @Test - void shouldReturnFalseWhenAssetIsNullForAttributePredicate() { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromAssetAttribute("id", "B04"); - - assertFalse(predicate.test(null)); - } - - @Test - void shouldReturnFalseWhenAssetIsNullForJsonPathPredicate() { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromJsonPath( - "eo:bands[*]>name", - "B04" - ); - - assertFalse(predicate.test(null)); - } - - @Test - void shouldRejectUnsupportedAssetAttribute() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( - "unsupportedAttribute", - "value" - ) - ); - - assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); - } - - @Test - void shouldRejectEmptyAssetAttributeName() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, - () -> STACPathExpression.STACAssetPredicate.fromAssetAttribute( - " ", - "value" - ) - ); - - assertTrue(exception.getMessage().contains("Asset attribute name cannot be empty")); - } - - -} \ No newline at end of file diff --git a/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java new file mode 100644 index 000000000..11c75358e --- /dev/null +++ b/adapters/klab.ogc/src/test/java/org/integratedmodelling/klab/ogc/stac/test/STACPathExpressionTest.java @@ -0,0 +1,407 @@ +package org.integratedmodelling.klab.ogc.stac.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.function.Predicate; + +import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.integratedmodelling.klab.stac.STACPathExpression; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +class STACPathExpressionTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static JsonNode stacItemNode() throws Exception { + String json = """ + { + "assets": { + "CHELSA_bio12_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_bio12_1981-2010_V.2.1.tif", + "title": "Climate Annual Precipitation (CHELSA bio12)", + "description": "Annual precipitation data from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "proj:bbox": [ + -180.0, + -90.0, + 180.0, + 90.0 + ], + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "bio12", + "description": "Annual precipitation" + } + ], + "file:size": 2608605060, + "raster:bands": [ + { + "nodata": -99999, + "data_type": "int32" + } + ] + }, + "CHELSA_gdd5_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gdd5_1981-2010_V.2.1.tif", + "title": "Climate Growing Degree Days > 5°C (CHELSA gdd5)", + "description": "Growing degree days (T > 5°C) from the CHELSA Climate dataset for the period 1981-2010", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gdd5", + "description": "Growing degree days with temperature > 5°C" + } + ], + "file:size": 3520234749, + "raster:bands": [ + { + "nodata": 2147483647, + "data_type": "int32" + } + ] + }, + "CHELSA_gsp_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gsp_1981-2010_V.2.1.tif", + "title": "Climate Precipitation in Growing Season (CHELSA gsp)", + "description": "Precipitation during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:3857", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gsp", + "description": "Precipitation in growing season" + } + ], + "file:size": 2447532198, + "raster:bands": [ + { + "nodata": 2147483648, + "data_type": "float32" + } + ] + }, + "CHELSA_gst_1981-2010_V.2.1": { + "href": "s3://bucket/CHELSA_gst_1981-2010_V.2.1.tif", + "title": "Climate Mean Temperature in Growing Season (CHELSA gst)", + "description": "Mean temperature during the growing season from the CHELSA Climate dataset for the period 1981-2010.", + "type": "image/tiff", + "proj:code": "EPSG:4326", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "gst", + "description": "Mean temperature in growing season" + } + ], + "file:size": 308275230, + "raster:bands": [ + { + "nodata": 65535, + "data_type": "int32" + } + ] + } + } + } + """; + + return OBJECT_MAPPER.readTree(json); + } + + private static JsonNode assetNode(String assetId) throws Exception { + return stacItemNode().get("assets").get(assetId); + } + + private static HMStacAsset bio12Asset() throws Exception { + return new HMStacAsset( + "CHELSA_bio12_1981-2010_V.2.1", + assetNode("CHELSA_bio12_1981-2010_V.2.1") + ); + } + + private static HMStacAsset gstAsset() throws Exception { + return new HMStacAsset( + "CHELSA_gst_1981-2010_V.2.1", + assetNode("CHELSA_gst_1981-2010_V.2.1") + ); + } + + @Test + void stacPathExpressionMatchesAssetByVirtualIdFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.id"); + + assertTrue(expression.matches(root, "CHELSA_gst_1981-2010_V.2.1")); + assertTrue(expression.matches(root, "CHELSA_bio12_1981-2010_V.2.1")); + assertFalse(expression.matches(root, "CHELSA_unknown")); + } + + @Test + void stacPathExpressionResolvesAllAssetIdsFromAssetsObjectMapKeys() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.id").resolve(root); + + assertEquals(4, resolved.size()); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_bio12_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gdd5_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gsp_1981-2010_V.2.1"))); + assertTrue(resolved.stream().anyMatch(node -> node.asText().equals("CHELSA_gst_1981-2010_V.2.1"))); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingImplicitArrayExpansion() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands.name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gdd5")); + assertTrue(expression.matches(root, "gsp")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesEoBandNameUsingExplicitArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.eo:bands[0].name"); + + assertTrue(expression.matches(root, "bio12")); + assertTrue(expression.matches(root, "gst")); + assertFalse(expression.matches(root, "unknown-band")); + } + + @Test + void stacPathExpressionMatchesAssetMediaTypeFromAssetsObjectMap() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.type"); + + assertTrue(expression.matches(root, "image/tiff")); + assertFalse(expression.matches(root, "application/json")); + } + + @Test + void stacPathExpressionMatchesRasterBandDataTypeUsingArrayIndex() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.raster:bands[0].data_type"); + + assertTrue(expression.matches(root, "int32")); + assertTrue(expression.matches(root, "float32")); + assertFalse(expression.matches(root, "uint8")); + } + + @Test + void stacPathExpressionMatchesNumericValuesUsingNumericComparison() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets.file:size"); + + assertTrue(expression.matches(root, "2608605060")); + assertTrue(expression.matches(root, "308275230.0")); + assertFalse(expression.matches(root, "123")); + } + + @Test + void hmStacAssetPredicateMatchesJsonExpressionAgainstAssetNode() throws Exception { + HMStacAsset asset = bio12Asset(); + + Predicate bandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate indexedBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands[0].name", "bio12"); + + Predicate typePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("type", "image/tiff"); + + Predicate rasterDataTypePredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("raster:bands[0].data_type", "int32"); + + assertTrue(bandPredicate.test(asset)); + assertTrue(indexedBandPredicate.test(asset)); + assertTrue(typePredicate.test(asset)); + assertTrue(rasterDataTypePredicate.test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "gst") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("type", "application/json") + .test(asset)); + } + + @Test + void hmStacAssetPredicateMatchesJavaAssetAttributes() throws Exception { + HMStacAsset asset = bio12Asset(); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Climate Annual Precipitation (CHELSA bio12)") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "image/tiff") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "4326.0") + .test(asset)); + + assertTrue(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("valid", Boolean.toString(asset.isValid())) + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("title", "Wrong title") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("type", "application/json") + .test(asset)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("epsg", "3857") + .test(asset)); + } + + @Test + void hmStacAssetAttributePredicateDistinguishesDifferentAssets() throws Exception { + HMStacAsset bio12 = bio12Asset(); + HMStacAsset gst = gstAsset(); + + Predicate bio12IdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1"); + + Predicate gstIdPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAssetId("CHELSA_gst_1981-2010_V.2.1"); + + Predicate bio12BandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "bio12"); + + Predicate gstBandPredicate = + STACPathExpression.STACAssetPredicate.fromHMStacAsset("eo:bands.name", "gst"); + + assertTrue(bio12IdPredicate.test(bio12)); + assertFalse(bio12IdPredicate.test(gst)); + + assertTrue(gstIdPredicate.test(gst)); + assertFalse(gstIdPredicate.test(bio12)); + + assertTrue(bio12BandPredicate.test(bio12)); + assertFalse(bio12BandPredicate.test(gst)); + + assertTrue(gstBandPredicate.test(gst)); + assertFalse(gstBandPredicate.test(bio12)); + } + + @Test + void predicatesReturnFalseForNullHMStacAsset() { + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAsset("eo:bands.name", "bio12") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetAttribute("id", "CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + + assertFalse(STACPathExpression.STACAssetPredicate + .fromHMStacAssetId("CHELSA_bio12_1981-2010_V.2.1") + .test(null)); + } + + @Test + void stacPathExpressionReturnsEmptyResultForMissingPath() throws Exception { + JsonNode root = stacItemNode(); + + List resolved = STACPathExpression.parse("assets.missing.attribute").resolve(root); + + assertTrue(resolved.isEmpty()); + } + + @Test + void parseRejectsOldWildcardArraySyntax() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[*].name") + ); + + assertTrue(exception.getMessage().contains("Invalid array index")); + } + + @Test + void oldGreaterThanSeparatorDoesNotMatchBecauseDotIsNowTheSeparator() throws Exception { + JsonNode root = stacItemNode(); + + STACPathExpression expression = STACPathExpression.parse("assets>eo:bands>name"); + + assertFalse(expression.matches(root, "bio12")); + } + + @Test + void parseRejectsEmptyPath() { + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(null)); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse("")); + assertThrows(IllegalArgumentException.class, () -> STACPathExpression.parse(" ")); + } + + @Test + void parseRejectsNegativeArrayIndex() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.parse("assets.eo:bands[-1].name") + ); + + assertTrue(exception.getMessage().contains("Array index cannot be negative")); + } + + @Test + void parseRejectsUnsupportedHMStacAssetAttribute() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> STACPathExpression.STACAssetPredicate.fromHMStacAssetAttribute("unsupported", "value") + ); + + assertTrue(exception.getMessage().contains("Unsupported HMStacAsset attribute")); + } +} \ No newline at end of file From 7ea3b47c293d58ce025c7690fe99f96a06c7c8f2 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 14:52:06 +0200 Subject: [PATCH 12/28] adds some try/catch --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index b483f39d7..ec93c0358 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -445,6 +445,7 @@ public void getEncodedData(IResource resource, Map urnParameters return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; }).collect(Collectors.toList()); if (items.size() == 0) { + manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); @@ -474,7 +475,13 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou }; } else if (resource.getParameters().get("jsonSelector", String.class) != null) { // based on the JSON Expression on JSONSelector and JSONValue - assetPredicate = getAssetPredicate(resource); + try { + assetPredicate = getAssetPredicate(resource); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); + } + } else { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); @@ -483,10 +490,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou try { source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); } catch (Exception e) { + manager.close(); throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); } encoder = new VectorEncoder(); ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); return; } // Once the support for customized predicate is added, we can apply for features as well From 7de1fe892a3edff317b76666e4e67d0a928a3e51 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 15:46:18 +0200 Subject: [PATCH 13/28] fixes --- .../klab/stac/STACEncoder.java | 204 +++++++++--------- 1 file changed, 100 insertions(+), 104 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index ec93c0358..8cf223eda 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -5,10 +5,6 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -92,7 +88,7 @@ import org.geotools.feature.simple.SimpleFeatureBuilder; import org.geotools.feature.simple.SimpleFeatureTypeBuilder; import org.geotools.data.DataUtilities; -import java.util.ArrayList; +import java.util.*; import kong.unirest.json.JSONObject; @@ -185,6 +181,9 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor } } + /* + Helper to Sort Items (of type HMStacItem) based on their timestamp + */ private void sortByDate(List items, IMonitor monitor) { if (items.stream().anyMatch(i -> i.getTimestamp() == null)) { throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date."); @@ -224,6 +223,8 @@ public void getEncodedData(IResource resource, Map urnParameters IEnvelope envelope = space.getEnvelope(); List bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + // Only for Backward Compatiability + // A new COG Adapter would be added if (resource.getParameters().get("cog") != null) { COGURL = resource.getParameters().get("cog", String.class); scope.getMonitor().info("Getting requested extent from the COG Asset from url" + COGURL); @@ -263,16 +264,24 @@ public void getEncodedData(IResource resource, Map urnParameters String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collectionData); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); Integer bandIndex = resource.getParameters().get("band", Integer.class); - boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); - // This is part of a WIP that will be removed in the future - boolean isIIASA = catalogUrl.contains("iiasa.blob"); String assetId = resource.getParameters().get("asset", String.class); - - Time time = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time) - .findFirst().orElseThrow(); + boolean hasSearchOption = STACUtils.containsLinkTo(catalogData, "search"); + final boolean allowTransform = true; + Time ctxTime = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow(); Time resourceTime = (Time) Scale.create(resource.getGeometry()).getDimension(Type.TIME); + + Time effectiveTime = ctxTime; + if (resourceTime != null + && resourceTime.getStart() != null + && resourceTime.getEnd() != null + && resourceTime.getCoveredExtent() > 0) { - if (isIIASA) { + effectiveTime = validateTemporalDimension(ctxTime, resourceTime); + } + + + // This is part of a WIP that will be removed in the future + if (catalogUrl.contains("iiasa.blob")) { FeatureSource source; try { source = STACIIASAExtension.getFeatures(collectionData, bbox); @@ -283,27 +292,55 @@ public void getEncodedData(IResource resource, Map urnParameters ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } + + /* + Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue + */ + + Predicate assetPredicate = null; + if (assetId != null) { + assetPredicate = new Predicate() { + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates + var bands = asset.getAssetNode().get("eo:bands"); + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } + return false; + } + }; + } else if (resource.getParameters().get("jsonSelector", String.class) != null) { + // based on the JSON Expression on JSONSelector and JSONValue + try { + assetPredicate = getAssetPredicate(resource); + } catch (Exception e){ + throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); + } + + } // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); - Time time2 = time; //TODO make the time and query time different features = features.stream().filter(f -> { Geometry fGeometry = (Geometry) f.getDefaultGeometry(); return fGeometry.intersects(space.getShape().getJTSGeometry()); }).toList(); - features = features.stream().filter(f -> isFeatureInTimeRange(time2, f)).toList(); - if (features.isEmpty()) { - throw new KlabResourceNotFoundException("There are no items in this context for the collection " + collectionId); - } CoordinateReferenceSystem crs = features.get(0).getFeatureType().getCoordinateReferenceSystem(); if (crs == null) { crs = CrsUtilities.getCrsFromSrid(4326); // We go to the standard } - + var time2 = effectiveTime; // TODO merge with similar code from below IGrid grid = space.getGrid(); - RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); @@ -313,15 +350,17 @@ public void getEncodedData(IResource resource, Map urnParameters RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); // end //TODO - - List items = features.stream().map(f -> { - try { - return HMStacItem.fromSimpleFeature(f); - } catch (Exception e) { - scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); - } - return null; - }).filter(i -> i != null).toList(); + List items = features.stream() + .map(f -> { + try { + return HMStacItem.fromSimpleFeature(f); + } catch (Exception e) { + scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); + return null; + }}) + .filter(Objects::nonNull) + .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) + .toList(); GridCoverage2D coverage = null; @@ -388,14 +427,7 @@ public void getEncodedData(IResource resource, Map urnParameters Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - - - if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { - time = validateTemporalDimension(time, resourceTime); - } - ITimeInstant start = time.getStart(); - ITimeInstant end = time.getEnd(); + collection.setGeometryFilter(poly); //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) GridCoverage2D coverage = null; @@ -426,24 +458,12 @@ public void getEncodedData(IResource resource, Map urnParameters Client s3Client = buildS3Client(bucketRegion); collection.setS3Client(s3Client); } - + var time = effectiveTime; // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC - items = items.stream().filter(item -> { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - return start.getMilliseconds() >= itemStart && end.getMilliseconds() <= itemEnd; - }).collect(Collectors.toList()); + items = items.stream() + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); + if (items.size() == 0) { manager.close(); throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); @@ -452,43 +472,13 @@ public void getEncodedData(IResource resource, Map urnParameters } // Allow transform ensures the process to finish, but I would not bet on the resulting data - Predicate assetPredicate = null; - final boolean allowTransform = true; - if (assetId != null) { - assetPredicate = new Predicate() { - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates - var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } - return false; - } - }; - } else if (resource.getParameters().get("jsonSelector", String.class) != null) { - // based on the JSON Expression on JSONSelector and JSONValue - try { - assetPredicate = getAssetPredicate(resource); - } catch (Exception e) { - manager.close(); - throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); - } - - } else { + if (assetPredicate == null) { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); // Only get the features from STAC Collection, no need to interact with Rasters FeatureSource source; try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, start, end); + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); } catch (Exception e) { manager.close(); throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); @@ -498,6 +488,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.close(); return; } + // Once the support for customized predicate is added, we can apply for features as well Set EPSGAtAssets = items.stream() @@ -550,26 +541,31 @@ private Predicate getAssetPredicate(IResource resource){ } } - private boolean isFeatureInTimeRange(Time time2, SimpleFeature f) { - Date datetime = (Date) f.getAttribute("datetime"); - if (datetime != null) { - if (isDateWithinRange(time2, datetime)) { - return true; - } - } - Date itemStart = (Date) f.getAttribute("start_datetime"); - if (itemStart == null) { - return false; - } - Date itemEnd = (Date) f.getAttribute("end_datetime"); - if (itemEnd == null) { - return itemStart.toInstant().getEpochSecond() <= time2.getStart().getMilliseconds(); - } - if (isDateWithinRange(time2, itemStart) || isDateWithinRange(time2, itemEnd)) { - return true; - } - return false; + /* + To check if an Item (of type HMStacItem) is within a time range + */ + private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + try { + long itemStart = LocalDateTime + .parse(item.getStartTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime + .parse(item.getEndTimestamp(), formatter) + .atZone(ZoneOffset.UTC) + .toInstant() + .toEpochMilli(); + + return startMillis >= itemStart && endMillis <= itemEnd; + + } catch (Exception e) { + e.printStackTrace(); + return false; + } } private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { From 4a40970aceb8a70fd1423f468f42fbbcb9a4b56e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:34:20 +0200 Subject: [PATCH 14/28] epsg at assets getting epsg at asset level if not, then at item level --- .../integratedmodelling/klab/stac/STACEncoder.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 8cf223eda..ea5cade2f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -490,12 +490,17 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } // Once the support for customized predicate is added, we can apply for features as well + + var pred = assetPredicate; Set EPSGAtAssets = items.stream() - .flatMap(item -> item.getAssets().stream()) - .filter(assetPredicate) - .map(HMStacAsset::getEpsg) + .flatMap(item -> item.getAssets().stream() + .filter(pred) + .map(asset -> asset.getEpsg() != null + ? asset.getEpsg() + : item.getEpsg())) .collect(Collectors.toUnmodifiableSet()); + if (EPSGAtAssets.size() > 1) { scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." From 1c853c9e95dfef2c24aa9dc5fbd01a61ac9357f0 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:41:49 +0200 Subject: [PATCH 15/28] finds first, and update the time check --- .../integratedmodelling/klab/stac/STACEncoder.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index ea5cade2f..9f01de4c1 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -496,9 +496,11 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou items.stream() .flatMap(item -> item.getAssets().stream() .filter(pred) + .findFirst() .map(asset -> asset.getEpsg() != null ? asset.getEpsg() - : item.getEpsg())) + : item.getEpsg()) + .stream()) .collect(Collectors.toUnmodifiableSet()); if (EPSGAtAssets.size() > 1) { @@ -552,7 +554,15 @@ To check if an Item (of type HMStacItem) is within a time range */ private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok + } + try { + long itemStart = LocalDateTime .parse(item.getStartTimestamp(), formatter) .atZone(ZoneOffset.UTC) From 44b09a726df187cdb6d22374acdcb9e7b2948f15 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Mon, 11 May 2026 16:52:22 +0200 Subject: [PATCH 16/28] updates string conv --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 100f999f7..055b12cea 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -73,7 +73,7 @@ private void importCollection(List ret, IParameters parameters, return; } - String assetId = (String) parameters.get("asset"); + String assetId = parameters.get("asset", String.class); JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); /* From 4d58c7aecb9d6bfbb80c577e848e6a45b6151f17 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:38:16 +0200 Subject: [PATCH 17/28] avoid making search twice --- .../klab/stac/STACEncoder.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 9f01de4c1..33c43999b 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -432,6 +432,25 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou GridCoverage2D coverage = null; try { + + // Allow transform ensures the process to finish, but I would not bet on the resulting data + if (assetPredicate == null) { + // NO JSONSelector and JSONValue found, NO assetID was passed as well + scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; + try { + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); + } catch (Exception e) { + manager.close(); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + } + encoder = new VectorEncoder(); + ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + manager.close(); + return; + } + List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -470,24 +489,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou } else { scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - - // Allow transform ensures the process to finish, but I would not bet on the resulting data - if (assetPredicate == null) { - // NO JSONSelector and JSONValue found, NO assetID was passed as well - scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); - // Only get the features from STAC Collection, no need to interact with Rasters - FeatureSource source; - try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); - } catch (Exception e) { - manager.close(); - throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); - } - encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); - manager.close(); - return; - } // Once the support for customized predicate is added, we can apply for features as well From e3ba3dcc6570ebee3f4468bc64ab0ecae0b15e14 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:41:05 +0200 Subject: [PATCH 18/28] fix grammar --- adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl index d17441ac1..899adae92 100644 --- a/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl +++ b/adapters/klab.ogc/src/main/resources/ogc/prototypes/stac.kdl @@ -11,7 +11,7 @@ "[REQUIRED] The URL pointing to the STAC collection file that contains the resource dataset." optional text 'asset' - "The asset that is going to be retrieved from the items. Should be left it blank when the information is stored in the feature." + "The asset that is going to be retrieved from the items. Should be left blank when the information is stored in the feature." optional number 'band' "Relevant only for raster resources. From 3a0dfa98b34ca52320d191d9b7eed7b9a953529b Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 14:55:56 +0200 Subject: [PATCH 19/28] remove dual cli logs --- .../org/integratedmodelling/klab/stac/STACEncoder.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 33c43999b..872842377 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -293,6 +293,7 @@ public void getEncodedData(IResource resource, Map urnParameters return; } + /* Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue */ @@ -509,13 +510,6 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou + "." + "The transformation process could affect the data."); } - - if (items.size() == 0) { - manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); - } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); - } HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); From e2eea1751b34221374ffabd0930f7ddf1061421a Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:18:01 +0200 Subject: [PATCH 20/28] fail fast on resource access problems for collection --- .../klab/stac/STACEncoder.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 872842377..836d09373 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -305,16 +305,16 @@ public void getEncodedData(IResource resource, Map urnParameters public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates var bands = asset.getAssetNode().get("eo:bands"); if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } + var bandsArray = (ArrayNode) bands; + for (var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } return false; } }; @@ -419,6 +419,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou if (collection == null) { scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + throw new KlabResourceAccessException("Collection" + resource.getParameters().get("collection", String.class) +" cannot be accessed."); // Fail fast } IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation From 87717976c5b363cd3f1eb573bc112a3fe80d44d8 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:22:40 +0200 Subject: [PATCH 21/28] put under a single try/catch --- .../klab/stac/STACEncoder.java | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 836d09373..920b75837 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -407,33 +407,23 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou try { manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e) { - try { - manager.close(); - } catch (Exception e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl + ". Reason :" + e.getMessage()); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - throw new KlabResourceAccessException("Collection" + resource.getParameters().get("collection", String.class) +" cannot be accessed."); // Fail fast - } - IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation - ? ((Observation) scope.getTargetArtifact()).getObservable() - : null; - HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); - - Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); - Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - - GridCoverage2D coverage = null; - try { + if (collection == null) { + scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + manager.close(); + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail fast + } + + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation + ? ((Observation) scope.getTargetArtifact()).getObservable() + : null; + HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); + collection.setGeometryFilter(poly); + //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) + + GridCoverage2D coverage = null; // Allow transform ensures the process to finish, but I would not bet on the resulting data if (assetPredicate == null) { From 7b922c1d9dc58c812db9836a28ae91eab9fc483e Mon Sep 17 00:00:00 2001 From: AM1729 Date: Tue, 12 May 2026 15:48:00 +0200 Subject: [PATCH 22/28] put under a single try/catch --- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index 920b75837..bd4e5f7e1 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -508,12 +508,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } manager.close(); + encoder = new RasterEncoder(); + ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } - encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } private Predicate getAssetPredicate(IResource resource){ From 57be8c606ba5e1d348d1e63fd9985d3732cc537c Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 12 May 2026 16:47:06 +0200 Subject: [PATCH 23/28] Reorganize code for asset search in import --- .../klab/stac/STACCollectionParser.java | 203 ++++++++---------- .../klab/stac/STACValidator.java | 106 ++++----- 2 files changed, 144 insertions(+), 165 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index a750c5b46..3fbacccc5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -3,7 +3,6 @@ import java.time.Instant; import java.util.*; import java.util.function.Predicate; -import java.util.stream.IntStream; import org.integratedmodelling.klab.api.data.IGeometry; import org.integratedmodelling.klab.common.Geometry; @@ -12,8 +11,6 @@ import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; -import com.fasterxml.jackson.module.jsonSchema.types.ArraySchema.Items; - import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.Unirest; @@ -37,7 +34,7 @@ public static String readCollectionId(JSONObject collection) { */ public static IGeometry readGeometry(JSONObject collection) { GeometryBuilder gBuilder = Geometry.builder(); - + JSONObject extent = collection.getJSONObject("extent"); List bbox = extent.getJSONObject("spatial").getJSONArray("bbox").getJSONArray(0).toList(); gBuilder.space().boundingBox(Double.valueOf(bbox.get(0).toString()), Double.valueOf(bbox.get(1).toString()), @@ -58,90 +55,46 @@ public static IGeometry readGeometry(JSONObject collection) { return gBuilder.build().withProjection(Projection.DEFAULT_PROJECTION_CODE).withTimeType("grid"); } - /** - * Reads the asset of a STAC collection and returns them as a JSON based on Asset Key. - * @param collection as a JSON - * @return The asset matching the assetId as a JSONObject - * @throws KlabResourceAccessException - */ - public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) throws KlabResourceAccessException { - String collectionId = collection.getString("id"); - String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); - JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - - Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") - ? STACUtils.getLinkTo(catalogData, "search") - : STACUtils.getLinkTo(collection, "search"); - - // Static catalogs should have their assets on the Collection - if (searchEndpoint.isEmpty()) { - // Check the assets - if (collection.has("assets")) { - return collection.getJSONObject("assets"); - } - // Try to get the assets from a link that has type `item` - Optional itemHref = STACUtils.getLinkTo(collection, "item"); - if (itemHref.isEmpty()) { - throw new KlabResourceNotFoundException("Cannot find items at STAC collection \"" + collectionUrl + "\""); - } - String itemUrl = itemHref.get().startsWith(".") - ? collectionUrl.replace("collection.json", "") + itemHref.get().replace("./", "") - : itemHref.get(); - // TODO get assets from the item - JSONObject itemData = STACUtils.requestMetadata(itemUrl, "feature"); - if (itemData.has("assets")) { - return itemData.getJSONObject("assets"); - } - throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } - - JSONObject searchPayload = new JSONObject() - .put("limit", 100) - .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) - .put("collections", new JSONArray().put(collectionId)); - - HttpResponse response = Unirest - .post(searchEndpoint.get()) - .header("Content-Type", "application/json") - .body(searchPayload) - .asJson(); - - if (!response.isSuccess()) { - throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message + private static JSONObject findAsset(JSONObject assets, String assetId, Predicate predicate) { + if (assets == null) { + return null; } - JSONObject searchResponse = response.getBody().getObject(); - if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + if (assetId != null) { + JSONObject asset = assets.optJSONObject(assetId); + if (asset == null) { + return null; + } + JSONObject result = new JSONObject(); + result.put(assetId, asset); + return result; } - - JSONArray features = searchResponse.getJSONArray("features"); - - for (int i = 0; i < features.length(); i++) { - JSONObject feature = features.getJSONObject(i); - - JSONObject assetInfo = feature - .getJSONObject("assets") - .optJSONObject(assetId); - if (assetInfo != null) { - return assetInfo; - } + if (predicate != null) { + return assets.keySet().stream().map(key -> { + JSONObject asset = assets.optJSONObject(key); + if (asset == null || !predicate.test(asset)) { + return null; + } + JSONObject result = new JSONObject(); + result.put(key, asset); + return result; + }).filter(Objects::nonNull).findFirst().orElse(null); } - throw new KlabResourceAccessException("No Asset with ID: " + assetId + " was found in the collection"); + return null; } - - /* - Retrieves the asset matching the Predicate - */ - public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, Predicate p) throws KlabResourceAccessException { + private static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId, + Predicate predicate) throws KlabResourceAccessException { + if ((assetId == null) == (predicate == null)) { + throw new KlabResourceAccessException("Exactly one of assetId or predicate must be provided"); + } String collectionId = collection.getString("id"); String catalogUrl = STACUtils.getCatalogUrl(collectionUrl, collectionId, collection); JSONObject catalogData = STACUtils.requestMetadata(catalogUrl, "catalog"); - Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") + Optional searchEndpoint = STACUtils.containsLinkTo(catalogData, "search") ? STACUtils.getLinkTo(catalogData, "search") : STACUtils.getLinkTo(collection, "search"); @@ -149,7 +102,10 @@ public static JSONObject readAssetInformationFromCollectionWithPredicate(String if (searchEndpoint.isEmpty()) { // Check the assets if (collection.has("assets")) { - return collection.getJSONObject("assets"); + JSONObject assetInfo = findAsset(collection.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } } // Try to get the assets from a link that has type `item` Optional itemHref = STACUtils.getLinkTo(collection, "item"); @@ -159,55 +115,74 @@ public static JSONObject readAssetInformationFromCollectionWithPredicate(String String itemUrl = itemHref.get().startsWith(".") ? collectionUrl.replace("collection.json", "") + itemHref.get().replace("./", "") : itemHref.get(); - // TODO get assets from the item JSONObject itemData = STACUtils.requestMetadata(itemUrl, "feature"); - if (itemData.has("assets")) { - return itemData.getJSONObject("assets"); + JSONObject assetInfo = findAsset(itemData.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; } throw new KlabResourceNotFoundException("Cannot find assets at STAC collection \"" + collectionUrl + "\""); - } - - JSONObject searchPayload = new JSONObject() - .put("limit", 100) - .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) - .put("collections", new JSONArray().put(collectionId)); - - HttpResponse response = Unirest - .post(searchEndpoint.get()) - .header("Content-Type", "application/json") - .body(searchPayload) - .asJson(); + } + + JSONObject searchPayload = new JSONObject().put("limit", 100) + .put("bbox", new JSONArray().put(-180.0).put(-90.0).put(180.0).put(90.0)) + .put("collections", new JSONArray().put(collectionId)); + + HttpResponse response = Unirest.post(searchEndpoint.get()).header("Content-Type", "application/json") + .body(searchPayload).asJson(); if (!response.isSuccess()) { - throw new KlabResourceAccessException("Unable to import collection, Search failed"); //TODO set message + throw new KlabResourceAccessException("Unable to import collection, Search failed"); } JSONObject searchResponse = response.getBody().getObject(); - if (searchResponse.getJSONArray("features").length() == 0) { - throw new KlabResourceAccessException("No features were found in the collection to be imported"); // TODO set message there is no feature + JSONArray features = searchResponse.optJSONArray("features"); + if (features == null || features.length() == 0) { + throw new KlabResourceAccessException("No features were found in the collection to be imported"); } - - JSONArray features = searchResponse.getJSONArray("features"); - JSONObject matchingFeature = IntStream.range(0, features.length()) - .mapToObj(i -> features.getJSONObject(i)) - .filter(p) - .findFirst() - .orElse(null); + for (int i = 0; i < features.length(); i++) { + JSONObject feature = features.optJSONObject(i); + if (feature == null) { + continue; + } - if (matchingFeature == null) { - throw new KlabResourceAccessException("No Feature found containing an asset matching the predicate based on JSONSelector and JSONValidator was found in the collection"); + JSONObject assetInfo = findAsset(feature.optJSONObject("assets"), assetId, predicate); + if (assetInfo != null) { + return assetInfo; + } } - JSONObject assets = matchingFeature.getJSONObject("assets"); - JSONObject assetInfo = assets.keySet().stream() - .map(assets::getJSONObject) - .filter(asset -> p.test(asset)) - .findFirst() - .orElse(null); - - if (assetInfo == null) { - throw new KlabResourceAccessException("No Asset was found in the matching Fearure"); // shouldn't happen + if (assetId != null) { + throw new KlabResourceAccessException("No asset with ID \"" + assetId + "\" was found in the collection"); } - return assetInfo; + + throw new KlabResourceAccessException("No asset matching the predicate was found in the collection"); + } + + /** + * Reads an asset from a STAC collection by asset key. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param assetId asset key to find inside each feature's assets object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, String assetId) + throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, assetId, null); + } + + /** + * Reads the first asset in a STAC collection whose asset JSON object matches the predicate. + * + * @param collectionUrl URL of the STAC collection + * @param collection collection metadata as JSON + * @param predicate predicate evaluated against each asset JSON object + * @return a JSONObject containing one entry: assetId -> asset JSON object + * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found + */ + public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, + Predicate predicate) throws KlabResourceAccessException { + return readAssetInformationFromCollection(collectionUrl, collection, null, predicate); } } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index b3d10d2f0..eed2dc509 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -45,61 +45,62 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni String collectionUrl = userData.get("collection", String.class); String collectionId = userData.get("collectionId", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); - if (collectionId == null) { + if (collectionId == null) { collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } IGeometry geometry = STACCollectionParser.readGeometry(collectionData); - Builder builder = new ResourceBuilder(urn) - .withParameters(userData) - .withGeometry(geometry) - .withType(Type.OBJECT); + Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); - // The default URL of the resource is the collection endpoint. May be overwritten. + // The default URL of the resource is the collection endpoint. May be overwritten. builder.withMetadata(IMetadata.DC_URL, collectionUrl); - JSONObject asset = null; - String assetId = null; + JSONObject assetNode; if (userData.contains("asset")) { - assetId = userData.get("asset", String.class); - asset = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - //JSONObject asset = STACAssetMapParser.getAsset(assets, assetId); - } else if (userData.contains("jsonSelector")) { - if (userData.contains("jsonValue")) { - Predicate predicate = - STACPathExpression.STACAssetPredicate.fromKongJsonObject( - userData.get("jsonSelector", String.class), - userData.get("jsonValue", String.class) - ); - asset = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, predicate); - assetId = (String) asset.get("id"); // Since the Asset Id is not directly defined - } - } else { - throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); - } - - Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. - Map vals = STACAssetParser.getFileValues(asset); - if (!vals.isEmpty()) { - CodelistReference codelist = populateCodelist(assetId, vals); - if (type == null) { - type = codelist.getType(); - } - builder.addCodeList(codelist); - } - if (type != null) { - builder.withType(type); - } - generateCodeList(builder, assetId, asset); - - if (userData.contains("cog")) { - if (userData.get("cog") != null) { - builder.withType(Type.NUMBER); - } + String requestedAssetId = userData.get("asset", String.class); + + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); + + } else if (userData.contains("jsonSelector")) { + if (!userData.contains("jsonValue")) { + throw new KlabIllegalArgumentException("Both jsonSelector and jsonValue must be provided"); + } + + Predicate predicate = STACPathExpression.STACAssetPredicate + .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); + + assetNode = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, + predicate); + + } else { + throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); + } + + String assetId = assetNode.keys().next(); + JSONObject asset = assetNode.getJSONObject(assetId); + + Type type = readRasterDataType(asset); + // Currently, only files:values is supported. If needed, the classification extension could be used too. + Map vals = STACAssetParser.getFileValues(asset); + if (!vals.isEmpty()) { + CodelistReference codelist = populateCodelist(assetId, vals); + if (type == null) { + type = codelist.getType(); } + builder.addCodeList(codelist); + } + if (type != null) { + builder.withType(type); + } + generateCodeList(builder, assetId, asset); + + if (userData.contains("cog")) { + if (userData.get("cog") != null) { + builder.withType(Type.NUMBER); + } + } readMetadata(collectionData, builder); return builder; @@ -107,7 +108,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni private void generateCodeList(Builder builder, String assetId, JSONObject asset) { Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. Map vals = STACAssetParser.getFileValues(asset); if (!vals.isEmpty()) { CodelistReference codelist = populateCodelist(assetId, vals); @@ -125,7 +127,7 @@ private Type readRasterDataType(JSONObject asset) { if (!asset.has("raster:bands")) { return null; } - + if (asset.getJSONArray("raster:bands").isEmpty() || !asset.getJSONArray("raster:bands").getJSONObject(0).has("data_type")) { // We assume that most rasters are numeric. When in doubt, we set the default to Number @@ -133,7 +135,8 @@ private Type readRasterDataType(JSONObject asset) { } String type = asset.getJSONArray("raster:bands").getJSONObject(0).getString("data_type"); // https://github.com/stac-extensions/raster?tab=readme-ov-file#data-types - final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", "float16", "float32", "float64"); + final Set NUMERIC_DATA_TYPES = Set.of("int8", "int16", "int32", "int64", "uint8", "unit16", "uint32", "uint64", + "float16", "float32", "float64"); if (NUMERIC_DATA_TYPES.contains(type)) { return Type.NUMBER; } @@ -155,8 +158,8 @@ private CodelistReference populateCodelist(String assetId, Map v MappingReference direct = new MappingReference(); MappingReference inverse = new MappingReference(); vals.entrySet().forEach(code -> { - direct.getMappings().add(new Pair<>(code.getKey(), (String)code.getValue())); - codelist.getCodeDescriptions().put(code.getKey(), (String)code.getValue()); + direct.getMappings().add(new Pair<>(code.getKey(), (String) code.getValue())); + codelist.getCodeDescriptions().put(code.getKey(), (String) code.getValue()); }); Type type = STACUtils.inferValueType(vals.entrySet().stream().findFirst().get().getKey()); codelist.setType(type); @@ -166,7 +169,8 @@ private CodelistReference populateCodelist(String assetId, Map v } private void readMetadata(final JSONObject json, Builder builder) { - // We could check the doi only if the Scientific Notation extension is provided, but we can try anyway + // We could check the doi only if the Scientific Notation extension is provided, but we can + // try anyway String doi = STACUtils.readDOI(json); if (doi != null && !doi.isBlank()) { builder.withMetadata(IMetadata.DC_URL, doi); @@ -233,7 +237,7 @@ public Collection getAllFilesForResource(File file) { } @Override - public Map describeResource(IResource resource) { + public Map< ? extends String, ? extends Object> describeResource(IResource resource) { // TODO Auto-generated method stub return null; } From 4df97a89e88a515121731cb40e45a48ec48ac99f Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Tue, 12 May 2026 16:49:00 +0200 Subject: [PATCH 24/28] Align code with changes --- .../java/org/integratedmodelling/klab/stac/STACImporter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index 055b12cea..c8a689020 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -74,8 +74,9 @@ private void importCollection(List ret, IParameters parameters, } String assetId = parameters.get("asset", String.class); - JSONObject assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + JSONObject assetData = assetNode.getJSONObject(assetId); /* * If the particular asset Id wasn't found, then * still proceed to create the resource since it can happen From 2abae11ff08126dd60db1ea947818c1e1a4cf395 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 09:24:25 +0200 Subject: [PATCH 25/28] Rename function to be coherent --- .../org/integratedmodelling/klab/stac/STACCollectionParser.java | 2 +- .../java/org/integratedmodelling/klab/stac/STACValidator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 3fbacccc5..77f1482a5 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -181,7 +181,7 @@ public static JSONObject readAssetInformationFromCollection(String collectionUrl * @return a JSONObject containing one entry: assetId -> asset JSON object * @throws KlabResourceAccessException if the collection cannot be searched or no matching asset is found */ - public static JSONObject readAssetInformationFromCollectionWithPredicate(String collectionUrl, JSONObject collection, + public static JSONObject readAssetInformationFromCollection(String collectionUrl, JSONObject collection, Predicate predicate) throws KlabResourceAccessException { return readAssetInformationFromCollection(collectionUrl, collection, null, predicate); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index eed2dc509..0384fc9c2 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -71,7 +71,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni Predicate predicate = STACPathExpression.STACAssetPredicate .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); - assetNode = STACCollectionParser.readAssetInformationFromCollectionWithPredicate(collectionUrl, collectionData, + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); } else { From 8633543fa0d7dc648d04f13a00874e325ad04108 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 10:50:45 +0200 Subject: [PATCH 26/28] Clean and format code --- .../klab/stac/STACCollectionParser.java | 2 +- .../klab/stac/STACEncoder.java | 403 +++++++++--------- .../klab/stac/STACImporter.java | 51 ++- .../klab/stac/STACPathExpression.java | 9 +- .../klab/stac/STACUtils.java | 22 +- .../klab/stac/STACValidator.java | 6 +- 6 files changed, 235 insertions(+), 258 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java index 77f1482a5..6dd160b9f 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACCollectionParser.java @@ -140,7 +140,7 @@ private static JSONObject readAssetInformationFromCollection(String collectionUr throw new KlabResourceAccessException("No features were found in the collection to be imported"); } - for (int i = 0; i < features.length(); i++) { + for(int i = 0; i < features.length(); i++) { JSONObject feature = features.optJSONObject(i); if (feature == null) { continue; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index bd4e5f7e1..c64465b84 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -5,6 +5,11 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -15,8 +20,8 @@ import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.processing.Operations; import org.geotools.geometry.jts.ReferencedEnvelope; -import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacCollection; import org.hortonmachine.gears.io.stac.HMStacItem; import org.hortonmachine.gears.io.stac.HMStacManager; import org.hortonmachine.gears.libs.modules.HMRaster; @@ -53,28 +58,17 @@ import org.integratedmodelling.klab.exceptions.KlabIllegalStateException; import org.integratedmodelling.klab.exceptions.KlabInternalErrorException; import org.integratedmodelling.klab.exceptions.KlabResourceAccessException; -import org.integratedmodelling.klab.exceptions.KlabResourceNotFoundException; import org.integratedmodelling.klab.exceptions.KlabValidationException; import org.integratedmodelling.klab.ogc.STACAdapter; -import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; +import org.integratedmodelling.klab.ogc.vector.files.VectorEncoder; import org.integratedmodelling.klab.raster.files.RasterEncoder; import org.integratedmodelling.klab.rest.ExternalAuthenticationCredentials; import org.integratedmodelling.klab.scale.Scale; -import org.integratedmodelling.klab.stac.STACPathExpression.AssetAttribute; import org.integratedmodelling.klab.stac.extensions.COGAssetExtension; -import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.stac.extensions.STACFeatureExtension; +import org.integratedmodelling.klab.stac.extensions.STACIIASAExtension; import org.integratedmodelling.klab.utils.s3.S3URLUtils; import org.locationtech.jts.geom.Envelope; -import org.geotools.coverage.processing.Operations; -import org.geotools.data.geojson.GeoJSONReader; -import org.geotools.data.memory.MemoryDataStore; -import org.locationtech.jts.geom.Geometry; -import org.locationtech.jts.geom.Polygon; -import org.geotools.api.feature.simple.SimpleFeature; -import org.geotools.api.feature.simple.SimpleFeatureType; -import org.geotools.data.simple.SimpleFeatureCollection; -import org.geotools.api.referencing.crs.CoordinateReferenceSystem; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; @@ -82,19 +76,11 @@ import com.github.davidmoten.aws.lw.client.Client; import com.github.davidmoten.aws.lw.client.Credentials; -import java.time.*; -import java.time.format.DateTimeFormatter; -import org.locationtech.jts.operation.union.UnaryUnionOp; -import org.geotools.feature.simple.SimpleFeatureBuilder; -import org.geotools.feature.simple.SimpleFeatureTypeBuilder; -import org.geotools.data.DataUtilities; -import java.util.*; - import kong.unirest.json.JSONObject; public class STACEncoder implements IResourceEncoder { - - /** + + /** * The raster or vector encoder that does the actual work after we get our coverage from the service. */ IResourceEncoder encoder; @@ -131,7 +117,8 @@ public ICodelist categorize(IResource resource, String attribute, IMonitor monit private Time refitTime(Time contextTime, Time resourceTime) { if (resourceTime.getCoveredExtent() < contextTime.getCoveredExtent()) { - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } if (contextTime.getStart().isBefore(resourceTime.getStart())) { ITimeInstant newEnd = TimeInstant.create(resourceTime.getStart().getMilliseconds() + contextTime.getLength()); @@ -141,7 +128,8 @@ private Time refitTime(Time contextTime, Time resourceTime) { ITimeInstant newStart = TimeInstant.create(resourceTime.getEnd().getMilliseconds() - contextTime.getLength()); return Time.create(newStart.getMilliseconds(), resourceTime.getEnd().getMilliseconds()); } - throw new KlabContextualizationException("Current observation is outside the bounds of the STAC resource and cannot be reffitted."); + throw new KlabContextualizationException( + "Current observation is outside the bounds of the STAC resource and cannot be reffitted."); } /** @@ -171,7 +159,7 @@ private HMRaster.MergeMode chooseMergeMode(IObservable targetSemantics, IMonitor case NUMBER: if (Observables.INSTANCE.isExtensive(targetSemantics)) { monitor.debug("Using sum as merge mode"); - return HMRaster.MergeMode.SUM; + return HMRaster.MergeMode.SUM; } monitor.debug("Using substitute as merge mode"); return HMRaster.MergeMode.SUBSTITUTE; @@ -189,8 +177,8 @@ private void sortByDate(List items, IMonitor monitor) { throw new KlabIllegalStateException("STAC items are lacking a timestamp and could not be sorted by date."); } items.sort((i1, i2) -> i1.getTimestamp().compareTo(i2.getTimestamp())); - monitor.debug( - "Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + items.get(items.size() - 1).getTimestamp() + "]"); + monitor.debug("Ordered STAC items. First: [" + items.get(0).getTimestamp() + "]; Last [" + + items.get(items.size() - 1).getTimestamp() + "]"); } private Client buildS3Client(String bucketRegion) throws IOException { @@ -201,15 +189,13 @@ private Client buildS3Client(String bucketRegion) throws IOException { } catch (Exception e) { throw new KlabIOException("Error defining S3 credenetials. " + e.getMessage()); } - return Client.s3() - .regionFromEnvironment() // TODO get region from other sources if needed - .credentials(credentials) - .build(); + return Client.s3().regionFromEnvironment() // TODO get region from other sources if needed + .credentials(credentials).build(); } private boolean isDateWithinRange(Time rangeTime, Date date) { Date start = new Date(rangeTime.getStart().getMilliseconds()); - Date end = new Date(rangeTime.getEnd().getMilliseconds()); + Date end = new Date(rangeTime.getEnd().getMilliseconds()); return date.after(start) && date.before(end); } @@ -218,8 +204,7 @@ public void getEncodedData(IResource resource, Map urnParameters IContextualizationScope scope) { String COGURL = null; - Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space) - .findFirst().orElseThrow(); + Space space = (Space) geometry.getDimensions().stream().filter(d -> d instanceof Space).findFirst().orElseThrow(); IEnvelope envelope = space.getEnvelope(); List bbox = List.of(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); @@ -229,35 +214,32 @@ public void getEncodedData(IResource resource, Map urnParameters COGURL = resource.getParameters().get("cog", String.class); scope.getMonitor().info("Getting requested extent from the COG Asset from url" + COGURL); GridCoverage2D coverage = COGAssetExtension.getCOGWindowCoverage(bbox, COGURL); - - String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters().get( - org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, - String.class); - - Projection crs = Projection.create(rcrs); - org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), - bbox.get(1), bbox.get(2), bbox.get(3)); - - HMRaster raster = HMRaster.fromGridCoverage(coverage); - HMRaster outRaster = new HMRasterWritableBuilder().setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, - (int) space.shape()[0], - (int) space.shape()[1])).setCrs(crs.getCoordinateReferenceSystem()) - .setNoValue(raster.getNovalue()) - .build(); - - GridCoverage2D adjCoverage = null; - try { - outRaster.mapRaster(null, raster, null); - adjCoverage = outRaster.buildCoverage(); - } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); - } - - encoder = new RasterEncoder(); + + String rcrs = geometry.getDimension(IGeometry.Dimension.Type.SPACE).getParameters() + .get(org.integratedmodelling.klab.common.Geometry.PARAMETER_SPACE_PROJECTION, String.class); + + Projection crs = Projection.create(rcrs); + org.locationtech.jts.geom.Envelope requestedExtend = new org.locationtech.jts.geom.Envelope(bbox.get(0), bbox.get(1), + bbox.get(2), bbox.get(3)); + + HMRaster raster = HMRaster.fromGridCoverage(coverage); + HMRaster outRaster = new HMRasterWritableBuilder() + .setRegion(RegionMap.fromEnvelopeAndGrid(requestedExtend, (int) space.shape()[0], (int) space.shape()[1])) + .setCrs(crs.getCoordinateReferenceSystem()).setNoValue(raster.getNovalue()).build(); + + GridCoverage2D adjCoverage = null; + try { + outRaster.mapRaster(null, raster, null); + adjCoverage = outRaster.buildCoverage(); + } catch (Exception e) { + throw new KlabResourceAccessException("Cannot build COG Output " + e.getMessage()); + } + + encoder = new RasterEncoder(); ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, adjCoverage, geometry, builder, scope); return; } - + String collectionUrl = resource.getParameters().get("collection", String.class); JSONObject collectionData = STACUtils.requestMetadata(collectionUrl, "collection"); String collectionId = collectionData.getString("id"); @@ -269,52 +251,51 @@ public void getEncodedData(IResource resource, Map urnParameters final boolean allowTransform = true; Time ctxTime = (Time) geometry.getDimensions().stream().filter(d -> d instanceof Time).findFirst().orElseThrow(); Time resourceTime = (Time) Scale.create(resource.getGeometry()).getDimension(Type.TIME); - + Time effectiveTime = ctxTime; - if (resourceTime != null - && resourceTime.getStart() != null - && resourceTime.getEnd() != null + if (resourceTime != null && resourceTime.getStart() != null && resourceTime.getEnd() != null && resourceTime.getCoveredExtent() > 0) { effectiveTime = validateTemporalDimension(ctxTime, resourceTime); } - // This is part of a WIP that will be removed in the future if (catalogUrl.contains("iiasa.blob")) { FeatureSource source; try { source = STACIIASAExtension.getFeatures(collectionData, bbox); } catch (IOException e) { - throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); + throw new KlabResourceAccessException("Cannot extract features from IIASA catalog - " + e.getMessage()); } encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); return; } - - + /* Select the Predicate based on the assetId, JSONSelector Query, and the JSONValue */ Predicate assetPredicate = null; if (assetId != null) { - assetPredicate = new Predicate() { - @Override - public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would be there, adding support for customised predicates + assetPredicate = new Predicate(){ + @Override + public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" would + // be there, adding support for customised + // predicates var bands = asset.getAssetNode().get("eo:bands"); - if (bands != null && bands.isArray()) { - var bandsArray = (ArrayNode) bands; - for (var bandNode : bandsArray) { - String bandName = bandNode.get("name").asText(); - if (bandName.equals(assetId)) { // under eo:band it's one of the band - return true; - } - } - } else { // meaning eo:bands is not present like Microsoft Planetary, in this case this would be like the asset key i.e. Id - return asset.getId().equals(assetId); - } + if (bands != null && bands.isArray()) { + var bandsArray = (ArrayNode) bands; + for(var bandNode : bandsArray) { + String bandName = bandNode.get("name").asText(); + if (bandName.equals(assetId)) { // under eo:band it's one of the band + return true; + } + } + } else { // meaning eo:bands is not present like Microsoft Planetary, in this + // case this would be like the asset key i.e. Id + return asset.getId().equals(assetId); + } return false; } }; @@ -322,12 +303,12 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // based on the JSON Expression on JSONSelector and JSONValue try { assetPredicate = getAssetPredicate(resource); - } catch (Exception e){ + } catch (Exception e) { throw new KlabResourceAccessException("Couldn't form a predicate with the JSON Expressions"); } - + } - + // These are the static STAC catalogue if (!hasSearchOption) { List features = getFeaturesFromStaticCollection(collectionUrl, collectionData, collectionId); @@ -343,64 +324,66 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou // TODO merge with similar code from below IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); RegionMap regionTransformed = RegionMap.fromEnvelopeAndGrid(regionEnvelope, (int) grid.getXCells(), (int) grid.getYCells()); // end //TODO - List items = features.stream() - .map(f -> { - try { - return HMStacItem.fromSimpleFeature(f); - } catch (Exception e) { - scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); - return null; - }}) - .filter(Objects::nonNull) - .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) - .toList(); + List items = features.stream().map(f -> { + try { + return HMStacItem.fromSimpleFeature(f); + } catch (Exception e) { + scope.getMonitor().warn("Cannot parse feature " + f.getID() + ". Ignored."); + return null; + } + }).filter(Objects::nonNull) + .filter(item -> isWithinRange(item, time2.getStart().getMilliseconds(), time2.getEnd().getMilliseconds())) + .toList(); GridCoverage2D coverage = null; try { - // TODO see if we can access to the same readRasterBandOnRegion without using a collection + // TODO see if we can access to the same readRasterBandOnRegion without using a + // collection LogProgressMonitor lpm = new LogProgressMonitor(); try (HMStacManager manager = new HMStacManager(catalogUrl, lpm)) { - HMStacCollection collection = null; - try { - manager.open(); - collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - } catch (Exception e1) { - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); - } - - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - } - Predicate predicate; - try { - predicate = getAssetPredicate(resource); - } catch (KlabIllegalArgumentException e) { - manager.close(); - throw e; - } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, MergeMode.SUBSTITUTE, lpm); - coverage = outRaster.buildCoverage(); - } + HMStacCollection collection = null; + try { + manager.open(); + collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); + } catch (Exception e1) { + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); + } + + if (collection == null) { + scope.getMonitor().error( + "Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + } + Predicate predicate; + try { + predicate = getAssetPredicate(resource); + } catch (KlabIllegalArgumentException e) { + manager.close(); + throw e; + } + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, predicate, items, true, + MergeMode.SUBSTITUTE, lpm); + coverage = outRaster.buildCoverage(); + } if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } } catch (Exception e) { - throw new KlabResourceAccessException("Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); + throw new KlabResourceAccessException( + "Cannot build output for static collection " + collectionId + ". Reason: " + e.getLocalizedMessage()); } encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); return; } - + LogProgressMonitor lpm = new LogProgressMonitor(); HMStacManager manager = new HMStacManager(catalogUrl, lpm); HMStacCollection collection = null; @@ -408,41 +391,46 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou manager.open(); collection = manager.getCollectionById(resource.getParameters().get("collectionId", String.class)); - if (collection == null) { - scope.getMonitor().error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); - manager.close(); - throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail fast - } - - IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation - ? ((Observation) scope.getTargetArtifact()).getObservable() - : null; - HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); - Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); - Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); - collection.setGeometryFilter(poly); - //collection.setTimestampFilter(new Date(start.getMilliseconds()), new Date(end.getMilliseconds())); --> Filter later :) - - GridCoverage2D coverage = null; - - // Allow transform ensures the process to finish, but I would not bet on the resulting data + if (collection == null) { + scope.getMonitor() + .error("Collection " + resource.getParameters().get("collection", String.class) + " cannot be found."); + manager.close(); + throw new KlabResourceAccessException("Cannot access to STAC collection " + collectionUrl); // Fail + // fast + } + + IObservable targetSemantics = scope.getTargetArtifact() instanceof Observation + ? ((Observation) scope.getTargetArtifact()).getObservable() + : null; + HMRaster.MergeMode mergeMode = chooseMergeMode(targetSemantics, scope.getMonitor()); + Envelope env = new Envelope(envelope.getMinX(), envelope.getMaxX(), envelope.getMinY(), envelope.getMaxY()); + Polygon poly = GeometryUtilities.createPolygonFromEnvelope(env); + collection.setGeometryFilter(poly); + // collection.setTimestampFilter(new Date(start.getMilliseconds()), new + // Date(end.getMilliseconds())); --> Filter later :) + + GridCoverage2D coverage = null; + + // Allow transform ensures the process to finish, but I would not bet on the resulting + // data if (assetPredicate == null) { // NO JSONSelector and JSONValue found, NO assetID was passed as well scope.getMonitor().debug("Query STAC " + collectionUrl + "to get the features"); - // Only get the features from STAC Collection, no need to interact with Rasters - FeatureSource source; + // Only get the features from STAC Collection, no need to interact with Rasters + FeatureSource source; try { - source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), effectiveTime.getEnd()); + source = STACFeatureExtension.getFeatures(catalogData, collectionId, bbox, effectiveTime.getStart(), + effectiveTime.getEnd()); } catch (Exception e) { manager.close(); - throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); + throw new KlabResourceAccessException("Cannot extract features from STAC Collection - " + e.getMessage()); } encoder = new VectorEncoder(); - ((VectorEncoder)encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); + ((VectorEncoder) encoder).encodeFromFeatures(source, resource, urnParameters, geometry, builder, scope); manager.close(); return; } - + List items = collection.searchItems(); if (items.isEmpty()) { manager.close(); @@ -456,8 +444,7 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou IGrid grid = space.getGrid(); RegionMap region = RegionMap.fromBoundsAndGrid(space.getEnvelope().getMinX(), space.getEnvelope().getMaxX(), - space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), - (int) grid.getYCells()); + space.getEnvelope().getMinY(), space.getEnvelope().getMaxY(), (int) grid.getXCells(), (int) grid.getYCells()); ReferencedEnvelope regionEnvelope = new ReferencedEnvelope(region.toEnvelope(), space.getProjection().getCoordinateReferenceSystem()); @@ -470,53 +457,49 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou collection.setS3Client(s3Client); } var time = effectiveTime; - // Filter here based on time, since in some STAC collections they don't yet support temporal filtering :( like ECDC + // Filter here based on time, since in some STAC collections they don't yet support + // temporal filtering :( like ECDC items = items.stream() - .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) - .collect(Collectors.toList()); + .filter(item -> isWithinRange(item, time.getStart().getMilliseconds(), time.getEnd().getMilliseconds())) + .collect(Collectors.toList()); if (items.size() == 0) { manager.close(); - throw new KlabIllegalStateException("No STAC items found covering the entire time duration of the context requested"); + throw new KlabIllegalStateException( + "No STAC items found covering the entire time duration of the context requested"); } else { - scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); + scope.getMonitor().debug("Found " + items.size() + " STAC items satisfying the temporal constraint."); } - + // Once the support for customized predicate is added, we can apply for features as well - + var pred = assetPredicate; - Set EPSGAtAssets = - items.stream() - .flatMap(item -> item.getAssets().stream() - .filter(pred) - .findFirst() - .map(asset -> asset.getEpsg() != null - ? asset.getEpsg() - : item.getEpsg()) - .stream()) - .collect(Collectors.toUnmodifiableSet()); - + Set EPSGAtAssets = items.stream() + .flatMap(item -> item.getAssets().stream().filter(pred).findFirst() + .map(asset -> asset.getEpsg() != null ? asset.getEpsg() : item.getEpsg()).stream()) + .collect(Collectors.toUnmodifiableSet()); + if (EPSGAtAssets.size() > 1) { - scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() - + "." - + "The transformation process could affect the data."); + scope.getMonitor().warn("Multiple EPSGs found on the assets in items " + EPSGAtAssets.toString() + "." + + "The transformation process could affect the data."); } - HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, + MergeMode.SUBSTITUTE, lpm); coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG - coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); + coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); } manager.close(); encoder = new RasterEncoder(); - ((RasterEncoder)encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); + ((RasterEncoder) encoder).encodeFromCoverage(resource, urnParameters, coverage, geometry, builder, scope); } catch (Exception e) { - e.printStackTrace(); + e.printStackTrace(); throw new KlabInternalErrorException("Cannot build STAC raster output. Reason " + e.getMessage()); } } - - private Predicate getAssetPredicate(IResource resource){ + + private Predicate getAssetPredicate(IResource resource) { String assetId = resource.getParameters().get("asset", String.class); if (assetId != null) { return STACPathExpression.STACAssetPredicate.fromHMStacAssetId(assetId); @@ -534,44 +517,40 @@ private Predicate getAssetPredicate(IResource resource){ } } - /* To check if an Item (of type HMStacItem) is within a time range */ private boolean isWithinRange(HMStacItem item, long startMillis, long endMillis) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - String startTimestamp = item.getStartTimestamp(); - String endTimestamp = item.getEndTimestamp(); - - if (startTimestamp == null || endTimestamp == null) { - return true; // Assume the time part is ok - } - - try { - - long itemStart = LocalDateTime - .parse(item.getStartTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - long itemEnd = LocalDateTime - .parse(item.getEndTimestamp(), formatter) - .atZone(ZoneOffset.UTC) - .toInstant() - .toEpochMilli(); - - return startMillis >= itemStart && endMillis <= itemEnd; - - } catch (Exception e) { - e.printStackTrace(); - return false; - } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String startTimestamp = item.getStartTimestamp(); + String endTimestamp = item.getEndTimestamp(); + + if (startTimestamp == null || endTimestamp == null) { + return true; // Assume the time part is ok + } + + try { + + long itemStart = LocalDateTime.parse(item.getStartTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + long itemEnd = LocalDateTime.parse(item.getEndTimestamp(), formatter).atZone(ZoneOffset.UTC).toInstant() + .toEpochMilli(); + + return startMillis >= itemStart && endMillis <= itemEnd; + + } catch (Exception e) { + e.printStackTrace(); + return false; + } } - private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, String collectionId) { - List links = collectionData.getJSONArray("links").toList().stream().filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("item")).toList(); - List urlOfLinks = links.stream().map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); + private List getFeaturesFromStaticCollection(String collectionUrl, JSONObject collectionData, + String collectionId) { + List links = collectionData.getJSONArray("links").toList().stream() + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("item")).toList(); + List urlOfLinks = links.stream() + .map(link -> STACUtils.getUrlOfItem(collectionUrl, collectionId, link.getString("href"))).toList(); return urlOfLinks.stream().map(i -> { try { return STACUtils.getItemAsFeature(i); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index c8a689020..dbaace6c9 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -9,10 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import org.integratedmodelling.kim.api.IParameters; -import org.integratedmodelling.klab.Logging; import org.integratedmodelling.klab.Resources; import org.integratedmodelling.klab.api.data.ILocator; import org.integratedmodelling.klab.api.data.IResource; @@ -43,9 +41,9 @@ public boolean acceptsMultiple() { // TODO Auto-generated method stub return false; } - + private boolean hasAssetSelector(IParameters parameters) { - return (parameters.contains("asset")||(parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); + return (parameters.contains("asset") || (parameters.contains("jsonSelector") && parameters.contains("jsonValue"))); } private void importCollection(List ret, IParameters parameters, IProject project, IMonitor monitor) @@ -55,12 +53,6 @@ private void importCollection(List ret, IParameters parameters, String collectionId = STACCollectionParser.readCollectionId(collectionData); parameters.put("collectionId", collectionId); - String regex = null; - if (parameters.contains("regex")) { - regex = parameters.get(Resources.REGEX_ENTRY, String.class); - parameters.remove(Resources.REGEX_ENTRY); - } - boolean isBulkImport = parameters.contains("bulkImport"); parameters.remove("bulkImport"); if (!hasAssetSelector(parameters) && !isBulkImport) { @@ -72,10 +64,10 @@ private void importCollection(List ret, IParameters parameters, } return; } - + String assetId = parameters.get("asset", String.class); JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - + JSONObject assetData = assetNode.getJSONObject(assetId); /* * If the particular asset Id wasn't found, then @@ -84,18 +76,17 @@ private void importCollection(List ret, IParameters parameters, * In that case however, a better handling of S3 URL needs to figured out */ if (assetData != null) { - if (!STACAssetParser.isSupportedMediaType(assetData)) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { throw new Exception("Unsupported media type for the asset"); - } - String href = assetData.getString("href"); + } + String href = assetData.getString("href"); if (S3URLUtils.isS3Endpoint(href)) { - String[] bucketAndObject = href.split("://")[1].split("/", 2); + // String[] bucketAndObject = href.split("://")[1].split("/", 2); String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } } - - + parameters.put("asset", assetId); String resourceUrn = collectionId + "-" + assetId; Builder builder = buildResource(parameters, project, monitor, resourceUrn); @@ -104,10 +95,16 @@ private void importCollection(List ret, IParameters parameters, } else { monitor.warn("STAC resource with asset " + resourceUrn + " is invalid and cannot be imported"); } - - // Think of a way to do the REGEX matching. Possibly not worth it to support + + // Think of a way to do the REGEX matching. Possibly not worth it to support // this for something as diverse as STAC - + + // String regex = null; + if (parameters.contains("regex")) { + // regex = parameters.get(Resources.REGEX_ENTRY, String.class); + parameters.remove(Resources.REGEX_ENTRY); + } + /* Set assetIds = STACAssetMapParser.readAssetNames(assets); for(String assetId : assetIds) { @@ -115,7 +112,7 @@ private void importCollection(List ret, IParameters parameters, Logging.INSTANCE.info("Asset " + assetId + " doesn't match REGEX, skipped"); continue; } - + JSONObject assetData = STACAssetMapParser.getAsset(assets, assetId); if (assetData != null) { if (!STACAssetParser.isSupportedMediaType(assetData)) { @@ -132,7 +129,7 @@ private void importCollection(List ret, IParameters parameters, String s3Region = "unknown"; // TODO resolve the region parameters.put("awsRegion", s3Region); } - + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); @@ -144,10 +141,10 @@ private void importCollection(List ret, IParameters parameters, */ } - private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) throws MalformedURLException { - Builder builder = validator.validate( - Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), new URL(parameters.get("collection", String.class)), - parameters, monitor); + private Builder buildResource(IParameters parameters, IProject project, IMonitor monitor, String resourceUrn) + throws MalformedURLException { + Builder builder = validator.validate(Resources.INSTANCE.createLocalResourceUrn(resourceUrn, project), + new URL(parameters.get("collection", String.class)), parameters, monitor); if (builder == null) { return null; diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java index 0a16ae462..dfdfdc726 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACPathExpression.java @@ -50,10 +50,10 @@ public List resolve(JsonNode root) { List currentNodes = new ArrayList<>(); currentNodes.add(root); - for (PathPart part : path) { + for(PathPart part : path) { List nextNodes = new ArrayList<>(); - for (JsonNode current : currentNodes) { + for(JsonNode current : currentNodes) { if (current == null || current.isNull() || current.isMissingNode()) { continue; } @@ -87,9 +87,8 @@ private static void resolveFromArray(JsonNode arrayNode, PathPart part, List keywords = json.getJSONArray("keywords").toList(); - return keywords.isEmpty() ? null : - keywords.stream().collect(Collectors.joining(",")); + return keywords.isEmpty() ? null : keywords.stream().collect(Collectors.joining(",")); } - final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", "properties.sci:doi", "item_assets.sci:doi"); + final private static Set DOI_KEYS_IN_STAC_JSON = Set.of("sci:doi", "assets.sci:doi", "summaries.sci:doi", + "properties.sci:doi", "item_assets.sci:doi"); public static String readDOI(JSONObject json) { - Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)).findFirst(); + Optional doi = DOI_KEYS_IN_STAC_JSON.stream().filter(key -> json.has(key)).map(key -> json.getString(key)) + .findFirst(); return doi.isPresent() ? doi.get() : null; } @@ -61,13 +62,13 @@ public static String readDOIAuthors(String doi) { */ public static boolean containsLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .anyMatch(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)); + .anyMatch(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)); } public static Optional getLinkTo(JSONObject data, String rel) { return data.getJSONArray("links").toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase(rel)) - .map(link -> ((JSONObject)link).getString("href")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase(rel)) + .map(link -> ((JSONObject) link).getString("href")).findFirst(); } public static JSONObject requestMetadata(String collectionUrl, String type) { @@ -87,7 +88,7 @@ public static String readLicense(JSONObject collection) { return null; } JSONArray links = collection.getJSONArray("links"); - for (int i = 0; i < links.length(); i++) { + for(int i = 0; i < links.length(); i++) { JSONObject link = links.getJSONObject(i); if (!link.has("rel") || !link.getString("rel").equals("license")) { continue; @@ -121,11 +122,12 @@ public static Type inferValueType(String key) { public static String getCatalogUrl(String collectionUrl, String collectionId, JSONObject collectionData) { // The URL of the catalog is the root if (!collectionData.has("links")) { - throw new KlabResourceAccessException("STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); + throw new KlabResourceAccessException( + "STAC collection is missing links. It is not fully complaiant and cannot be accessed by the adapter."); } JSONArray links = collectionData.getJSONArray("links"); Optional rootLink = links.toList().stream() - .filter(link -> ((JSONObject)link).getString("rel").equalsIgnoreCase("root")).findFirst(); + .filter(link -> ((JSONObject) link).getString("rel").equalsIgnoreCase("root")).findFirst(); if (rootLink.isEmpty()) { throw new KlabResourceAccessException("STAC collection is missing a relationship to the root catalog"); } diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 0384fc9c2..f046da579 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -71,8 +71,7 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni Predicate predicate = STACPathExpression.STACAssetPredicate .fromKongJsonObject(userData.get("jsonSelector", String.class), userData.get("jsonValue", String.class)); - assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, - predicate); + assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, predicate); } else { throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); @@ -82,7 +81,8 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni JSONObject asset = assetNode.getJSONObject(assetId); Type type = readRasterDataType(asset); - // Currently, only files:values is supported. If needed, the classification extension could be used too. + // Currently, only files:values is supported. If needed, the classification extension could + // be used too. Map vals = STACAssetParser.getFileValues(asset); if (!vals.isEmpty()) { CodelistReference codelist = populateCodelist(assetId, vals); From 2703a2cd14800c92ecfe5e4a8b782aa125d60487 Mon Sep 17 00:00:00 2001 From: Enrico Girotto Date: Wed, 13 May 2026 14:40:19 +0200 Subject: [PATCH 27/28] Solve NPE if no asset match all the conditions --- .../java/org/integratedmodelling/klab/stac/STACEncoder.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java index c64465b84..0570919d6 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACEncoder.java @@ -486,6 +486,10 @@ public boolean test(HMStacAsset asset) { // Assuming for now that "eo:bands" wou HMRaster outRaster = collection.readRasterBandOnRegion(regionTransformed, assetPredicate, items, allowTransform, MergeMode.SUBSTITUTE, lpm); + if (outRaster == null) { + scope.getMonitor().error("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + throw new KlabIllegalStateException("No STAC assets were found. Please check the spatial/temporal coverage of the resource"); + } coverage = outRaster.buildCoverage(); if (bandIndex != null) { // Which means theat it's a Multi Band COG coverage = (GridCoverage2D) Operations.DEFAULT.selectSampleDimension(coverage, new int[]{bandIndex}); From 3c7ed198d78c60bc3f25df03a5b3ad2ec5ad31a9 Mon Sep 17 00:00:00 2001 From: AM1729 Date: Wed, 13 May 2026 15:52:46 +0200 Subject: [PATCH 28/28] final fixes --- .../klab/stac/STACImporter.java | 56 ++++++++++++------- .../klab/stac/STACValidator.java | 9 ++- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java index c8a689020..cb29f1996 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACImporter.java @@ -74,30 +74,48 @@ private void importCollection(List ret, IParameters parameters, } String assetId = parameters.get("asset", String.class); - JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); - - JSONObject assetData = assetNode.getJSONObject(assetId); + String resourceUrn = collectionId; + JSONObject assetData = null; /* - * If the particular asset Id wasn't found, then - * still proceed to create the resource since it can happen - * for a number of reasons. Pagination, STAC Backend etc. - * In that case however, a better handling of S3 URL needs to figured out + This is only for the part, to check the Asset is under an S3 Bucket */ - if (assetData != null) { - if (!STACAssetParser.isSupportedMediaType(assetData)) { - throw new Exception("Unsupported media type for the asset"); - } - String href = assetData.getString("href"); - if (S3URLUtils.isS3Endpoint(href)) { - String[] bucketAndObject = href.split("://")[1].split("/", 2); - String s3Region = "unknown"; // TODO resolve the region - parameters.put("awsRegion", s3Region); + if (assetId != null) { + JSONObject assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, assetId); + assetData = assetNode.getJSONObject(assetId); + /* + * If the particular asset Id wasn't found, then + * still proceed to create the resource since it can happen + * for a number of reasons. Pagination, STAC Backend etc. + * In that case however, a better handling of S3 URL needs to figured out + */ + if (assetData != null) { + if (!STACAssetParser.isSupportedMediaType(assetData)) { + throw new Exception("Unsupported media type for the asset"); + } + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } } - } + resourceUrn = collectionId + "-" + assetId; + } + //TODO: Check if assetId is null, and instead a jsonSelector and Value is passed! + //else if (parameters.get("jsonSelector", String.class) != null) { + // String selector = parameters.get("jsonSelector", String.class); + // String val = parameters.get("jsonValue", String.class); + // assetData = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, null, STACPathExpression.STACAssetPredicate.fromKongJsonObject(selector, val)); + //} - parameters.put("asset", assetId); - String resourceUrn = collectionId + "-" + assetId; + String href = assetData.getString("href"); + if (S3URLUtils.isS3Endpoint(href)) { + String[] bucketAndObject = href.split("://")[1].split("/", 2); + String s3Region = "unknown"; // TODO resolve the region + parameters.put("awsRegion", s3Region); + } + Builder builder = buildResource(parameters, project, monitor, resourceUrn); if (builder != null) { ret.add(builder); diff --git a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java index 0384fc9c2..b89bf7e94 100644 --- a/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java +++ b/adapters/klab.ogc/src/main/java/org/integratedmodelling/klab/stac/STACValidator.java @@ -49,8 +49,9 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni collectionId = collectionData.getString("id"); userData.put("collectionId", collectionId); } - IGeometry geometry = STACCollectionParser.readGeometry(collectionData); + + IGeometry geometry = STACCollectionParser.readGeometry(collectionData); Builder builder = new ResourceBuilder(urn).withParameters(userData).withGeometry(geometry).withType(Type.OBJECT); // The default URL of the resource is the collection endpoint. May be overwritten. @@ -60,7 +61,6 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni if (userData.contains("asset")) { String requestedAssetId = userData.get("asset", String.class); - assetNode = STACCollectionParser.readAssetInformationFromCollection(collectionUrl, collectionData, requestedAssetId); } else if (userData.contains("jsonSelector")) { @@ -75,7 +75,10 @@ public Builder validate(String urn, URL url, IParameters userData, IMoni predicate); } else { - throw new KlabIllegalArgumentException("Either asset or jsonSelector/jsonValue must be provided"); + // Just import Features + monitor.info("import STAC Collection for Features"); + readMetadata(collectionData, builder); + return builder; } String assetId = assetNode.keys().next();