From 51c5ea682c9dba2d56c0c2bf20d91240f649bdfa Mon Sep 17 00:00:00 2001 From: nehem Date: Wed, 26 Aug 2020 02:16:02 +1000 Subject: [PATCH 01/10] S3 filename need not to have full version --- eodatasets3/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 92c1adff..6e45a095 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -273,7 +273,7 @@ def measurement_file_path( def _dataset_label(self, sub_name: str = None): p = self.dataset - version = p.dataset_version.replace(".", "-") if p.dataset_version else None + version = p.dataset_version.split(".")[0] if p.dataset_version else None maturity: str = p.properties.get("dea:dataset_maturity") return "_".join( [ From b03c755f6cf4a702854d3a51b2d1b1d5d97f2365 Mon Sep 17 00:00:00 2001 From: nehem Date: Thu, 27 Aug 2020 11:56:01 +1000 Subject: [PATCH 02/10] Do not distinguish platform versions for lansat --- eodatasets3/model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 6e45a095..4bb04eda 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -196,7 +196,7 @@ def _product_group(self, subname=None) -> str: platform = self.platform_abbreviated inst = self.instrument_abbreviated if platform and inst: - parts.append(f"{platform}{inst}") + parts.append(platform) if not subname: raise ValueError( @@ -326,7 +326,8 @@ def platform_abbreviated(self) -> Optional[str]: f"TODO: implement non-landsat platform abbreviation " f"(got {p!r})" ) - return f"ls{p[-1]}" + return "ls" + @property def instrument_abbreviated(self) -> Optional[str]: From b895d53fdd2a09d36eec399985fcf7d0bc0582c2 Mon Sep 17 00:00:00 2001 From: nehem Date: Wed, 30 Sep 2020 14:32:48 +1000 Subject: [PATCH 03/10] Changes for the new fileformat --- eodatasets3/model.py | 13 +++++++------ eodatasets3/properties.py | 20 ++++++++++++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 4bb04eda..1da00c0b 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -164,6 +164,7 @@ def _check_enough_properties_to_name(self): hint = f" ({hint})" examples.append(f"\n- {p!r}{hint}") + raise ValueError( f"Need more properties to fulfill naming conventions." f"{''.join(examples)}" @@ -172,7 +173,6 @@ def _check_enough_properties_to_name(self): @property def product_name(self) -> str: self._check_enough_properties_to_name() - org_number = self._org_collection_number if org_number: return f"{self._product_group()}_{org_number}" @@ -180,6 +180,8 @@ def product_name(self) -> str: @property def _org_collection_number(self) -> Optional[int]: + if self.dataset.collection_number: + return int(self.dataset.collection_number) if not self.dataset.dataset_version: return None return int(self.dataset.dataset_version.split(".")[0]) @@ -225,10 +227,10 @@ def dataset_label(self) -> str: def destination_folder(self, base: Path): self._check_enough_properties_to_name() - # DEA naming conventions folder hierarchy. - # Example: "ga_ls8c_ard_3/092/084/2016/06/28" - parts = [self.product_name] + # DEA naming conventions folder hierarchy. + # Example: "ga_ls_wo_3/1-6-0/090/081/1998/07/30" + parts = [self.product_name, self.dataset.dataset_version.replace(".", "-")] # Cut the region code in subfolders region_code = self.dataset.region_code @@ -272,8 +274,7 @@ def measurement_file_path( def _dataset_label(self, sub_name: str = None): p = self.dataset - - version = p.dataset_version.split(".")[0] if p.dataset_version else None + version = p.collection_number if p.collection_number else p.dataset_version.split(".")[0] if p.dataset_version else None maturity: str = p.properties.get("dea:dataset_maturity") return "_".join( [ diff --git a/eodatasets3/properties.py b/eodatasets3/properties.py index dd29849f..022643aa 100644 --- a/eodatasets3/properties.py +++ b/eodatasets3/properties.py @@ -7,10 +7,9 @@ from typing import Tuple, Dict, Optional, Any, Mapping, Callable, Union import ciso8601 -from eodatasets3.utils import default_utc - from ruamel.yaml.timestamp import TimeStamp as RuamelTimeStamp +from eodatasets3.utils import default_utc class FileFormat(Enum): GeoTIFF = 1 @@ -226,6 +225,7 @@ class StacPropertyView(collections.abc.Mapping): "landsat:wrs_path": int, "landsat:wrs_row": int, "odc:dataset_version": None, + "odc:collection_number": None, # Not strict as there may be more added in ODC... "odc:file_format": of_enum_type(FileFormat, strict=False), "odc:processing_datetime": datetime_type, @@ -413,10 +413,26 @@ def dataset_version(self) -> str: """ return self.properties.get("odc:dataset_version") + @property + def collection_number(self) -> str: + """ + The version of the collection. + Eg: + metadata: + product_family: wofs + dataset_version: 1.6.0 + collection_number: 3 + """ + return self.properties.get("odc:collection_number", "0") + @dataset_version.setter def dataset_version(self, value): self.properties["odc:dataset_version"] = value + @collection_number.setter + def collection_number(self, value): + self.properties["odc:collection_number"] = value + @property def product_family(self) -> str: """ From 273714f95cf386b438cfcc71b2780bce348980f4 Mon Sep 17 00:00:00 2001 From: nehem Date: Wed, 30 Sep 2020 14:47:36 +1000 Subject: [PATCH 04/10] Addressing Travis complaints --- eodatasets3/model.py | 6 +++--- eodatasets3/properties.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 1da00c0b..d190531a 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -164,7 +164,6 @@ def _check_enough_properties_to_name(self): hint = f" ({hint})" examples.append(f"\n- {p!r}{hint}") - raise ValueError( f"Need more properties to fulfill naming conventions." f"{''.join(examples)}" @@ -274,7 +273,9 @@ def measurement_file_path( def _dataset_label(self, sub_name: str = None): p = self.dataset - version = p.collection_number if p.collection_number else p.dataset_version.split(".")[0] if p.dataset_version else None + version = p.collection_number if p.collection_number \ + else p.dataset_version.split(".")[0] if p.dataset_version \ + else None maturity: str = p.properties.get("dea:dataset_maturity") return "_".join( [ @@ -329,7 +330,6 @@ def platform_abbreviated(self) -> Optional[str]: return "ls" - @property def instrument_abbreviated(self) -> Optional[str]: """Abbreviated form of an instrument name, as used in dea product names. eg. 'c'.""" diff --git a/eodatasets3/properties.py b/eodatasets3/properties.py index 022643aa..9b5c696c 100644 --- a/eodatasets3/properties.py +++ b/eodatasets3/properties.py @@ -11,6 +11,7 @@ from eodatasets3.utils import default_utc + class FileFormat(Enum): GeoTIFF = 1 NetCDF = 2 From cc353481da3c0c309c7a476f1f08fc7ab4384547 Mon Sep 17 00:00:00 2001 From: nehem Date: Tue, 6 Oct 2020 03:41:21 +1100 Subject: [PATCH 05/10] Inherited ComplicatedNamingConventions to adapt changes for Alchemist specific ones --- eodatasets3/assemble.py | 3 + eodatasets3/model.py | 94 +++++++++++++++++++++++++----- eodatasets3/properties.py | 9 +++ tests/integration/test_assemble.py | 29 ++++++++- 4 files changed, 119 insertions(+), 16 deletions(-) diff --git a/eodatasets3/assemble.py b/eodatasets3/assemble.py index c1df3fcf..5a87d1d6 100644 --- a/eodatasets3/assemble.py +++ b/eodatasets3/assemble.py @@ -25,6 +25,7 @@ ComplicatedNamingConventions, AccessoryDoc, Location, + ComplicatedNamingConventionsAlchemist, ) from eodatasets3.properties import EoFields from eodatasets3.validate import Level, ValidationMessage @@ -169,6 +170,8 @@ def __init__( self.names = ComplicatedNamingConventions.for_standard_dea(self) elif naming_conventions == "dea_s2": self.names = ComplicatedNamingConventions.for_standard_dea_s2(self) + elif naming_conventions == "dea_c3": + self.names = ComplicatedNamingConventionsAlchemist(self) else: raise NotImplementedError("configurable naming conventions") diff --git a/eodatasets3/model.py b/eodatasets3/model.py index d190531a..61dc9f45 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -5,17 +5,16 @@ import affine import attr -from eodatasets3 import utils -from eodatasets3.properties import StacPropertyView, EoFields from ruamel.yaml.comments import CommentedMap from shapely.geometry.base import BaseGeometry +from eodatasets3 import utils +from eodatasets3.properties import StacPropertyView, EoFields # TODO: these need discussion. DEA_URI_PREFIX = "https://collections.dea.ga.gov.au" ODC_DATASET_SCHEMA_URL = "https://schemas.opendatacube.org/dataset" - # Either a local filesystem path or a string URI. # (the URI can use any scheme supported by rasterio, such as tar:// or https:// or ...) Location = Union[Path, str] @@ -172,6 +171,7 @@ def _check_enough_properties_to_name(self): @property def product_name(self) -> str: self._check_enough_properties_to_name() + org_number = self._org_collection_number if org_number: return f"{self._product_group()}_{org_number}" @@ -179,8 +179,6 @@ def product_name(self) -> str: @property def _org_collection_number(self) -> Optional[int]: - if self.dataset.collection_number: - return int(self.dataset.collection_number) if not self.dataset.dataset_version: return None return int(self.dataset.dataset_version.split(".")[0]) @@ -197,7 +195,7 @@ def _product_group(self, subname=None) -> str: platform = self.platform_abbreviated inst = self.instrument_abbreviated if platform and inst: - parts.append(platform) + parts.append(f"{platform}{inst}") if not subname: raise ValueError( @@ -226,10 +224,10 @@ def dataset_label(self) -> str: def destination_folder(self, base: Path): self._check_enough_properties_to_name() - # DEA naming conventions folder hierarchy. - # Example: "ga_ls_wo_3/1-6-0/090/081/1998/07/30" - parts = [self.product_name, self.dataset.dataset_version.replace(".", "-")] + # Example: "ga_ls8c_ard_3/092/084/2016/06/28" + + parts = [self.product_name] # Cut the region code in subfolders region_code = self.dataset.region_code @@ -273,9 +271,8 @@ def measurement_file_path( def _dataset_label(self, sub_name: str = None): p = self.dataset - version = p.collection_number if p.collection_number \ - else p.dataset_version.split(".")[0] if p.dataset_version \ - else None + + version = p.dataset_version.replace(".", "-") if p.dataset_version else None maturity: str = p.properties.get("dea:dataset_maturity") return "_".join( [ @@ -328,7 +325,7 @@ def platform_abbreviated(self) -> Optional[str]: f"TODO: implement non-landsat platform abbreviation " f"(got {p!r})" ) - return "ls" + return f"ls{p[-1]}" @property def instrument_abbreviated(self) -> Optional[str]: @@ -375,6 +372,77 @@ def producer_abbreviated(self) -> Optional[str]: ) +class ComplicatedNamingConventionsAlchemist(ComplicatedNamingConventions): + """ + This class is inherited from ComplicatedNamingConventions + and overrides few attributes specific to C3 dataprocessing at the Alchemist end. + """ + + @property + def _org_collection_number(self) -> Optional[int]: + + # Deliberately fail if collection_number is not defined in the config yaml + return int(self.dataset.collection_number) + + def _product_group(self, subname=None) -> str: + # Computues product group, e.g "ga_ls_wo_3" + # Deliberately fail if any of these attributes not found. + parts = [ + self.producer_abbreviated, + self.platform_abbreviated, + self.dataset.product_family, + ] + return "_".join(parts) + + def destination_folder(self, base: Path): + + # Copycat of the parent method, except dataset_version to be included in the parts. + self._check_enough_properties_to_name() + + # DEA naming conventions folder hierarchy. + # Example: "ga_ls_wo_3/1-6-0/090/081/1998/07/30" + parts = [self.product_name, self.dataset.dataset_version.replace(".", "-")] + + # Cut the region code in subfolders + region_code = self.dataset.region_code + if region_code: + parts.extend(utils.subfolderise(region_code)) + + parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) + + if self.dataset_separator_field is not None: + val = self.dataset.properties[self.dataset_separator_field] + # TODO: choosable formatter? + if isinstance(val, datetime): + val = f"{val:%Y%m%dT%H%M%S}" + parts.append(val) + return base.joinpath(*parts) + + def _dataset_label(self, sub_name: str = None): + # Copycat of the parent method except version is simply collection number + p = self.dataset + version = p.collection_number + maturity: str = p.properties.get("dea:dataset_maturity") + return "_".join( + [ + p + for p in ( + self._product_group(sub_name), + version, + self._displayable_region_code, + f"{p.datetime:%Y-%m-%d}", + maturity, + ) + if p + ] + ) + + @property + def platform_abbreviated(self) -> Optional[str]: + # For now from Alchemist the platform is always landsat for C3 processing + return "ls" + + @attr.s(auto_attribs=True, slots=True) class DatasetDoc(EoFields): id: UUID = None diff --git a/eodatasets3/properties.py b/eodatasets3/properties.py index 9b5c696c..32761966 100644 --- a/eodatasets3/properties.py +++ b/eodatasets3/properties.py @@ -227,6 +227,7 @@ class StacPropertyView(collections.abc.Mapping): "landsat:wrs_row": int, "odc:dataset_version": None, "odc:collection_number": None, + "odc:naming_conventions": None, # Not strict as there may be more added in ODC... "odc:file_format": of_enum_type(FileFormat, strict=False), "odc:processing_datetime": datetime_type, @@ -434,6 +435,14 @@ def dataset_version(self, value): def collection_number(self, value): self.properties["odc:collection_number"] = value + @property + def naming_conventions(self) -> str: + return self.properties.get("odc:naming_conventions") + + @naming_conventions.setter + def naming_conventions(self, value): + self.properties["odc:naming_conventions"] = value + @property def product_family(self) -> str: """ diff --git a/tests/integration/test_assemble.py b/tests/integration/test_assemble.py index 6a70565a..d29059fc 100644 --- a/tests/integration/test_assemble.py +++ b/tests/integration/test_assemble.py @@ -5,13 +5,13 @@ import numpy import pytest +from ruamel import yaml + from eodatasets3 import DatasetAssembler from eodatasets3.images import GridSpec from eodatasets3.model import DatasetDoc -from ruamel import yaml -from tests.integration.common import assert_same_as_file - from tests import assert_file_structure +from tests.integration.common import assert_same_as_file def test_dea_style_package( @@ -309,6 +309,7 @@ def test_s2_naming_conventions(tmp_path: Path): # The s2 naming conventions have an extra subfolder of the datatake start time. metadata_path_offset = metadata_path.relative_to(tmp_path).as_posix() + assert metadata_path_offset == ( "ga_s2am_blueberries_1/Oz/2018/11/04/20170822T015626/" "ga_s2am_blueberries_1-0-0_Oz_2018-11-04.odc-metadata.yaml" @@ -399,3 +400,25 @@ def test_complain_about_missing_fields(tmp_path: Path, l1_ls8_folder: Path): f"Expected field {needed_field_name} to " f"be listed as mandatory in the error message" ) + + +def test_dea_c3_naming_conventions(tmp_path: Path): + """ + A sample scene for Alchemist C3 processing that tests the naming conventions. + """ + p = DatasetAssembler(tmp_path, naming_conventions="dea_c3") + p.platform = "ga_ls5t" + p.datetime = datetime(1998, 7, 30) + p.product_family = "wo" + p.processed = "1998-07-30T12:23:23" + p.maturity = "interim" + p.producer = "ga.gov.au" + p.dataset_version = "1.6.0" + p.collection_number = "3" + p.region_code = "090081" + dataset_id, metadata_path = p.done() + metadata_path_offset = metadata_path.relative_to(tmp_path).as_posix() + assert ( + metadata_path_offset + == "ga_ls_wo_3/1-6-0/090/081/1998/07/30/ga_ls_wo_3_090081_1998-07-30_interim.odc-metadata.yaml" + ) From 17abe9e71bd9985125d98958ff04df351baa2535 Mon Sep 17 00:00:00 2001 From: nehem Date: Tue, 6 Oct 2020 09:49:22 +1100 Subject: [PATCH 06/10] Updated the testcase to test required fields --- eodatasets3/assemble.py | 2 +- eodatasets3/model.py | 40 ++++++++++++++++-------------- tests/integration/test_assemble.py | 12 ++++++++- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/eodatasets3/assemble.py b/eodatasets3/assemble.py index 5a87d1d6..b3ecbc2a 100644 --- a/eodatasets3/assemble.py +++ b/eodatasets3/assemble.py @@ -171,7 +171,7 @@ def __init__( elif naming_conventions == "dea_s2": self.names = ComplicatedNamingConventions.for_standard_dea_s2(self) elif naming_conventions == "dea_c3": - self.names = ComplicatedNamingConventionsAlchemist(self) + self.names = ComplicatedNamingConventionsAlchemist.for_c3_processing(self) else: raise NotImplementedError("configurable naming conventions") diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 61dc9f45..ce9336f8 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -373,11 +373,32 @@ def producer_abbreviated(self) -> Optional[str]: class ComplicatedNamingConventionsAlchemist(ComplicatedNamingConventions): + """ This class is inherited from ComplicatedNamingConventions and overrides few attributes specific to C3 dataprocessing at the Alchemist end. """ + @classmethod + def for_c3_processing(cls, dataset: EoFields, uri=DEA_URI_PREFIX): + """ + The required fields for the c3 data processing are controlled here. + """ + return cls( + dataset=dataset, + base_product_uri=uri, + required_fields=( + "eo:platform", + "odc:dataset_version", + "odc:collection_number", + "odc:processing_datetime", + "odc:producer", + "odc:product_family", + "odc:region_code", + "dea:dataset_maturity", + ), + ) + @property def _org_collection_number(self) -> Optional[int]: @@ -395,27 +416,10 @@ def _product_group(self, subname=None) -> str: return "_".join(parts) def destination_folder(self, base: Path): - - # Copycat of the parent method, except dataset_version to be included in the parts. self._check_enough_properties_to_name() - - # DEA naming conventions folder hierarchy. - # Example: "ga_ls_wo_3/1-6-0/090/081/1998/07/30" parts = [self.product_name, self.dataset.dataset_version.replace(".", "-")] - - # Cut the region code in subfolders - region_code = self.dataset.region_code - if region_code: - parts.extend(utils.subfolderise(region_code)) - + parts.extend(utils.subfolderise(self.dataset.region_code)) parts.extend(f"{self.dataset.datetime:%Y/%m/%d}".split("/")) - - if self.dataset_separator_field is not None: - val = self.dataset.properties[self.dataset_separator_field] - # TODO: choosable formatter? - if isinstance(val, datetime): - val = f"{val:%Y%m%dT%H%M%S}" - parts.append(val) return base.joinpath(*parts) def _dataset_label(self, sub_name: str = None): diff --git a/tests/integration/test_assemble.py b/tests/integration/test_assemble.py index d29059fc..0799f63c 100644 --- a/tests/integration/test_assemble.py +++ b/tests/integration/test_assemble.py @@ -413,9 +413,19 @@ def test_dea_c3_naming_conventions(tmp_path: Path): p.processed = "1998-07-30T12:23:23" p.maturity = "interim" p.producer = "ga.gov.au" + p.region_code = "090081" + + # Try missing few fields and expect ValueError + with pytest.raises( + ValueError, match="Need more properties to fulfill naming conventions." + ): + p.done() + + # Put back the missed ones p.dataset_version = "1.6.0" p.collection_number = "3" - p.region_code = "090081" + + # Success case dataset_id, metadata_path = p.done() metadata_path_offset = metadata_path.relative_to(tmp_path).as_posix() assert ( From df5fc4a542188762feb4bf2296d9d76b36aa6206 Mon Sep 17 00:00:00 2001 From: nehem Date: Tue, 6 Oct 2020 11:06:53 +1100 Subject: [PATCH 07/10] Update eodatasets3/model.py Comments suggestion by Alex Co-authored-by: Alex Leith --- eodatasets3/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index ce9336f8..a9d7f489 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -376,7 +376,7 @@ class ComplicatedNamingConventionsAlchemist(ComplicatedNamingConventions): """ This class is inherited from ComplicatedNamingConventions - and overrides few attributes specific to C3 dataprocessing at the Alchemist end. + and overrides few attributes specific to C3 data processing for data other than ARD. """ @classmethod From 3e30fe104d472fd6cd3c2e45ca2f9a5236895026 Mon Sep 17 00:00:00 2001 From: nehem Date: Tue, 6 Oct 2020 11:12:47 +1100 Subject: [PATCH 08/10] Renamed class and method for eodatasets3/model.py --- eodatasets3/assemble.py | 6 ++++-- eodatasets3/model.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eodatasets3/assemble.py b/eodatasets3/assemble.py index b3ecbc2a..997f1e8d 100644 --- a/eodatasets3/assemble.py +++ b/eodatasets3/assemble.py @@ -25,7 +25,7 @@ ComplicatedNamingConventions, AccessoryDoc, Location, - ComplicatedNamingConventionsAlchemist, + ComplicatedNamingConventionsDerivatives, ) from eodatasets3.properties import EoFields from eodatasets3.validate import Level, ValidationMessage @@ -171,7 +171,9 @@ def __init__( elif naming_conventions == "dea_s2": self.names = ComplicatedNamingConventions.for_standard_dea_s2(self) elif naming_conventions == "dea_c3": - self.names = ComplicatedNamingConventionsAlchemist.for_c3_processing(self) + self.names = ComplicatedNamingConventionsDerivatives.for_c3_derivatives( + self + ) else: raise NotImplementedError("configurable naming conventions") diff --git a/eodatasets3/model.py b/eodatasets3/model.py index a9d7f489..130feea3 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -372,7 +372,7 @@ def producer_abbreviated(self) -> Optional[str]: ) -class ComplicatedNamingConventionsAlchemist(ComplicatedNamingConventions): +class ComplicatedNamingConventionsDerivatives(ComplicatedNamingConventions): """ This class is inherited from ComplicatedNamingConventions @@ -380,7 +380,7 @@ class ComplicatedNamingConventionsAlchemist(ComplicatedNamingConventions): """ @classmethod - def for_c3_processing(cls, dataset: EoFields, uri=DEA_URI_PREFIX): + def for_c3_derivatives(cls, dataset: EoFields, uri=DEA_URI_PREFIX): """ The required fields for the c3 data processing are controlled here. """ From baf85057a0cbde0aac4e45d54822fe34a755738f Mon Sep 17 00:00:00 2001 From: nehem Date: Tue, 6 Oct 2020 14:02:53 +1100 Subject: [PATCH 09/10] Removed the dead code, simplied the _dataset_label --- eodatasets3/model.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index 130feea3..ba0f9d2b 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -373,7 +373,6 @@ def producer_abbreviated(self) -> Optional[str]: class ComplicatedNamingConventionsDerivatives(ComplicatedNamingConventions): - """ This class is inherited from ComplicatedNamingConventions and overrides few attributes specific to C3 data processing for data other than ARD. @@ -401,7 +400,6 @@ def for_c3_derivatives(cls, dataset: EoFields, uri=DEA_URI_PREFIX): @property def _org_collection_number(self) -> Optional[int]: - # Deliberately fail if collection_number is not defined in the config yaml return int(self.dataset.collection_number) @@ -423,23 +421,19 @@ def destination_folder(self, base: Path): return base.joinpath(*parts) def _dataset_label(self, sub_name: str = None): - # Copycat of the parent method except version is simply collection number - p = self.dataset - version = p.collection_number - maturity: str = p.properties.get("dea:dataset_maturity") - return "_".join( - [ - p - for p in ( - self._product_group(sub_name), - version, - self._displayable_region_code, - f"{p.datetime:%Y-%m-%d}", - maturity, - ) - if p - ] - ) + """ + Responsible for producing the string of product name, regioncode, datetime and maturity + ex: 'ga_ls_wo_3_090081_1998-07-30_interim' + + Redundant parameter sub_name is required, since the parent class and other invocations wants it so. + """ + parts = [ + self.product_name, + self._displayable_region_code, + f"{self.dataset.datetime:%Y-%m-%d}", + self.dataset.maturity, + ] + return "_".join(parts) @property def platform_abbreviated(self) -> Optional[str]: From b75c38526958a61064e086639d5391341e247e27 Mon Sep 17 00:00:00 2001 From: nehem Date: Fri, 9 Oct 2020 16:27:46 +1100 Subject: [PATCH 10/10] Ensure platform is landsat --- eodatasets3/model.py | 4 ++++ eodatasets3/properties.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/eodatasets3/model.py b/eodatasets3/model.py index ba0f9d2b..0cfaa6a9 100644 --- a/eodatasets3/model.py +++ b/eodatasets3/model.py @@ -438,6 +438,10 @@ def _dataset_label(self, sub_name: str = None): @property def platform_abbreviated(self) -> Optional[str]: # For now from Alchemist the platform is always landsat for C3 processing + if "ls" not in self.dataset.platform: + raise ValueError( + "ComplicatedNamingConventionsDerivatives assumes lansat as platform" + ) return "ls" diff --git a/eodatasets3/properties.py b/eodatasets3/properties.py index 32761966..68f53bd4 100644 --- a/eodatasets3/properties.py +++ b/eodatasets3/properties.py @@ -226,7 +226,7 @@ class StacPropertyView(collections.abc.Mapping): "landsat:wrs_path": int, "landsat:wrs_row": int, "odc:dataset_version": None, - "odc:collection_number": None, + "odc:collection_number": int, "odc:naming_conventions": None, # Not strict as there may be more added in ODC... "odc:file_format": of_enum_type(FileFormat, strict=False),