Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: update wekeo sentinel product types #902

Merged
merged 15 commits into from Nov 20, 2023
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/index.rst
Expand Up @@ -37,7 +37,8 @@ types (Sentinel 1, Sentinel 2, Sentinel 3, Landsat, etc.) that can be searched a
`meteoblue <https://content.meteoblue.com/en/business-solutions/weather-apis/dataset-api>`_,
`cop_dataspace <https://dataspace.copernicus.eu/>`_,
`planetary_computer <https://planetarycomputer.microsoft.com/>`_,
`hydroweb_next <https://hydroweb.next.theia-land.fr>`_
`hydroweb_next <https://hydroweb.next.theia-land.fr>`_,
`wekeo <https://www.wekeo.eu/>`_

EODAG has the following primary features:

Expand Down
25 changes: 19 additions & 6 deletions eodag/api/product/metadata_mapping.py
Expand Up @@ -579,13 +579,14 @@ def convert_get_processing_level_from_s2_id(product_id):
def convert_split_id_into_s3_params(product_id):
parts = re.split(r"_(?!_)", product_id)
params = {"productType": product_id[4:15]}
start_date = datetime.strptime(
product_id[16:31], "%Y%m%dT%H%M%S"
) - timedelta(seconds=1)
dates = re.findall("[0-9]{8}T[0-9]{6}", product_id)
start_date = datetime.strptime(dates[0], "%Y%m%dT%H%M%S") - timedelta(
seconds=1
)
params["startDate"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
end_date = datetime.strptime(
product_id[32:47], "%Y%m%dT%H%M%S"
) + timedelta(seconds=1)
end_date = datetime.strptime(dates[1], "%Y%m%dT%H%M%S") + timedelta(
seconds=1
)
params["endDate"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
params["timeliness"] = parts[-2]
params["sat"] = "Sentinel-" + parts[0][1:]
Expand Down Expand Up @@ -705,6 +706,18 @@ def convert_get_ecmwf_time(date: str) -> list:
MetadataFormatter.convert_to_datetime_dict(date, "str")["hour"] + ":00"
]

@staticmethod
def convert_get_dates_from_string(text: str, split_param="-"):
reg = "[0-9]{8}" + split_param + "[0-9]{8}"
dates_str = re.search(reg, text).group()
dates = dates_str.split(split_param)
start_date = datetime.strptime(dates[0], "%Y%m%d")
end_date = datetime.strptime(dates[1], "%Y%m%d")
return {
"startDate": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
"endDate": end_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
}

# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
search_param = re.sub(
Expand Down
2 changes: 1 addition & 1 deletion eodag/plugins/search/data_request_search.py
Expand Up @@ -146,7 +146,7 @@ def query(self, *args, count=True, **kwargs):
)

# update dates if needed
if getattr(self.config, "dates_required", True):
if getattr(self.config, "dates_required", True) and "id" not in keywords:
if not keywords.get("startTimeFromAscendingNode", None):
keywords["startTimeFromAscendingNode"] = getattr(
self.config, "product_type_config", {}
Expand Down
296 changes: 288 additions & 8 deletions eodag/resources/product_types.yml

Large diffs are not rendered by default.

1,006 changes: 908 additions & 98 deletions eodag/resources/providers.yml

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions eodag/rest/utils.py
Expand Up @@ -9,6 +9,7 @@
import os
import re
from collections import namedtuple
from json import JSONDecodeError
from shutil import make_archive, rmtree
from typing import Dict, Optional

Expand Down Expand Up @@ -365,6 +366,12 @@ def get_arguments_query_paths(arguments):
:returns: Mapping of query paths with their corresponding values
:rtype: dict
"""
for a, value in arguments.items():
if isinstance(value, str):
try:
arguments[a] = json.loads(value)
except JSONDecodeError:
logger.warning("argument is not a dict")
return dict(
(str(match.full_path), match.value)
for match in string_to_jsonpath(STAC_QUERY_PATTERN, force=True).find(arguments)
Expand Down
21 changes: 19 additions & 2 deletions tests/units/test_core.py
Expand Up @@ -168,8 +168,8 @@ class TestCore(TestCoreBase):
"sara",
"wekeo",
],
"S1_SAR_OCN": ["cop_dataspace", "creodias", "onda", "peps", "sara"],
"S1_SAR_RAW": ["cop_dataspace", "creodias", "onda"],
"S1_SAR_OCN": ["cop_dataspace", "creodias", "onda", "peps", "sara", "wekeo"],
"S1_SAR_RAW": ["cop_dataspace", "creodias", "onda", "wekeo"],
"S1_SAR_SLC": [
"cop_dataspace",
"creodias",
Expand Down Expand Up @@ -203,6 +203,7 @@ class TestCore(TestCoreBase):
"sara",
"wekeo",
],
"S2_MSI_L2AP": ["wekeo"],
"S2_MSI_L2A_COG": ["earth_search_cog"],
"S2_MSI_L2A_MAJA": ["theia"],
"S2_MSI_L2B_MAJA_SNOW": ["theia"],
Expand Down Expand Up @@ -255,6 +256,21 @@ class TestCore(TestCoreBase):
"S3_SY_VG1": ["cop_dataspace", "creodias", "onda", "sara"],
"S3_SY_VGP": ["cop_dataspace", "creodias", "onda", "sara"],
"S3_WAT": ["cop_dataspace", "creodias", "onda", "sara", "wekeo"],
"S3_OLCI_L2WFR_BC003": ["wekeo"],
"S3_OLCI_L2WRR_BC003": ["wekeo"],
"S3_SRA_1A_BC004": ["wekeo"],
"S3_SRA_1B_BC004": ["wekeo"],
"S3_SRA_BS_BC004": ["wekeo"],
"S3_WAT_BC004": ["wekeo"],
"S3_SLSTR_L1RBT_BC004": ["wekeo"],
"S3_SLSTR_L2WST_BC003": ["wekeo"],
"S3_OLCI_L4BALTIC": ["wekeo"],
"S6_L1A_AHR": ["wekeo"],
"S6_L1B_ALR": ["wekeo"],
"S6_L1B_AHR": ["wekeo"],
"S6_L2_ALR": ["wekeo"],
"S6_L2_AHR": ["wekeo"],
"S6_L2_R": ["wekeo"],
"S5P_L1B2_IR_ALL": ["wekeo"],
"S5P_L1B_IR_SIR": ["cop_dataspace", "creodias"],
"S5P_L1B_IR_UVN": ["cop_dataspace", "creodias"],
Expand Down Expand Up @@ -1243,6 +1259,7 @@ def test_guess_product_type_with_kwargs(self):
expected = [
"S2_MSI_L1C",
"S2_MSI_L2A",
"S2_MSI_L2AP",
"S2_MSI_L2A_COG",
"S2_MSI_L2A_MAJA",
"S2_MSI_L2B_MAJA_SNOW",
Expand Down
16 changes: 16 additions & 0 deletions tests/units/test_metadata_mapping.py
Expand Up @@ -498,3 +498,19 @@ def test_convert_get_ecmwf_time(self):
self.assertEqual(
format_metadata(to_format, date="2023-01-31T23:59"), str(["23:00"])
)

def test_convert_get_dates_from_string(self):
to_format = "{text#get_dates_from_string}"
self.assertEqual(
format_metadata(to_format, text="20231019-20231020"),
str(
{"startDate": "2023-10-19T00:00:00Z", "endDate": "2023-10-20T00:00:00Z"}
),
)
to_format = "{text#get_dates_from_string(_)}"
self.assertEqual(
format_metadata(to_format, text="20231019_20231020"),
str(
{"startDate": "2023-10-19T00:00:00Z", "endDate": "2023-10-20T00:00:00Z"}
),
)