Add support for fetching historical data for all French overseas regions

parsers/FR_O.py

@@ -1,5 +1,6 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from logging import getLogger
+from typing import NamedTuple
 
 from requests import Response, Session
 
@@ -14,6 +15,7 @@
 )
 from electricitymap.contrib.lib.types import ZoneKey
 
+from .lib.config import refetch_frequency
 from .lib.exceptions import ParserException
 
 DOMAIN_MAPPING = {
@@ -32,12 +34,12 @@
     "MQ": "production-delectricite-par-filiere-en-temps-reel",
 }
 
-HISTORICAL_DATASETS = {
-    "FR-COR": "production-delectricite-par-filiere",
-    "RE": "courbe-de-charge-de-la-production-delectricite-par-filiere",
-    "GF": "courbe-de-charge-de-la-production-delectricite-par-filiere",
-    "MQ": "courbe-de-charge-de-la-production-delectricite-par-filiere",
-    "GP": "courbe-de-charge-de-la-production-delectricite-par-filiere",
+HISTORICAL_MAPPING = {
+    "FR-COR": "Corse",
+    "RE": "Réunion",
+    "GF": "Guyane",
+    "MQ": "Martinique",
+    "GP": "Guadeloupe",
 }
 
 API_PARAMETER_GROUPS = {
@@ -115,50 +117,45 @@
 
 
 def generate_url(zone_key, target_datetime):
-    return f"{DOMAIN_MAPPING[zone_key]}/api/v2/catalog/datasets/{HISTORICAL_DATASETS[zone_key] if target_datetime else LIVE_DATASETS[zone_key]}/exports/json"
-
-
-def generate_source(zone_key: ZoneKey):
-    # Return the domain name of the source without the protocol
-    return DOMAIN_MAPPING[zone_key].split("//")[1]
+    if target_datetime:
+        return "https://opendata.edf.fr/api/explore/v2.1/catalog/datasets/courbe-de-charge-de-la-production-delectricite-par-filiere/exports/json"
+    return f"{DOMAIN_MAPPING[zone_key]}/api/v2/catalog/datasets/{LIVE_DATASETS[zone_key]}/exports/json"
 
 
 def fetch_data(
     zone_key: ZoneKey,
     session: Session | None = None,
     target_datetime: datetime | None = None,
-) -> tuple[list, str]:
+) -> tuple[list, str, str]:
     ses = session or Session()
 
-    DATE_STRING_MAPPING = {
-        "FR-COR": "date_heure" if target_datetime else "date",
-        "RE": "date_heure" if target_datetime else "date",
-        "GF": "date",
-        "MQ": "date_heure" if target_datetime else "date",
-        "GP": "date",
-    }
-
-    if target_datetime and zone_key not in HISTORICAL_DATASETS.keys():
-        raise ParserException(
-            "FR_O.py",
-            f"Historical data not implemented for {zone_key} in this parser.",
-            zone_key,
-        )
-    elif target_datetime is None and zone_key not in LIVE_DATASETS.keys():
+    if target_datetime is None and zone_key not in LIVE_DATASETS.keys():
         raise ParserException(
             "FR_O.py",
             f"Live data not implemented for {zone_key} in this parser.",
             zone_key,
         )
 
-    URL_QUERIES: dict[str, str | None] = {
-        #   "refine": "statut:Validé" if target_datetime else None,
-        "timezone": "UTC",
-        "order_by": f"{DATE_STRING_MAPPING[zone_key]} desc",
-        "refine": f"{DATE_STRING_MAPPING[zone_key]}:{target_datetime.strftime('%Y')}"
+    target_date = target_datetime.strftime("%Y-%m-%d") if target_datetime else None
+    past_date = (
+        (target_datetime - timedelta(days=3)).strftime("%Y-%m-%d")
+        if target_datetime
+        else None
+    )
+
+    URL_QUERIES: dict[str, str | None] = (
+        {
+            "timezone": "UTC",
+            "order_by": "date_heure",
+            "where": f"date_heure >= date'{past_date}' AND date_heure <= date'{target_date}'",
+            "refine": f"territoire:{HISTORICAL_MAPPING[zone_key]}",
+        }
         if target_datetime
-        else None,
-    }
+        else {
+            "timezone": "UTC",
+            "order_by": "date",
+        }
+    )
 
     url = generate_url(zone_key, target_datetime)
     response: Response = ses.get(url, params=URL_QUERIES)
@@ -185,21 +182,25 @@
     if not isinstance(data, list):
         raise ParserException(
             "FR_O.py",
-            f"Unexpected data format for {zone_key} for {target_datetime.strftime('%Y')}"
+            f"Unexpected data format for {zone_key} for {target_datetime}"
             if target_datetime
             else f"Unexpected data format for {zone_key}.",
             zone_key,
         )
-    return data, DATE_STRING_MAPPING[zone_key]
+    source = url.split("//")[1].split("/")[0]
+    return data, "date_heure" if target_datetime else "date", source
 
 
+@refetch_frequency(timedelta(hours=72))
 def fetch_production(
     zone_key: ZoneKey,
     session: Session | None = None,
     target_datetime: datetime | None = None,
     logger=getLogger(__name__),
 ):
-    production_objects, date_string = fetch_data(zone_key, session, target_datetime)
+    production_objects, date_string, source = fetch_data(
+        zone_key, session, target_datetime
+    )
 
     production_breakdown_list = ProductionBreakdownList(logger=logger)
     for production_object in production_objects:
@@ -228,21 +229,22 @@
             datetime=datetime.fromisoformat(production_object[date_string]),
             production=production,
             storage=storage,
-            source=generate_source(zone_key),
+            source=source,
             sourceType=EventSourceType.estimated
             if production_object.get("statut") == "Estimé"
             else EventSourceType.measured,
         )
     return production_breakdown_list.to_list()
 
 
+@refetch_frequency(timedelta(hours=72))
 def fetch_price(
     zone_key: ZoneKey,
     session: Session | None = None,
     target_datetime: datetime | None = None,
     logger=getLogger(__name__),
 ):
-    data_objects, date_string = fetch_data(zone_key, session, target_datetime)
+    data_objects, date_string, source = fetch_data(zone_key, session, target_datetime)
 
     price_list = PriceList(logger=logger)
     for data_object in data_objects:
@@ -256,7 +258,7 @@
                 zoneKey=zone_key,
                 currency="EUR",
                 datetime=datetime.fromisoformat(data_object[date_string]),
-                source=generate_source(zone_key),
+                source=source,
                 price=price,
             )
     return price_list.to_list()