Update FR production parser to use other API (#1560)

* RTE API is not meant to be an open API, use opendata.reseaux-energies.fr * Add special case for hydro calculation * rows parameter to return all datetimes * Update sources * Add contributor * Read apikey from os.environ
electricitymaps · Aug 16, 2018 · 37c79b5 · 37c79b5
1 parent 6b53a19
commit 37c79b5
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 74 deletions.
diff --git a/README.md b/README.md
@@ -103,7 +103,7 @@ Real-time electricity data is obtained using [parsers](https://github.com/tmrowc
 - Estonia: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html)
 - Faroe Islands: [SEV](https://w3.sev.fo/framleidsla/)
 - Finland: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html)
-- France: [RTE](http://www.rte-france.com/en/eco2mix/eco2mix-mix-energetique-en)
+- France: [RTE](https://opendata.reseaux-energies.fr)
 - Germany: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html)
 - Georgia: [GSE](http://www.gse.com.ge/home)
 - Great Britain: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html)

diff --git a/config/zones.json b/config/zones.json
@@ -1383,7 +1383,8 @@
       "wind": 13550
     },
     "contributors": [
-      "https://github.com/corradio"
+      "https://github.com/corradio",
+      "https://github.com/lorrieq"
     ],
     "parsers": {
       "consumption": "ENTSOE.fetch_consumption",

diff --git a/parsers/FR.py b/parsers/FR.py
@@ -1,92 +1,90 @@
 #!/usr/bin/env python3
 
 import arrow
+import json
 import logging
+import os
+import pandas as pd
 import requests
 import xml.etree.ElementTree as ET
 
+API_ENDPOINT = 'https://opendata.reseaux-energies.fr/api/records/1.0/search/'
+
 MAP_GENERATION = {
-    u'Nucl\xe9aire': 'nuclear',
-    'Charbon': 'coal',
-    'Gaz': 'gas',
-    'Fioul': 'oil',
-    'Hydraulique': 'hydro',
-    'Eolien': 'wind',
-    'Solaire': 'solar',
-    'Autres': 'biomass'
-}
-MAP_STORAGE = {
-    'Pompage': 'hydro',
-    'Hydraulique': 'hydro',
+    'nucleaire': 'nuclear',
+    'charbon': 'coal',
+    'gaz': 'gas',
+    'fioul': 'oil',
+    'eolien': 'wind',
+    'solaire': 'solar',
+    'bioenergies': 'biomass'
 }
 
+MAP_HYDRO = [
+    'hydraulique_fil_eau_eclusee',
+    'hydraulique_lacs',
+    'hydraulique_step_turbinage',
+    'pompage'
+]
+
 
 def fetch_production(zone_key='FR', session=None, target_datetime=None,
                      logger=logging.getLogger(__name__)):
     if target_datetime:
-        now = arrow.get(target_datetime, 'Europe/Paris')
+        to = arrow.get(target_datetime, 'Europe/Paris')
     else:
-        now = arrow.now(tz='Europe/Paris')
+        to = arrow.now(tz='Europe/Paris')
 
+    # setup request
     r = session or requests.session()
-    formatted_from = now.shift(days=-1).format('DD/MM/YYYY')
-    formatted_to = now.format('DD/MM/YYYY')
-    url = 'http://www.rte-france.com/getEco2MixXml.php?type=mix&dateDeb={}&' \
-          'dateFin={}&mode=NORM'.format(formatted_from, formatted_to)
-    response = r.get(url)
-    obj = ET.fromstring(response.content)
-    datas = {}
-    mixtr = obj[7]
-
-    for mixtr in obj:
-        if mixtr.tag != 'mixtr':
-            continue
-
-        start_date = arrow.get(arrow.get(mixtr.attrib['date']).datetime, 'Europe/Paris')
-
-        # Iterate over mix
-        for item in mixtr:
-            key = item.get('v')
-            granularite = item.get('granularite')
-            value = None
-            # Iterate over time
-            for value in item:
-                period = int(value.attrib['periode'])
-                datetime = start_date.replace(minutes=+(period * 15.0)).datetime
-                if not datetime in datas:
-                    datas[datetime] = {
-                        'zoneKey': zone_key,
-                        'datetime': datetime,
-                        'production': {},
-                        'storage': {},
-                        'source': 'rte-france.com',
-                    }
-                data = datas[datetime]
-
-                if key == 'Hydraulique':
-                    # Hydro is a special case!
-                    if granularite == 'Global':
-                        continue
-                    elif granularite in ['FEE', 'LAC']:
-                        if not MAP_GENERATION[key] in data['production']:
-                            data['production'][MAP_GENERATION[key]] = 0
-                        # Run of the river or conventional
-                        data['production'][MAP_GENERATION[key]] += float(
-                            value.text)
-                    elif granularite == 'STT':
-                        if not MAP_STORAGE[key] in data['storage']:
-                            data['storage'][MAP_STORAGE[key]] = 0
-                        # Pumped storage generation
-                        data['storage'][MAP_STORAGE[key]] += -1 * float(value.text)
-                elif granularite == 'Global':
-                    if key in MAP_GENERATION:
-                        data['production'][MAP_GENERATION[key]] = float(value.text)
-                    elif key in MAP_STORAGE:
-                        if not MAP_STORAGE[key] in data['storage']:
-                            data['storage'][MAP_STORAGE[key]] = 0
-                        data['storage'][MAP_STORAGE[key]] += -1 * float(value.text)
-
-    return list(datas.values())
+    formatted_from = to.shift(days=-1).format('YYYY-MM-DDTHH:mm')
+    formatted_to = to.format('YYYY-MM-DDTHH:mm')
+
+    params = {
+        'dataset': 'eco2mix-national-tr',
+        'q': 'date_heure >= {} AND date_heure <= {}'.format(
+            formatted_from, formatted_to),
+        'timezone': 'Europe/Paris',
+        'rows': 100
+    }
+
+    if 'RESEAUX_ENERGIES_TOKEN' not in os.environ:
+        raise Exception(
+            'No RESEAUX_ENERGIES_TOKEN found! Please add it into secrets.env!')
+    params['apikey'] = os.environ['RESEAUX_ENERGIES_TOKEN']
+
+    # make request and create dataframe with response
+    response = r.get(API_ENDPOINT, params=params)
+    data = json.loads(response.content)
+    data = [d['fields'] for d in data['records']]
+    df = pd.DataFrame(data)
+
+    # filter out desired columns and convert values to float
+    value_columns = MAP_GENERATION.keys() + MAP_HYDRO
+    df = df[['date_heure'] + value_columns]
+    df[value_columns] = df[value_columns].astype(float)
+
+    datapoints = list()
+    for row in df.iterrows():
+        production = dict()
+        for key, value in MAP_GENERATION.items():
+            production[value] = row[1][key]
+
+        # Hydro is a special case!
+        production['hydro'] = row[1]['hydraulique_lacs'] + row[1]['hydraulique_fil_eau_eclusee']
+        storage = {
+            'hydro': row[1]['pompage'] * -1 + row[1]['hydraulique_step_turbinage'] * -1
+        }
+
+        datapoints.append({
+            'zoneKey': zone_key,
+            'datetime': arrow.get(row[1]['date_heure']).datetime,
+            'production': production,
+            'storage': storage,
+            'source': 'opendata.reseaux-energies.fr'
+        })
+
+    return datapoints
 
 
 def fetch_price(zone_key, session=None, target_datetime=None,