DK real-time data

README.md

@@ -99,7 +99,7 @@ Real-time electricity data is obtained using [parsers](https://github.com/tmrowc
 - Costa Rica: [ICE](https://appcenter.grupoice.com/CenceWeb/CencePosdespachoNacional.jsf)
 - Croatia (Exchanges): [HOPS](https://www.hops.hr/wps/portal/hr/web)
 - Cyprus : [TSO](http://www.dsm.org.cy/en/daily-system-generation-on-the-transmission-system-mw)
-- Denmark: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html)
+- Denmark: [TSO](https://www.energidataservice.dk/en/group/production-and-consumption)
 - Denmark (Bornholm): [PowerlabDK](http://bornholm.powerlab.dk/)
 - Dominican Republic: [OC](http://www.oc.org.do/Reportes/postdespacho.aspx)
 - El Salvador: [Unidad de Transacciones](http://estadistico.ut.com.sv/OperacionDiaria.aspx)

config/exchanges.json

@@ -722,7 +722,7 @@
       54.9
     ],
     "parsers": {
-      "exchange": "statnett.fetch_exchange",
+      "exchange": "DK.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
     },
     "rotation": 0
@@ -737,7 +737,7 @@
       54.486043
     ],
     "parsers": {
-      "exchange": "statnett.fetch_exchange",
+      "exchange": "DK.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
     },
     "rotation": 0
@@ -830,7 +830,7 @@
       55.407235
     ],
     "parsers": {
-      "exchange": "statnett.fetch_exchange_by_bidding_zone",
+      "exchange": "DK.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
     },
     "rotation": 90
@@ -846,7 +846,7 @@
     ],
     "parsers": {
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast",
-      "exchange": "statnett.fetch_exchange"
+      "exchange": "DK.fetch_exchange"
     },
     "rotation": -25
   },
@@ -874,7 +874,7 @@
       56.857802
     ],
     "parsers": {
-      "exchange": "statnett.fetch_exchange"
+      "exchange": "DK.fetch_exchange"
     },
     "rotation": 70
   },
@@ -894,7 +894,7 @@
       55.952282
     ],
     "parsers": {
-      "exchange": "ENTSOE.fetch_exchange"
+      "exchange": "DK.fetch_exchange"
     },
     "rotation": 70
   },

config/zones.json

@@ -1026,15 +1026,16 @@
       "wind": 3809
     },
     "contributors": [
-      "https://github.com/corradio"
+      "https://github.com/corradio",
+      "https://github.com/tmslaine"
     ],
     "parsers": {
       "consumption": "ENTSOE.fetch_consumption",
       "consumptionForecast": "ENTSOE.fetch_consumption_forecast",
       "generationForecast": "ENTSOE.fetch_generation_forecast",
       "productionPerModeForecast": "ENTSOE.fetch_wind_solar_forecasts",
       "price": "ENTSOE.fetch_price",
-      "production": "ENTSOE.fetch_production",
+      "production": "DK.fetch_production",
       "productionPerUnit": "ENTSOE.fetch_production_per_units"
     },
     "timezone": "Europe/Copenhagen"
@@ -1064,15 +1065,16 @@
       "wind": 1036
     },
     "contributors": [
-      "https://github.com/corradio"
+      "https://github.com/corradio",
+      "https://github.com/tmslaine"
     ],
     "parsers": {
       "consumption": "ENTSOE.fetch_consumption",
       "consumptionForecast": "ENTSOE.fetch_consumption_forecast",
       "generationForecast": "ENTSOE.fetch_generation_forecast",
       "productionPerModeForecast": "ENTSOE.fetch_wind_solar_forecasts",
       "price": "ENTSOE.fetch_price",
-      "production": "ENTSOE.fetch_production",
+      "production": "DK.fetch_production",
       "productionPerUnit": "ENTSOE.fetch_production_per_units"
     },
     "timezone": "Europe/Copenhagen"

parsers/DK.py

@@ -0,0 +1,189 @@
+import logging
+import datetime
+
+import pandas as pd
+# The arrow library is used to handle datetimes
+import arrow
+# The request library is used to fetch content through HTTP
+import requests
+
+
+ids = {'real_time':'06380963-b7c6-46b7-aec5-173d15e4648b',
+       'energy_bal':'02356e88-7c4e-4ee9-b896-275d217cc1b9'}
+
+def fetch_production(zone_key='DK-DK1', session=None,target_datetime=None,
+                     logger: logging.Logger = logging.getLogger(__name__)):
+    """
+    Queries "Electricity balance Non-Validated" from energinet api
+    for Danish bidding zones
+    """
+    r = session or requests.session()
+
+
+
+    if zone_key not in ['DK-DK1', 'DK-DK2']:
+        raise NotImplementedError(
+            'fetch_production() for {} not implemented'.format(zone_key))
+
+    zone = zone_key[-3:]
+
+    timestamp = arrow.get(target_datetime).strftime('%Y-%m-%d %H:%M')
+
+    # fetch hourly energy balance from recent hours
+    sqlstr = 'SELECT "HourUTC" as timestamp, "Biomass", "Waste", \
+                     "OtherRenewable", "FossilGas" as gas, "FossilHardCoal" as coal, \
+                     "FossilOil" as oil, "HydroPower" as hydro, \
+                     ("OffshoreWindPower"%2B"OnshoreWindPower") as wind, \
+                     "SolarPower" as solar from "{0}" \
+                     WHERE "PriceArea" = \'{1}\' AND \
+                     "HourUTC" >= (timestamp\'{2}\'-INTERVAL \'8 hours\') AND \
+                     "HourUTC" <= timestamp\'{2}\' \
+                     ORDER BY "HourUTC" ASC'.format(ids['energy_bal'], zone, timestamp)
+
+
+    url = 'https://api.energidataservice.dk/datastore_search_sql?sql={}'.format(sqlstr)
+
+    response = r.get(url)
+
+    assert response.status_code == 200 and response.json()['result']['records'] != [], \
+        'Exception when fetching production for ' \
+        '{}: error when calling url={}'.format(zone_key, url)
+
+    df = pd.DataFrame(response.json()['result']['records'])
+    # index response dataframe by time
+    df = df.set_index('timestamp')
+    df.index = pd.DatetimeIndex(df.index)
+    # drop empty rows from energy balance
+    df.dropna(how='all', inplace=True)
+
+    # Divide waste into 55% renewable and 45% non-renewable parts according to
+    # https://ens.dk/sites/ens.dk/files/Statistik/int.reporting_2016.xls (visited Jan 24th, 2019)
+    df['unknown'] = 0.45*df['Waste'] # Report fossil waste as unknown
+    df['renwaste'] = 0.55*df['Waste']
+    # Report biomass, renewable waste and other renewables (biogas etc.) as biomass
+    df['biomass'] = df.filter(['Biomass', 'renwaste', 'OtherRenewable']).sum(axis=1)
+
+    fuels = ['biomass', 'coal', 'oil', 'gas', 'unknown', 'hydro']
+    # Format output as a list of dictionaries
+    output = []
+    for dt in df.index:
+
+        data = {
+              'zoneKey': zone_key,
+              'datetime': None,
+              'production': {
+                  'biomass': 0,
+                  'coal': 0,
+                  'gas': 0,
+                  'hydro': None,
+                  'nuclear': 0,
+                  'oil': 0,
+                  'solar': None,
+                  'wind': None,
+                  'geothermal': None,
+                  'unknown': 0
+              },
+              'storage': {},
+              'source': 'api.energidataservice.dk'
+            }
+
+        data['datetime'] = dt.to_pydatetime()
+        for f in ['solar', 'wind']+fuels:
+            data['production'][f] = df.loc[dt,f]
+        output.append(data)
+    return output
+
+def fetch_exchange(zone_key1='DK-DK1', zone_key2='DK-DK2', session=None,
+                   target_datetime=None, logger=logging.getLogger(__name__)):
+
+    """
+    Fetches 5-minute frequency exchange data for Danish bidding zones
+    from api.energidataservice.dk
+    """
+    r = session or requests.session()
+    sorted_keys = '->'.join(sorted([zone_key1, zone_key2]))
+
+    # pick the correct zone to search
+    if 'DK1' in sorted_keys and 'DK2' in sorted_keys:
+        zone = 'DK1'
+    elif 'DK1' in sorted_keys:
+        zone = 'DK1'
+    elif 'DK2' in sorted_keys:
+        zone = 'DK2'
+    else:
+        raise NotImplementedError(
+            'Only able to fetch exchanges for Danish bidding zones')
+
+
+    exch_map = {
+        'DE->DK-DK1':'"ExchangeGermany"',
+        'DE->DK-DK2':'"ExchangeGermany"',
+        'DK-DK1->DK-DK2':'"ExchangeGreatBelt"',
+        'DK-DK1->NO-NO2':'"ExchangeNorway"',
+        'DK-DK1->NL':'"ExchangeNetherlands"',
+        'DK-DK1->SE':'"ExchangeSweden"',
+        'DK-DK1->SE-SE3':'"ExchangeSweden"',
+        'DK-DK2->SE-SE':'("ExchangeSweden" - "BornholmSE4")',# Exchange from Bornholm to Sweden is included in "ExchangeSweden"
+        'DK-DK2->SE-SE4':'("ExchangeSweden" - "BornholmSE4")' #but Bornholm island is reported separately from DK-DK2 in eMap
+
+    }
+    if sorted_keys not in exch_map:
+        raise NotImplementedError(
+            'Exchange {} not implemented'.format(sorted_keys))
+
+    timestamp = arrow.get(target_datetime).strftime('%Y-%m-%d %H:%M')
+
+
+
+
+    # fetch real-time/5-min data
+    sqlstr = 'SELECT "Minutes5UTC" as timestamp, {0} as "netFlow" \
+                     from "{1}" WHERE "PriceArea" = \'{2}\' AND \
+                     "Minutes5UTC" >= (timestamp\'{3}\'-INTERVAL \'8 hours\') AND \
+                     "Minutes5UTC" <= timestamp\'{3}\' \
+                     ORDER BY "Minutes5UTC" ASC'.format(exch_map[sorted_keys],
+                                                        ids['real_time'],
+                                                        zone,
+                                                        timestamp)
+
+    url = 'https://api.energidataservice.dk/datastore_search_sql?sql={}'.format(sqlstr)
+
+    response = r.get(url)
+
+    assert response.status_code == 200 and response.json()['result']['records'] != [], \
+        'Exception when fetching flow for ' \
+        '{}: error when calling url={}'.format(sorted_keys, url)
+
+    df = pd.DataFrame(response.json()['result']['records'])
+    df = df.set_index('timestamp')
+    df.index = pd.DatetimeIndex(df.index)
+    # drop empty rows
+    df.dropna(how='all', inplace=True)
+
+    # all exchanges are reported as net import,
+    # where as eMap expects net export from
+    # the first zone in alphabetical order
+    if 'DE' not in sorted_keys:
+        df['netFlow'] = -1*df['netFlow']
+    # Format output
+    output = []
+    for dt in df.index:
+
+        data = {
+            'sortedZoneKeys': sorted_keys,
+            'datetime': None,
+            'netFlow': None,
+            'source': 'api.energidataservice.dk'
+        }
+
+        data['datetime'] = dt.to_pydatetime()
+        data['netFlow'] = df.loc[dt,'netFlow']
+        output.append(data)
+    return output
+
+if __name__ == '__main__':
+    print('fetch_production() ->')
+    print(fetch_production())
+    print('fetch_exchange(DK-DK2, SE-SE4) ->')
+    print(fetch_exchange('DK-DK2', 'SE-SE4'))
+