-
Notifications
You must be signed in to change notification settings - Fork 902
/
CL.py
222 lines (170 loc) · 6.8 KB
/
CL.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
#!/usr/bin/env python3
"""Parser for the electricity grid of Chile"""
from collections import defaultdict
from datetime import datetime, timedelta
from logging import Logger, getLogger
from operator import itemgetter
from typing import Optional
import arrow
from requests import Session
from parsers.lib.config import refetch_frequency
from .lib.exceptions import ParserException
from .lib.validation import validate
# Historical API
API_BASE_URL = "https://sipub.coordinador.cl/api/v1/recursos/generacion_centrales_tecnologia_horario?"
# Live API
API_BASE_URL_LIVE_TOT = "http://panelapp.coordinadorelectrico.cl/api/chart/demanda"
API_BASE_URL_LIVE_REN = "http://panelapp.coordinadorelectrico.cl/api/chart/ernc" # ERNC = energias renovables no convencionales
# Live parser disabled because of a insuffcient breakdown of Unknown.
# It lumps Hydro & Geothermal into unknown which makes it difficult to calculate a proper CFE%/co2 intensity
# We can reenable if we find a way to get the Hydro production
ENABLE_LIVE_PARSER = False
TYPE_MAPPING = {
"hidraulica": "hydro",
"termica": "unknown",
"eolica": "wind",
"solar": "solar",
"geotermica": "geothermal",
}
def get_data_live(session: Optional[Session], logger: Logger):
"""Requests live generation data in json format."""
s = session or Session()
json_total = s.get(API_BASE_URL_LIVE_TOT).json()
json_ren = s.get(API_BASE_URL_LIVE_REN).json()
return json_total, json_ren
def production_processor_live(json_tot, json_ren):
"""
Extracts generation data and timestamp into dictionary.
Returns a list of dictionaries for all of the available "live" data, usually that day.
"""
gen_total = json_tot["data"][0]["values"]
if json_ren["data"][1]["key"] == "ENERGÍA SOLAR":
rawgen_sol = json_ren["data"][1]["values"]
else:
raise ParserException(
"CL.py",
f"Unexpected data label. Expected 'ENERGÍA SOLAR' and got {json_ren['data'][1]['key']}",
"CL",
)
if json_ren["data"][0]["key"] == "ENERGÍA EÓLICA":
rawgen_wind = json_ren["data"][0]["values"]
else:
raise ParserException(
"CL.py",
f"Unexpected data label. Expected 'ENERGÍA EÓLICA' and got {json_ren['data'][0]['key']}",
"CL",
)
mapped_totals = []
for total in gen_total:
datapoint = {}
dt = total[0]
for pair in rawgen_sol:
if pair[0] == dt:
solar = pair[1]
break
for pair in rawgen_wind:
if pair[0] == dt:
wind = pair[1]
break
datapoint["datetime"] = arrow.get(
dt / 1000, tzinfo="Chile/Continental"
).datetime
datapoint["unknown"] = total[1] - wind - solar
datapoint["wind"] = wind
datapoint["solar"] = solar
mapped_totals.append(datapoint)
return mapped_totals
def production_processor_historical(raw_data):
"""Takes raw json data and groups by datetime while mapping generation to type.
Returns a list of dictionaries.
"""
clean_datapoints = []
for datapoint in raw_data:
clean_datapoint = {}
date, hour = datapoint["fecha"], datapoint["hora"]
hour -= 1 # `hora` starts at 1
date = arrow.get(date, "YYYY-MM-DD", tzinfo="Chile/Continental").shift(
hours=hour
)
clean_datapoint["datetime"] = date.datetime
gen_type_es = datapoint["tipo_central"]
mapped_gen_type = TYPE_MAPPING[gen_type_es]
value_mw = float(datapoint["generacion_sum"])
clean_datapoint[mapped_gen_type] = value_mw
clean_datapoints.append(clean_datapoint)
combined = defaultdict(dict)
for elem in clean_datapoints:
combined[elem["datetime"]].update(elem)
ordered_data = sorted(combined.values(), key=itemgetter("datetime"))
if ENABLE_LIVE_PARSER:
# For consistency with live API, hydro and geothermal must be squeezed into unknown
for datapoint in ordered_data:
if "unknown" not in datapoint:
datapoint["unknown"] = 0
if "hydro" in datapoint:
datapoint["unknown"] += datapoint["hydro"]
del datapoint["hydro"]
if "geothermal" in datapoint:
datapoint["unknown"] += datapoint["geothermal"]
del datapoint["geothermal"]
return ordered_data
@refetch_frequency(timedelta(days=1))
def fetch_production(
zone_key: str = "CL-SEN",
session: Optional[Session] = None,
target_datetime: Optional[datetime] = None,
logger: Logger = getLogger(__name__),
):
if target_datetime is None and ENABLE_LIVE_PARSER:
gen_tot, gen_ren = get_data_live(session, logger)
processed_data = production_processor_live(gen_tot, gen_ren)
data = []
for production_data in processed_data:
dt = production_data.pop("datetime")
datapoint = {
"zoneKey": zone_key,
"datetime": dt,
"production": production_data,
"storage": {
"hydro": None,
},
"source": "coordinadorelectrico.cl",
}
datapoint = validate(datapoint, logger, remove_negative=True, floor=1000)
data.append(datapoint)
return data
arr_target_datetime = arrow.get(target_datetime)
start = arr_target_datetime.shift(days=-1).format("YYYY-MM-DD")
end = arr_target_datetime.format("YYYY-MM-DD")
date_component = "fecha__gte={}&fecha__lte={}".format(start, end)
# required for access
headers = {
"Referer": "https://www.coordinador.cl/operacion/graficos/operacion-real/generacion-real-del-sistema/",
"Origin": "https://www.coordinador.cl",
}
s = session or Session()
url = API_BASE_URL + date_component
req = s.get(url, headers=headers)
raw_data = req.json()["aggs"]
processed_data = production_processor_historical(raw_data)
data = []
for production_data in processed_data:
dt = production_data.pop("datetime")
datapoint = {
"zoneKey": zone_key,
"datetime": dt,
"production": production_data,
"storage": {
"hydro": None,
},
"source": "coordinador.cl",
}
data.append(datapoint)
return data[:-9]
"""The last 9 datapoints should be omitted because they usually are incomplete and shouldn't appear on the map."""
if __name__ == "__main__":
"""Main method, never used by the Electricity Map backend, but handy for testing."""
print("fetch_production() ->")
print(fetch_production())
# For fetching historical data instead, try:
print(fetch_production(target_datetime=arrow.get("20200220", "YYYYMMDD")))