Skip to content

Commit

Permalink
refactor: upgrade CA-NS with event classes
Browse files Browse the repository at this point in the history 
Upgrade the parser to use the ExchangeList, ProductionBreakdownList,
ProductionMix, and ZoneKey classes. This should yield no functional
change. Additionally:

- Comply with Black's 88-column default line length limit.
- Define global constants to replace a number of local variables and
  literals.
- Try to use more consistent names throughout.

Refs: #6011, #6050
  • Loading branch information
@kruschk
kruschk committed Oct 25, 2023
1 parent e53c5a4 commit a0d61d6
Showing 1 changed file with 139 additions and 142 deletions.
281 changes: 139 additions & 142 deletions parsers/CA_NS.py
View file
Original file line number Diff line number Diff line change
@@ -1,213 +1,210 @@
#!/usr/bin/env python3

# The datetime library is used to handle datetimes
from datetime import datetime, timezone
from logging import Logger, getLogger
from typing import Any

from requests import Session


def _get_ns_info(requests_obj, logger: Logger):
zone_key = "CA-NS"

# This is based on validation logic in https://www.nspower.ca/site/renewables/assets/js/site.js
# In practical terms, I've seen hydro production go way too high (>70%) which is way more
# than reported capacity.
valid_percent = {
# The validation JS reports error when Solid Fuel (coal) is over 85%,
# but as far as I can tell, that can actually be a valid result, I've seen it a few times.
# Use 98% instead.
"coal": (0, 0.98),
"gas": (0, 0.5),
"biomass": (0, 0.15),
"hydro": (0, 0.60),
"wind": (0, 0.55),
"imports": (0, 0.50),
}

# Sanity checks: verify that reported production doesn't exceed listed capacity by a lot.
# In particular, we've seen error cases where hydro production ends up calculated as 900 MW
# which greatly exceeds known capacity of 418 MW.
valid_absolute = {
"coal": 1300,
"gas": 700,
"biomass": 100,
"hydro": 500,
"wind": 700,
}

mix_url = "https://www.nspower.ca/library/CurrentLoad/CurrentMix.json"
mix_data = requests_obj.get(mix_url).json()

load_url = "https://www.nspower.ca/library/CurrentLoad/CurrentLoad.json"
load_data = requests_obj.get(load_url).json()

production = []
imports = []
for mix in mix_data:
percent_mix = {
"coal": mix["Solid Fuel"] / 100.0,
"gas": (mix["HFO/Natural Gas"] + mix["CT's"] + mix["LM 6000's"]) / 100.0,
"biomass": mix["Biomass"] / 100.0,
"hydro": mix["Hydro"] / 100.0,
"wind": mix["Wind"] / 100.0,
"imports": mix["Imports"] / 100.0,
from electricitymap.contrib.config import ZoneKey
from electricitymap.contrib.lib.models.event_lists import (
ExchangeList,
ProductionBreakdownList,
)
from electricitymap.contrib.lib.models.events import ProductionMix
from parsers.lib.exceptions import ParserException

LOAD_URL = "https://www.nspower.ca/library/CurrentLoad/CurrentLoad.json"
MIX_URL = "https://www.nspower.ca/library/CurrentLoad/CurrentMix.json"
PARSER = "CA_NS.py"
SOURCE = "nspower.ca"
# Sanity checks: verify that reported production doesn't exceed listed capacity
# by a lot. In particular, we've seen error cases where hydro production ends
# up calculated as 900 MW which greatly exceeds known capacity of 418 MW.
MEGAWATT_LIMITS = {
"coal": 1300,
"gas": 700,
"biomass": 100,
"hydro": 500,
"wind": 700,
}
# This is based on validation logic in
# https://www.nspower.ca/site/renewables/assets/js/site.js. In practical terms,
# I've seen hydro production go way too high (>70%) which is way more than
# reported capacity.
FRACTION_LIMITS = {
# The validation JS reports an error when Solid Fuel (coal) is over 85%,
# but as far as I can tell, that can actually be a valid result, I've seen
# it a few times. Use 98% instead.
"coal": (0, 0.98),
"gas": (0, 0.5),
"biomass": (0, 0.15),
"hydro": (0, 0.60),
"wind": (0, 0.55),
"imports": (0, 0.50),
}
ZONE_KEY = ZoneKey("CA-NS")


def _get_ns_info(
session: Session, logger: Logger
) -> (ExchangeList, ProductionBreakdownList):
base_loads = session.get(LOAD_URL).json() # Base loads in MW
mixes_percent = session.get(MIX_URL).json() # Electricity breakdowns in %

exchanges = ExchangeList(logger)
production_breakdowns = ProductionBreakdownList(logger)
for mix_percent in mixes_percent:
mix_fraction = {
"coal": mix_percent["Solid Fuel"] / 100.0,
"gas": (
mix_percent["HFO/Natural Gas"]
+ mix_percent["CT's"]
+ mix_percent["LM 6000's"]
)
/ 100.0,
"biomass": mix_percent["Biomass"] / 100.0,
"hydro": mix_percent["Hydro"] / 100.0,
"wind": mix_percent["Wind"] / 100.0,
"imports": mix_percent["Imports"] / 100.0,
}

# datetime is in format '/Date(1493924400000)/'
# get the timestamp 1493924400 (cutting out last three zeros as well)
data_timestamp = int(mix["datetime"][6:-5])
data_date = datetime.fromtimestamp(data_timestamp, tz=timezone.utc)
# The datetime key is in the format '/Date(1493924400000)/'; extract
# the timestamp 1493924400 (cutting out the last three zeros as well).
date_time = datetime.fromtimestamp(
int(mix_percent["datetime"][6:-5]), tz=timezone.utc
)

# validate
# Ensure the fractions are within bounds.
valid = True
for gen_type, value in percent_mix.items():
percent_bounds = valid_percent[gen_type]
if not (percent_bounds[0] <= value <= percent_bounds[1]):
# skip this datapoint in the loop
for mode, fraction in mix_fraction.items():
lower, upper = FRACTION_LIMITS[mode]
if not (lower <= fraction <= upper):
valid = False
logger.warning(
"discarding datapoint at {dt} due to {fuel} percentage "
"out of bounds: {value}".format(
dt=data_date, fuel=gen_type, value=value
),
extra={"key": zone_key},
f"discarding datapoint at {date_time} because {mode} "
f"fraction is out of bounds: {fraction}",
extra={"key": ZONE_KEY},
)
if not valid:
# continue the outer loop, not the inner
continue

# in mix_data, the values are expressed as percentages,
# and have to be multiplied by load to find the actual MW value.
# Find the base load that corresponds with this mix so we can convert
# the mix values to megawatts.
corresponding_load = [
load_period
for load_period in load_data
if load_period["datetime"] == mix["datetime"]
base_load
for base_load in base_loads
if base_load["datetime"] == mix_percent["datetime"]
]
if corresponding_load:
load = corresponding_load[0]["Base Load"]
else:
# if not found, assume 1244 MW, based on average yearly electricity available for use
# in 2014 and 2015 (Statistics Canada table Table 127-0008 for Nova Scotia)
# If not found, assume 1244 MW, based on average yearly electricity
# available for use in 2014 and 2015 (Statistics Canada table
# 127-0008 for Nova Scotia).
load = 1244
logger.warning(
f"unable to find load for {data_date}, assuming 1244 MW",
extra={"key": zone_key},
f"unable to find load for {date_time}, assuming 1244 MW",
extra={"key": ZONE_KEY},
)

electricity_mix = {
gen_type: percent_value * load
for gen_type, percent_value in percent_mix.items()
# Convert the mix fractions to megawatts.
mix_megawatt = {
mode: fraction * load for mode, fraction in mix_fraction.items()
}

# validate again
# Ensure the power values (MW) are within bounds.
valid = True
for gen_type, value in electricity_mix.items():
absolute_bound = valid_absolute.get(
gen_type
) # imports are not in valid_absolute
if absolute_bound and value > absolute_bound:
for mode, power in mix_megawatt.items():
limit = MEGAWATT_LIMITS.get(mode) # Imports are excluded.
if limit and limit < power:
valid = False
logger.warning(
"discarding datapoint at {dt} due to {fuel} "
"too high: {value} MW".format(
dt=data_date, fuel=gen_type, value=value
),
extra={"key": zone_key},
f"discarding datapoint at {date_time} because {mode} "
f"is too high: {power} MW",
extra={"key": ZONE_KEY},
)
if not valid:
# continue the outer loop, not the inner
continue

production.append(
{
"zoneKey": zone_key,
"datetime": data_date,
"production": {
key: value
for key, value in electricity_mix.items()
if key != "imports"
},
"source": "nspower.ca",
}
# In this source, imports are positive. In the expected result for
# CA-NB->CA-NS, "net" represents a flow from NB to NS, i.e., an import
# to NS, so the value can be used directly. Note that this API only
# specifies imports; when NS is exporting energy, the API returns 0.
exchanges.append(
datetime=date_time,
netFlow=mix_megawatt["imports"],
source=SOURCE,
zoneKey="CA-NB->CA-NS",
)

# In this source, imports are positive. In the expected result for CA-NB->CA-NS,
# "net" represents a flow from NB to NS, that is, an import to NS.
# So the value can be used directly.
# Note that this API only specifies imports. When NS is exporting energy, the API returns 0.
imports.append(
{
"datetime": data_date,
"netFlow": electricity_mix["imports"],
"sortedZoneKeys": "CA-NB->CA-NS",
"source": "nspower.ca",
}
production_mix = ProductionMix()
for mode, power in mix_megawatt.items():
if mode != "imports":
production_mix.add_value(mode, power)
production_breakdowns.append(
datetime=date_time,
production=production_mix,
source=SOURCE,
zoneKey=ZONE_KEY,
)

return production, imports
return exchanges, production_breakdowns


def fetch_production(
zone_key: str = "CA-NS",
zone_key: ZoneKey = ZONE_KEY,
session: Session | None = None,
target_datetime: datetime | None = None,
logger: Logger = getLogger(__name__),
) -> list[dict]:
) -> list[dict[str, Any]]:
"""Requests the last known production mix (in MW) of a given country."""
if target_datetime:
raise NotImplementedError(
"This parser is unable to give information more than 24 hours in the past"
)
raise ParserException(PARSER, "Unable to fetch historical data", zone_key)

r = session or Session()
if zone_key != ZONE_KEY:
raise ParserException(PARSER, f"Cannot parse zone '{zone_key}'", zone_key)

production, imports = _get_ns_info(r, logger)

return production
_, production_breakdowns = _get_ns_info(session or Session(), logger)
return production_breakdowns.to_list()


def fetch_exchange(
zone_key1: str,
zone_key2: str,
zone_key1: ZoneKey,
zone_key2: ZoneKey,
session: Session | None = None,
target_datetime: datetime | None = None,
logger: Logger = getLogger(__name__),
) -> list[dict]:
) -> list[dict[str, Any]]:
"""
Requests the last known power exchange (in MW) between two regions.
Note: As of early 2017, Nova Scotia only has an exchange with New Brunswick (CA-NB).
(An exchange with Newfoundland, "Maritime Link", is scheduled to open in "late 2017").
Note: As of early 2017, Nova Scotia only has an exchange with New Brunswick
(CA-NB). (An exchange with Newfoundland, "Maritime Link", is scheduled to
open in "late 2017").
The API for Nova Scotia only specifies imports.
When NS is exporting energy, the API returns 0.
The API for Nova Scotia only specifies imports. When NS is exporting
energy, the API returns 0.
"""
if target_datetime:
raise NotImplementedError(
"This parser is unable to give information more than 24 hours in the past"
)

sorted_zone_keys = "->".join(sorted([zone_key1, zone_key2]))
raise ParserException(PARSER, "Unable to fetch historical data", ZONE_KEY)

sorted_zone_keys = "->".join(sorted((zone_key1, zone_key2)))
if sorted_zone_keys != "CA-NB->CA-NS":
raise NotImplementedError("This exchange pair is not implemented")
raise ParserException(PARSER, "Unimplemented exchange pair", sorted_zone_keys)

requests_obj = session or Session()
_, imports = _get_ns_info(requests_obj, logger)

return imports
exchanges, _ = _get_ns_info(session or Session(), logger)
return exchanges.to_list()


if __name__ == "__main__":
"""Main method, never used by the Electricity Map backend, but handy for testing."""

"""
Main method, never used by the Electricity Map backend, but handy for
testing.
"""
from pprint import pprint

test_logger = getLogger()

print("fetch_production() ->")
pprint(fetch_production(logger=test_logger))

pprint(fetch_production())
print('fetch_exchange("CA-NS", "CA-NB") ->')
pprint(fetch_exchange("CA-NS", "CA-NB", logger=test_logger))
pprint(fetch_exchange("CA-NS", "CA-NB"))

0 comments on commit a0d61d6

Please sign in to comment.