diff --git a/parsers/SG.py b/parsers/SG.py index a14f972fb8..771c699245 100644 --- a/parsers/SG.py +++ b/parsers/SG.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from collections import defaultdict +import logging +import re import arrow from PIL import Image from pytesseract import image_to_string -import re import requests TIMEZONE = 'Asia/Singapore' @@ -40,18 +41,10 @@ For Electricity Map, we map CCGT and GT to gas, and ST to "unknown". -There appears to be no real-time data for solar production. -Installed solar has been rising rapidly, but from a very low base. -Per Singapore Energy Statistics 2016 pg 96, total installed solar PV capacity at end of 2015 was 45.8 MWac. -Per https://www.ema.gov.sg/cmsmedia/Publications_and_Statistics/Statistics/47RSU.pdf -total installed solar capacity in 2017Q1 was 129.8 MWp / 99.9 MWac, so capacity doubled during 2016. -However, when producing at max capacity this would only be about 2% of summer mid-night demand of around 5 GW. -So for now this won't introduce a big inaccuracy. - -There exists an interconnection to Malaysia. Its capacity is apparently 2x 200 MW, -which is potentially 5-10% of normal use. I was unable to find data on its use -(not even historical, let along real-time). The Singapore Energy Statistics 2016 document -does not note any electricity exports or imports. +The Energy Market Authority estimates current solar production and publishes it at +https://www.ema.gov.sg/solarmap.aspx + +There exists an interconnection to Malaysia, it is implemented in MY_WM.py. """ TYPE_MAPPINGS = { @@ -61,35 +54,42 @@ } -def get_solar(session=None): +def get_solar(session, logger): """ Fetches a graphic showing estimated solar production data. Uses OCR (tesseract) to extract MW value. Returns a float or None. """ - s = session or requests.Session() url = 'https://www.ema.gov.sg/cmsmedia/irradiance/plot.png' - solar_image = Image.open(s.get(url, stream=True).raw) + solar_image = Image.open(session.get(url, stream=True).raw) gray = solar_image.convert('L') - threshold_filter = lambda x: 0 if x<77 else 255 + threshold_filter = lambda x: 0 if x < 77 else 255 black_white = gray.point(threshold_filter, '1') text = image_to_string(black_white, lang='eng') - pattern = r'Est. PV Output: (.*)MWac' - val = re.search(pattern, text, re.MULTILINE).group(1) + try: + pattern = r'Est. PV Output: (.*)MWac' + val = re.search(pattern, text, re.MULTILINE).group(1) + + time_pattern = r'\d+-\d+-\d+\s+\d+:\d+' + time_string = re.search(time_pattern, text, re.MULTILINE).group(0) + except AttributeError: + msg = 'Unable to get values for SG solar from OCR text: {}'.format(text) + logger.warning(msg, extra={'key': 'SG'}) + return None - time_pattern = r'\d+-\d+-\d+\s+\d+:\d+' - time_string = re.search(time_pattern, text, re.MULTILINE).group(0) solar_dt = arrow.get(time_string).replace(tzinfo='Asia/Singapore') singapore_dt = arrow.now('Asia/Singapore') diff = singapore_dt - solar_dt # Need to be sure we don't get old data if image stops updating. if diff.seconds > 3600: - print('Singapore solar data is too old to use.') + msg = ('Singapore solar data is too old to use, ' + 'parsed data timestamp was {}.').format(solar_dt) + logger.warning(msg, extra={'key': 'SG'}) return None # At night format changes from 0.00 to 0 @@ -97,16 +97,18 @@ def get_solar(session=None): # This try/except will make sure no invalid data is returned. try: solar = float(val) - except ValueError as err: + except ValueError: if len(val) == 1 and 'O' in val: solar = 0.0 else: - print("Singapore solar data is unreadable - got {}.".format(val)) - solar = None + msg = "Singapore solar data is unreadable - got {}.".format(val) + logger.warning(msg, extra={'key': 'SG'}) + return None else: if solar > 200.0: - print("Singapore solar generation is way over capacity - got {}".format(val)) - solar = None + msg = "Solar generation is way over capacity - got {}".format(val) + logger.warning(msg, extra={'key': 'SG'}) + return None return solar @@ -159,7 +161,8 @@ def sg_data_to_datetime(data): return data_datetime -def fetch_production(zone_key='SG', session=None, target_datetime=None, logger=None): +def fetch_production(zone_key='SG', session=None, target_datetime=None, + logger=logging.getLogger(__name__)): """Requests the last known production mix (in MW) of Singapore. Arguments: @@ -202,12 +205,12 @@ def fetch_production(zone_key='SG', session=None, target_datetime=None, logger=N else: # unrecognized - log it, then add into unknown - print( - 'Singapore has unrecognized generation type "{}" with production share {}%'.format( - gen_type, gen_percent)) + msg = ('Singapore has unrecognized generation type "{}" ' + 'with production share {}%').format(gen_type, gen_percent) + logger.warning(msg) generation_by_type['unknown'] += gen_mw - generation_by_type['solar'] = get_solar(session=None) + generation_by_type['solar'] = get_solar(requests_obj, logger) # some generation methods that are not used in Singapore generation_by_type.update({ @@ -221,7 +224,7 @@ def fetch_production(zone_key='SG', session=None, target_datetime=None, logger=N 'zoneKey': zone_key, 'production': generation_by_type, 'storage': {}, # there is no known electricity storage in Singapore - 'source': 'emcsg.com' + 'source': 'emcsg.com, ema.gov.sg' }