From 6297383a1081d471f519813ca17e01e2dc52ff00 Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Fri, 30 Sep 2016 06:23:06 +0200 Subject: [PATCH 1/8] Initial version of "haveibeenpwned" sensor component --- .../components/sensor/haveibeenpwned.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 homeassistant/components/sensor/haveibeenpwned.py diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py new file mode 100644 index 00000000000000..48fa525245d906 --- /dev/null +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -0,0 +1,149 @@ +""" +Support for haveibeenpwned (email breaches) sensor. + +For more details about this platform, please refer to the documentation at +https://home-assistant.io/components/sensor.haveibeenpwned/ +""" +from datetime import timedelta +import logging + +import voluptuous as vol +import requests + +from homeassistant.components.sensor import PLATFORM_SCHEMA +from homeassistant.const import (STATE_UNKNOWN, CONF_EMAIL) +from homeassistant.helpers.entity import Entity +import homeassistant.helpers.config_validation as cv +from homeassistant.util import Throttle +import homeassistant.util.dt as dt_util + +_LOGGER = logging.getLogger(__name__) + +DATE_STR_FORMAT = "%Y-%m-%d %H:%M:%S" +USER_AGENT = "Home Assistant HaveIBeenPwned Sensor Component" + +MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=4) +SCAN_INTERVAL = 5 + +PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ + vol.Required(CONF_EMAIL): vol.All(cv.ensure_list, [cv.string]), +}) + +# pylint: disable=unused-argument +def setup_platform(hass, config, add_devices, discovery_info=None): + """Setup the RESTful sensor.""" + emails = config.get(CONF_EMAIL) + + data = HaveIBeenPwnedData(emails) + data.update = Throttle(MIN_TIME_BETWEEN_UPDATES)(data.update) + + dev = [] + for email in emails: + dev.append(HaveIBeenPwnedSensor(data, email)) + + add_devices(dev) + + +class HaveIBeenPwnedSensor(Entity): + """Implementation of HaveIBeenPwnedSensor.""" + + def __init__(self, data, email): + """Initialize the HaveIBeenPwnedSensor sensor.""" + self._value = None + self._data = data + self._email = email + self._unit_of_measurement = "Hits" + self.update() + + @property + def name(self): + """Return the name of the sensor.""" + return "Breaches {}".format(self._email) + + @property + def unit_of_measurement(self): + """Return the unit the value is expressed in.""" + return self._unit_of_measurement + + @property + def state(self): + """Return the state of the device.""" + if self._value is None: + return STATE_UNKNOWN + else: + return len(self._value) + + @property + def state_attributes(self): + """Return the atrributes of the sensor.""" + val = {} + if self._value is None: + return val + + for idx, value in enumerate(self._value): + tmpname = "breach {}".format(idx+1) + tmpvalue = "{} {}".format( + value["Title"], + dt_util.as_local(dt_util.parse_datetime( + value["AddedDate"])).strftime(DATE_STR_FORMAT)) + val[tmpname] = tmpvalue + + return val + + def update(self): + """Get data for (next) email and set value if it's our email.""" + self._data.update() + if self._data.email == self._email: + self._value = self._data.data + +# pylint: disable=too-few-public-methods +class HaveIBeenPwnedData(object): + """Class for handling the data retrieval.""" + + def __init__(self, emails): + """Initialize the data object.""" + self._email_count = len(emails) + self._current_index = -1 + self.data = None + self.email = None + self._current_url = None + self._emails = emails + + def update(self): + """Get the latest data for current email from REST service.""" + try: + self.data = None + self._current_index = (self._current_index + 1) % self._email_count + self.email = self._emails[self._current_index] + url = "https://haveibeenpwned.com/api/v2/breachedaccount/{}". \ + format(self.email) + + _LOGGER.error("Checking for breaches for email %s", self.email) + + req = requests.get(url, headers={"User-agent": USER_AGENT}, + verify=False, allow_redirects=True, timeout=5) + + # Intial data for all email addresses have been gathered + # Throttle the amount of requests made to 1 per 15 minutes to + # prevent abuse and because the data will almost never change. + # This means the more email addresses that are specified the + # longer it will take to update them all, this is part of + # abuse protection. I emailed the owner of the api to see + # if he was ok with this abuse protection scheme and it + # was fine for him like this + if self._current_index == self._email_count - 1: + self.update = Throttle(timedelta(minutes=14))(self.update) + + except requests.exceptions.RequestException as exception: + _LOGGER.error(exception) + return + + if req.status_code == 200: + self.data = sorted(req.json(), key=lambda k: k["AddedDate"], + reverse=True) + elif req.status_code == 404: + self.data = [] + else: + _LOGGER.error("failed fetching HaveIBeenPwned Data for '%s'" + "(HTTP Status_code = %d)", self.email, + req.status_code) From 5dea22e4b8f28c04b980729afe99399389f2aac5 Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Fri, 30 Sep 2016 06:25:08 +0200 Subject: [PATCH 2/8] 2 flake8 fixes --- homeassistant/components/sensor/haveibeenpwned.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index 48fa525245d906..5de169c41d9d84 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -29,6 +29,7 @@ vol.Required(CONF_EMAIL): vol.All(cv.ensure_list, [cv.string]), }) + # pylint: disable=unused-argument def setup_platform(hass, config, add_devices, discovery_info=None): """Setup the RESTful sensor.""" @@ -96,6 +97,7 @@ def update(self): if self._data.email == self._email: self._value = self._data.data + # pylint: disable=too-few-public-methods class HaveIBeenPwnedData(object): """Class for handling the data retrieval.""" From b257badb700a56168db85209364b3f7481f1067e Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Fri, 30 Sep 2016 15:56:14 +0200 Subject: [PATCH 3/8] remove debugging error message --- homeassistant/components/sensor/haveibeenpwned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index 5de169c41d9d84..26ef4d64c862dd 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -120,7 +120,7 @@ def update(self): url = "https://haveibeenpwned.com/api/v2/breachedaccount/{}". \ format(self.email) - _LOGGER.error("Checking for breaches for email %s", self.email) + _LOGGER.info("Checking for breaches for email %s", self.email) req = requests.get(url, headers={"User-agent": USER_AGENT}, verify=False, allow_redirects=True, timeout=5) From 9fb47cfbe31e33890306f0628c2451f0ecb2ff82 Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Fri, 30 Sep 2016 19:03:38 +0200 Subject: [PATCH 4/8] Increase scan_interval as well as throttle to make sure that during initial startup of hass the request happens with 5 seconds delays and after startup with 15 minutes delays. Scan_interval is increased also to not call update as often --- homeassistant/components/sensor/haveibeenpwned.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index 26ef4d64c862dd..0627d8008f7ce2 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -22,7 +22,6 @@ DATE_STR_FORMAT = "%Y-%m-%d %H:%M:%S" USER_AGENT = "Home Assistant HaveIBeenPwned Sensor Component" -MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=4) SCAN_INTERVAL = 5 PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ @@ -36,7 +35,7 @@ def setup_platform(hass, config, add_devices, discovery_info=None): emails = config.get(CONF_EMAIL) data = HaveIBeenPwnedData(emails) - data.update = Throttle(MIN_TIME_BETWEEN_UPDATES)(data.update) + data.update = Throttle(timedelta(seconds=SCAN_INTERVAL-1))(data.update) dev = [] for email in emails: @@ -108,7 +107,6 @@ def __init__(self, emails): self._current_index = -1 self.data = None self.email = None - self._current_url = None self._emails = emails def update(self): @@ -134,7 +132,10 @@ def update(self): # if he was ok with this abuse protection scheme and it # was fine for him like this if self._current_index == self._email_count - 1: - self.update = Throttle(timedelta(minutes=14))(self.update) + global SCAN_INTERVAL + SCAN_INTERVAL = 60*15 + self.update = Throttle(timedelta( + seconds=SCAN_INTERVAL-5))(self.update) except requests.exceptions.RequestException as exception: _LOGGER.error(exception) From 0677d4e9ea862f7619bf1357804ec730f8103e5e Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Fri, 30 Sep 2016 19:19:54 +0200 Subject: [PATCH 5/8] update .coveragerc --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 5cedb8c5c485c5..8d91e8642af8f5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -235,6 +235,7 @@ omit = homeassistant/components/sensor/google_travel_time.py homeassistant/components/sensor/gpsd.py homeassistant/components/sensor/gtfs.py + homeassistant/components/sensor/haveibeenpwned.py homeassistant/components/sensor/hp_ilo.py homeassistant/components/sensor/imap.py homeassistant/components/sensor/imap_email_content.py From a21df61103a332f07c68bdc94c0416ae4057d75a Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Mon, 3 Oct 2016 23:41:59 +0200 Subject: [PATCH 6/8] remove (ssl) verify=False --- homeassistant/components/sensor/haveibeenpwned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index 0627d8008f7ce2..44947c3ed53357 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -121,7 +121,7 @@ def update(self): _LOGGER.info("Checking for breaches for email %s", self.email) req = requests.get(url, headers={"User-agent": USER_AGENT}, - verify=False, allow_redirects=True, timeout=5) + allow_redirects=True, timeout=5) # Intial data for all email addresses have been gathered # Throttle the amount of requests made to 1 per 15 minutes to From 79331c47b15130b70b45bd0ae075c1385fc3e87b Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Wed, 5 Oct 2016 22:32:11 +0200 Subject: [PATCH 7/8] - use dict to keep the request values with email as key - use track_point_in_time system to make sure data updates initially at 5 seconds between each call until all sensor's email have a result in the dict. --- .../components/sensor/haveibeenpwned.py | 128 +++++++++++------- 1 file changed, 79 insertions(+), 49 deletions(-) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index 44947c3ed53357..af52face868246 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -16,13 +16,15 @@ import homeassistant.helpers.config_validation as cv from homeassistant.util import Throttle import homeassistant.util.dt as dt_util +from homeassistant.helpers.event import track_point_in_time _LOGGER = logging.getLogger(__name__) DATE_STR_FORMAT = "%Y-%m-%d %H:%M:%S" USER_AGENT = "Home Assistant HaveIBeenPwned Sensor Component" -SCAN_INTERVAL = 5 +MIN_TIME_BETWEEN_UPDATES = timedelta(minutes=15) +MIN_TIME_BETWEEN_FORCED_UPDATES = timedelta(seconds=5) PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ vol.Required(CONF_EMAIL): vol.All(cv.ensure_list, [cv.string]), @@ -32,28 +34,33 @@ # pylint: disable=unused-argument def setup_platform(hass, config, add_devices, discovery_info=None): """Setup the RESTful sensor.""" - emails = config.get(CONF_EMAIL) + emails = config.get(CONF_EMAIL) data = HaveIBeenPwnedData(emails) - data.update = Throttle(timedelta(seconds=SCAN_INTERVAL-1))(data.update) - dev = [] + devices = [] for email in emails: - dev.append(HaveIBeenPwnedSensor(data, email)) + devices.append(HaveIBeenPwnedSensor(data, hass, email)) - add_devices(dev) + add_devices(devices) + + # To make sure we get initial data for the sensors + # ignoring the normal throttle of 15 minutes but using + # an update throttle of 5 seconds + for sensor in devices: + sensor.update_nothrottle() class HaveIBeenPwnedSensor(Entity): """Implementation of HaveIBeenPwnedSensor.""" - def __init__(self, data, email): + def __init__(self, data, hass, email): """Initialize the HaveIBeenPwnedSensor sensor.""" - self._value = None + self._state = STATE_UNKNOWN self._data = data + self._hass = hass self._email = email - self._unit_of_measurement = "Hits" - self.update() + self._unit_of_measurement = "Breaches" @property def name(self): @@ -68,19 +75,16 @@ def unit_of_measurement(self): @property def state(self): """Return the state of the device.""" - if self._value is None: - return STATE_UNKNOWN - else: - return len(self._value) + return self._state @property def state_attributes(self): """Return the atrributes of the sensor.""" val = {} - if self._value is None: + if self._email not in self._data.data: return val - for idx, value in enumerate(self._value): + for idx, value in enumerate(self._data.data[self._email]): tmpname = "breach {}".format(idx+1) tmpvalue = "{} {}".format( value["Title"], @@ -90,63 +94,89 @@ def state_attributes(self): return val + def update_nothrottle(self, dummy=None): + """Update sensor without throttle.""" + self._data.update_no_throttle() + + # Schedule a forced update 5 seconds in the future if the + # update above returned no data for this sensors email. + # this is mainly to make sure that we don't + # get http error "too many requests" and to have initial + # data after hass startup once we have the data it will + # update as normal using update + if self._email not in self._data.data: + track_point_in_time(self._hass, + self.update_nothrottle, + dt_util.now() + + MIN_TIME_BETWEEN_FORCED_UPDATES) + return + + if self._email in self._data.data: + self._state = len(self._data.data[self._email]) + self.update_ha_state() + def update(self): - """Get data for (next) email and set value if it's our email.""" + """Update data and see if it contains data for our email.""" self._data.update() - if self._data.email == self._email: - self._value = self._data.data + + if self._email in self._data.data: + self._state = len(self._data.data[self._email]) -# pylint: disable=too-few-public-methods class HaveIBeenPwnedData(object): """Class for handling the data retrieval.""" def __init__(self, emails): """Initialize the data object.""" self._email_count = len(emails) - self._current_index = -1 - self.data = None - self.email = None + self._current_index = 0 + self.data = {} + self._email = emails[0] self._emails = emails - def update(self): + def set_next_email(self): + """Set the next email to be looked up.""" + self._current_index = (self._current_index + 1) % self._email_count + self._email = self._emails[self._current_index] + + def update_no_throttle(self): + """Get the data for a specific email.""" + self.update(no_throttle=True) + + @Throttle(MIN_TIME_BETWEEN_UPDATES, MIN_TIME_BETWEEN_FORCED_UPDATES) + def update(self, **kwargs): """Get the latest data for current email from REST service.""" try: - self.data = None - self._current_index = (self._current_index + 1) % self._email_count - self.email = self._emails[self._current_index] url = "https://haveibeenpwned.com/api/v2/breachedaccount/{}". \ - format(self.email) + format(self._email) - _LOGGER.info("Checking for breaches for email %s", self.email) + _LOGGER.info("Checking for breaches for email %s", self._email) req = requests.get(url, headers={"User-agent": USER_AGENT}, allow_redirects=True, timeout=5) - # Intial data for all email addresses have been gathered - # Throttle the amount of requests made to 1 per 15 minutes to - # prevent abuse and because the data will almost never change. - # This means the more email addresses that are specified the - # longer it will take to update them all, this is part of - # abuse protection. I emailed the owner of the api to see - # if he was ok with this abuse protection scheme and it - # was fine for him like this - if self._current_index == self._email_count - 1: - global SCAN_INTERVAL - SCAN_INTERVAL = 60*15 - self.update = Throttle(timedelta( - seconds=SCAN_INTERVAL-5))(self.update) - - except requests.exceptions.RequestException as exception: - _LOGGER.error(exception) + except requests.exceptions.RequestException: + _LOGGER.error("failed fetching HaveIBeenPwned Data for '%s'", + self._email) return if req.status_code == 200: - self.data = sorted(req.json(), key=lambda k: k["AddedDate"], - reverse=True) + self.data[self._email] = sorted(req.json(), + key=lambda k: k["AddedDate"], + reverse=True) + + # only goto next email if we had data so that + # the forced updates try this current email again + self.set_next_email() + elif req.status_code == 404: - self.data = [] + self.data[self._email] = [] + + # only goto next email if we had data so that + # the forced updates try this current email again + self.set_next_email() + else: _LOGGER.error("failed fetching HaveIBeenPwned Data for '%s'" - "(HTTP Status_code = %d)", self.email, + "(HTTP Status_code = %d)", self._email, req.status_code) From a4106b85105d32e873c3a0abf5c729fd7aabc47b Mon Sep 17 00:00:00 2001 From: Willems Davy Date: Wed, 5 Oct 2016 22:42:33 +0200 Subject: [PATCH 8/8] fix a pylint error that happend on the py35 tests --- homeassistant/components/sensor/haveibeenpwned.py | 1 - 1 file changed, 1 deletion(-) diff --git a/homeassistant/components/sensor/haveibeenpwned.py b/homeassistant/components/sensor/haveibeenpwned.py index af52face868246..f317ef1456574c 100644 --- a/homeassistant/components/sensor/haveibeenpwned.py +++ b/homeassistant/components/sensor/haveibeenpwned.py @@ -34,7 +34,6 @@ # pylint: disable=unused-argument def setup_platform(hass, config, add_devices, discovery_info=None): """Setup the RESTful sensor.""" - emails = config.get(CONF_EMAIL) data = HaveIBeenPwnedData(emails)