Initial Import of Historical Data

custom_components/wnsm/api/client.py

@@ -1,14 +1,17 @@
 """Contains the Smartmeter API Client."""
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta, date
 from urllib import parse
 
 import requests
 from lxml import html
 
 from . import constants as const
-from .errors import SmartmeterConnectionError
-from .errors import SmartmeterLoginError
+from .errors import (
+    SmartmeterConnectionError,
+    SmartmeterLoginError,
+    SmartmeterQueryError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +32,9 @@ def __init__(self, username, password):
         self._access_token = None
         self._refresh_token = None
         self._api_gateway_token = None
+        self._access_token_expiration = None
+        self._refresh_token_expiration = None
+        self._api_gateway_b2b_token = None
 
     def load_login_page(self):
         """
@@ -94,7 +100,10 @@ def load_tokens(self, code):
             raise SmartmeterConnectionError(
                 f"Could not obtain access token: {result.content}"
             )
-        return result.json()
+        tokens = result.json()
+        if tokens['token_type'] != 'Bearer':
+            raise SmartmeterLoginError(f'Bearer token required, but got {tokens["token_type"]!r}')
+        return tokens
 
     def login(self):
         """
@@ -103,31 +112,59 @@ def login(self):
         url = self.load_login_page()
         code = self.credentials_login(url)
         tokens = self.load_tokens(code)
+
         self._access_token = tokens["access_token"]
         # TODO: use this to refresh the token of this session instead of re-login. may be nicer for the API
         self._refresh_token = tokens["refresh_token"]
-        self._api_gateway_token = self._get_api_key(self._access_token)
+        now = datetime.now()
+        self._access_token_expiration = now + timedelta(seconds=tokens['expires_in'])
+        self._refresh_token_expiration = now + timedelta(seconds=tokens['refresh_expires_in'])
+
+        logger.debug("Access Token valid until %s" % self._access_token_expiration)
+
+        self._api_gateway_token, self._api_gateway_b2b_token = self._get_api_key(self._access_token)
+        return self
+
+    def _access_valid_or_raise(self):
+        """Checks if the access token is still valid or raises an exception"""
+        if datetime.now() >= self._access_token_expiration:
+            # TODO: If the refresh token is still valid, it could be refreshed here
+            raise SmartmeterConnectionError("Access Token is not valid anymore, please re-log!")
 
     def _get_api_key(self, token):
+        key_b2c = None
+        key_b2b = None
+
+        self._access_valid_or_raise()
+
         headers = {"Authorization": f"Bearer {token}"}
         try:
             result = self.session.get(const.PAGE_URL, headers=headers)
         except Exception as exception:
             raise SmartmeterConnectionError("Could not obtain API key") from exception
         tree = html.fromstring(result.content)
         scripts = tree.xpath("(//script/@src)")
+
+        # sort the scripts in some order to find the keys faster
+        # so far, the script was called main.XXXX.js
+        scripts = sorted(scripts, key=lambda x: 'main' not in x)
+
         for script in scripts:
+            if key_b2c is not None and key_b2b is not None:
+                break
             try:
                 response = self.session.get(const.PAGE_URL + script)
             except Exception as exception:
                 raise SmartmeterConnectionError(
                     "Could not obtain API Key from scripts"
                 ) from exception
-            for match in const.API_GATEWAY_TOKEN_REGEX.findall(response.text):
-                return match
-        raise SmartmeterConnectionError(
-            "Could not obtain API Key - no match"
-        )
+            key_b2c = const.API_GATEWAY_TOKEN_REGEX.search(response.text)
+            key_b2b = const.API_GATEWAY_B2B_TOKEN_REGEX.search(response.text)
+        if key_b2c is None or key_b2b is None:
+            raise SmartmeterConnectionError(
+                "Could not obtain API Key - no match"
+            )
+        return key_b2c.group(1), key_b2b.group(1)
 
     @staticmethod
     def _dt_string(datetime_string):
@@ -142,19 +179,32 @@ def _call_api(
         query=None,
         return_response=False,
         timeout=60.0,
+        extra_headers=None,
     ):
+        self._access_valid_or_raise()
+
         if base_url is None:
             base_url = const.API_URL
         url = f"{base_url}{endpoint}"
 
         if query:
             url += ("?" if "?" not in endpoint else "&") + parse.urlencode(query)
 
+        logger.debug("REQUEST: %s" % url)
+
         headers = {
             "Authorization": f"Bearer {self._access_token}",
-            "X-Gateway-APIKey": self._api_gateway_token,
         }
 
+        # For API calls to B2C or B2B, we need to add the Gateway-APIKey:
+        if base_url == const.API_URL:
+            headers['X-Gateway-APIKey'] = self._api_gateway_token
+        elif base_url == const.API_URL_B2B:
+            headers['X-Gateway-APIKey'] = self._api_gateway_b2b_token
+
+        if extra_headers:
+            headers.update(extra_headers)
+
         if data:
             headers["Content-Type"] = "application/json"
 
@@ -187,16 +237,19 @@ def meter_readings(self):
         return self._call_api("zaehlpunkt/meterReadings")
 
     def verbrauch_raw(
-        self, date_from: datetime, date_to: datetime = None, zaehlpunkt=None
+        self, date_from: datetime, date_to: datetime = None, zaehlpunkt: str | None = None
     ):
         """Returns energy usage.
 
+        This can be used to query the daily consumption for a long period of time,
+        for example several months or a week.
+
         Args:
             date_from (datetime): Start date for energy usage request
             date_to (datetime, optional): End date for energy usage request.
                 Defaults to datetime.now().
             zaehlpunkt (str, optional): Id for desired smartmeter.
-                If None check for first meter in user profile.
+                If None, check for first meter in user profile.
 
         Returns:
             dict: JSON response of api call to
@@ -214,50 +267,46 @@ def verbrauch_raw(
         }
         return self._call_api(endpoint, query=query)
 
-    def verbrauch(self, date_from: datetime, date_to: datetime = None, zaehlpunkt=None):
-        """Returns energy usage.
+    def verbrauch(self, date_from: datetime, zaehlpunkt: str | None = None, resolution: const.Resolution = const.Resolution.HOUR):
+        """Returns energy usage for 24h after date_to.
 
         Args:
             date_from (datetime.datetime): Starting date for energy usage request
-            date_to (datetime.datetime, optional): Ending date for energy usage request.
-                Defaults to datetime.datetime.now().
             zaehlpunkt (str, optional): Id for desired smartmeter.
-                If None check for first meter in user profile.
+                If None, check for first meter in user profile.
+            resolution (const.Resolution, optinal): Specify either 1h or 15min resolution
 
         Returns:
             dict: JSON response of api call to
                 'm/messdaten/zaehlpunkt/ZAEHLPUNKT/verbrauch'
         """
-        if date_to is None:
-            date_to = datetime.now()
         if zaehlpunkt is None:
             zaehlpunkt = self._get_first_zaehlpunkt()
         endpoint = f"messdaten/zaehlpunkt/{zaehlpunkt}/verbrauch"
         query = const.build_verbrauchs_args(
-            dateFrom=self._dt_string(date_from), dateTo=self._dt_string(date_to)
+            dateFrom=self._dt_string(date_from),
+            dayViewResolution=resolution.value,
         )
         return self._call_api(endpoint, query=query)
 
-    def tages_verbrauch(self, day: datetime, zaehlpunkt=None):
-        """Returns energy usage.
+    def tages_verbrauch(self, day: datetime, zaehlpunkt: str | None = None, resolution: const.Resolution = const.Resolution.QUARTER_HOUR):
+        """Returns energy usage for the current day.
+
 
         Args:
             day (datetime.datetime): Day date for the request
             zaehlpunkt (str, optional): Id for desired smartmeter.
                 If None, check for first meter in user profile.
+            resolution (const.Resolution, optinal): Specify either 1h or 15min resolution
 
         Returns:
             dict: JSON response of api call to
                 'messdaten/zaehlpunkt/ZAEHLPUNKT/verbrauch'
         """
-
-        if zaehlpunkt is None:
-            zaehlpunkt = self._get_first_zaehlpunkt()
-        endpoint = f"messdaten/zaehlpunkt/{zaehlpunkt}/verbrauch"
-        query = const.build_verbrauchs_args(
-            dateFrom=self._dt_string(day.replace(hour=0, minute=0, second=0))
-        )
-        return self._call_api(endpoint, query=query)
+        # FIXME: Actually, using 00:00:00.000 does not query the beginning of the day!
+        # The problem is, that the time has to specified in UTC and during standard time,
+        # UTC day starts at 23:00:00 and during summer time even on 22:00:00!
+        return self.verbrauch(day.replace(hour=0, minute=0, second=0, microsecond=0), zaehlpunkt, resolution)
 
     def profil(self):
         """Returns profile of a logged-in user.
@@ -298,7 +347,7 @@ def create_ereignis(self, zaehlpunkt, name, date_from, date_to=None):
 
         Args:
             zaehlpunkt (str): Id for desired smartmeter.
-                If None check for first meter in user profile
+                If None, check for first meter in user profile
             name (str): Event name
             date_from (datetime.datetime): (Starting) date for request
             date_to (datetime.datetime, optional): Ending date for request.
@@ -326,3 +375,53 @@ def create_ereignis(self, zaehlpunkt, name, date_from, date_to=None):
     def delete_ereignis(self, ereignis_id):
         """Deletes ereignis."""
         return self._call_api(f"user/ereignis/{ereignis_id}", method="DELETE")
+
+    def historical_data(
+        self,
+        zaehlpunkt: str = None,
+        date_from: date = None,
+        date_until: date = None,
+        valuetype: const.ValueType = const.ValueType.QUARTER_HOUR,
+    ):
+        """
+        Query historical data in a batch
+
+        If no arguments are given, a span of three year is queried (same day as today but from current year - 3).
+        If date_from is not given but date_until, again a three year span is assumed.
+        """
+        if zaehlpunkt is None:
+            zaehlpunkt = self._get_first_zaehlpunkt()
+
+        if date_until is None:
+            date_until = date.today()
+
+        if date_from is None:
+            date_from = date_until.replace(year=date_until.year - 3)
+
+        query = {
+            'zaehlpunkt': zaehlpunkt,
+            'datumVon': date_from.strftime('%Y-%m-%d'),
+            'datumBis': date_until.strftime('%Y-%m-%d'),
+            'wertetyp': valuetype.value,
+        }
+
+        extra = {
+            # For this API Call, requesting json is important!
+            "Accept": "application/json"
+        }
+
+        data = self._call_api('zaehlpunkte/messwerte', base_url=const.API_URL_B2B, query=query, extra_headers=extra)
+
+        # Some Sanity Checks...
+        if len(data) != 1 or data[0]['zaehlpunkt'] != zaehlpunkt or len(data[0]['zaehlwerke']) != 1:
+            # TODO: Is it possible to have multiple zaehlwerke in one zaehlpunkt?
+            # I guess so, otherwise it would not be a list...
+            # Probably (my guess), we would see this on the OBIS Code.
+            # The OBIS Code can code for channels, thus we would probably see that there.
+            # Keep that in mind if for someone this fails.
+            logger.debug(f"Returned data: {data}")
+            raise SmartmeterQueryError("Returned data does not match given zaehlpunkt!")
+        obis_code = data[0]['zaehlwerke'][0]['obisCode']
+        if obis_code[0] != '1':
+            logger.warning(f"The OBIS code of the meter ({obis_code}) reports that this meter does not count electrical energy!")
+        return data[0]['zaehlwerke'][0]

custom_components/wnsm/api/constants.py

@@ -2,13 +2,17 @@
     api constants
 """
 import re
+import enum
 
 MAIN_SCRIPT_REGEX = re.compile(r"^main\S+\.js$")
 API_GATEWAY_TOKEN_REGEX = re.compile(r'b2cApiKey\:\s*\"([A-Za-z0-9\-_]+)\"', re.IGNORECASE)
+API_GATEWAY_B2B_TOKEN_REGEX = re.compile(r'b2bApiKey\:\s*\"([A-Za-z0-9\-_]+)\"', re.IGNORECASE)
 
 PAGE_URL = "https://smartmeter-web.wienernetze.at/"
 API_URL_ALT = "https://service.wienernetze.at/sm/api/"
+# These two URLS are also coded in the js as b2cApiUrl and b2bApiUrl
 API_URL = "https://api.wstw.at/gateway/WN_SMART_METER_PORTAL_API_B2C/1.0/"
+API_URL_B2B = "https://api.wstw.at/gateway/WN_SMART_METER_PORTAL_API_B2B/1.0/"
 REDIRECT_URI = "https://smartmeter-web.wienernetze.at/"
 API_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
 AUTH_URL = "https://log.wien/auth/realms/logwien/protocol/openid-connect/"  # noqa
@@ -24,6 +28,19 @@
 }
 
 
+class Resolution(enum.Enum):
+    """Possible resolution for consumption data of one day"""
+    HOUR = "HOUR"  #: gets consumption data per hour
+    QUARTER_HOUR = "QUARTER-HOUR"  #: gets consumption data per 15min
+
+
+class ValueType(enum.Enum):
+    """Possible 'wertetyp' for querying historical data"""
+    METER_READ = "METER_READ"  #: Meter reading for the day
+    DAY = "DAY"  #: Consumption for the day
+    QUARTER_HOUR = "QUARTER_HOUR"  #: Consumption for 15min slots
+
+
 def build_access_token_args(**kwargs):
     """
     build access token and add kwargs
@@ -43,9 +60,9 @@ def build_verbrauchs_args(**kwargs):
     """
     args = {
         "period": "DAY",
-        "accumulate": False,
-        "offset": 0,
-        "dayViewResolution": "QUARTER-HOUR",
+        "accumulate": False,  # can be changed to True to get a cum-sum
+        "offset": 0,  # additional offset to start cum-sum with
+        "dayViewResolution": "HOUR",
     }
     args.update(**kwargs)
     return args

custom_components/wnsm/api/errors.py

@@ -25,3 +25,7 @@ class SmartmeterLoginError(SmartmeterError):
 
 class SmartmeterConnectionError(SmartmeterError):
     """Raised due to network connectivity-related issues."""
+
+
+class SmartmeterQueryError(SmartmeterError):
+    """Raised if query went not as expected."""