Defer module loading to improve startup speed and memory use

The RequestRegistry (ex. ApiEndpoints) class now provides their
members lazily. While it's still not optimal, at least access to its
details is now encapsulated using appropriate methods (discover,
resolve, get_provider_names, get_network_names), to make subsequent
refactoring easier.

Other than this, a few other modules will now only be loaded at runtime
when they are actually needed.

wetterdienst/api.py

@@ -1,56 +1,123 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) 2018-2021, earthobservations developers.
 # Distributed under the MIT License. See LICENSE for more info.
-from enum import Enum
-
 from wetterdienst.exceptions import InvalidEnumeration, ProviderError
 from wetterdienst.metadata.provider import Provider
-from wetterdienst.provider.dwd.mosmix import DwdMosmixRequest
-from wetterdienst.provider.dwd.observation import DwdObservationRequest
-from wetterdienst.provider.dwd.radar import DwdRadarValues
-from wetterdienst.provider.eaufrance.hubeau import HubeauRequest
-from wetterdienst.provider.eccc.observation import EcccObservationRequest
-from wetterdienst.provider.environment_agency.hydrology.api import EaHydrologyRequest
-from wetterdienst.provider.geosphere.observation import GeosphereObservationRequest
-from wetterdienst.provider.noaa.ghcn.api import NoaaGhcnRequest
-from wetterdienst.provider.nws.observation.api import NwsObservationRequest
-from wetterdienst.provider.wsv.pegel.api import WsvPegelRequest
 from wetterdienst.util.enumeration import parse_enumeration_from_template
 from wetterdienst.util.parameter import DatasetTreeCore
 
 
-class ApiEndpoints(DatasetTreeCore):
-    class DWD(Enum):
-        OBSERVATION = DwdObservationRequest  # generic name
-        MOSMIX = DwdMosmixRequest
-        RADAR = DwdRadarValues
+class RequestRegistry(DatasetTreeCore):
+    class DWD(DatasetTreeCore):
+        @staticmethod
+        @property
+        def OBSERVATION():
+            from wetterdienst.provider.dwd.observation import DwdObservationRequest
+
+            return DwdObservationRequest
+
+        @staticmethod
+        @property
+        def MOSMIX():
+            from wetterdienst.provider.dwd.mosmix import DwdMosmixRequest
+
+            return DwdMosmixRequest
+
+        @staticmethod
+        @property
+        def RADAR():
+            from wetterdienst.provider.dwd.radar import DwdRadarValues
+
+            return DwdRadarValues
+
+    class ECCC(DatasetTreeCore):
+        @staticmethod
+        @property
+        def OBSERVATION():
+            from wetterdienst.provider.eccc.observation import EcccObservationRequest
 
-    class ECCC(Enum):
-        OBSERVATION = EcccObservationRequest  # generic name
+            return EcccObservationRequest
 
-    class NOAA(Enum):
-        GHCN = NoaaGhcnRequest
+    class NOAA(DatasetTreeCore):
+        @staticmethod
+        @property
+        def GHCN():
+            from wetterdienst.provider.noaa.ghcn import NoaaGhcnRequest
 
-    class WSV(Enum):
-        PEGEL = WsvPegelRequest
+            return NoaaGhcnRequest
 
-    class EA(Enum):
-        HYDROLOGY = EaHydrologyRequest
+    class WSV(DatasetTreeCore):
+        @staticmethod
+        @property
+        def PEGEL():
+            from wetterdienst.provider.wsv.pegel import WsvPegelRequest
 
-    class NWS(Enum):
-        OBSERVATION = NwsObservationRequest
+            return WsvPegelRequest
 
-    class EAUFRANCE(Enum):
-        HUBEAU = HubeauRequest
+    class EA(DatasetTreeCore):
+        @staticmethod
+        @property
+        def HYDROLOGY():
+            from wetterdienst.provider.environment_agency.hydrology import (
+                EaHydrologyRequest,
+            )
 
-    class GEOSPHERE(Enum):
-        OBSERVATION = GeosphereObservationRequest
+            return EaHydrologyRequest
+
+    class NWS(DatasetTreeCore):
+        @staticmethod
+        @property
+        def OBSERVATION():
+            from wetterdienst.provider.nws.observation import NwsObservationRequest
+
+            return NwsObservationRequest
+
+    class EAUFRANCE(DatasetTreeCore):
+        @staticmethod
+        @property
+        def HUBEAU():
+            from wetterdienst.provider.eaufrance.hubeau import HubeauRequest
+
+            return HubeauRequest
+
+    class GEOSPHERE(DatasetTreeCore):
+        @staticmethod
+        @property
+        def OBSERVATION():
+            from wetterdienst.provider.geosphere.observation import (
+                GeosphereObservationRequest,
+            )
+
+            return GeosphereObservationRequest
+
+    @classmethod
+    def discover(cls):
+        api_endpoints = {}
+        for provider in cls:
+            api_endpoints[provider.__name__] = [network.fget.__name__ for network in cls[provider.__name__]]
+        return api_endpoints
+
+    @classmethod
+    def resolve(cls, provider: str, network: str):
+        try:
+            # `.fget()` is needed to access the <property> instance.
+            return cls[provider][network.upper()].fget()
+        except AttributeError as ex:
+            raise KeyError(ex)
+
+    @classmethod
+    def get_provider_names(cls):
+        return [provider.__name__ for provider in cls]
+
+    @classmethod
+    def get_network_names(cls, provider):
+        return [network.fget.__name__ for network in cls[provider]]
 
 
 class Wetterdienst:
     """Wetterdienst top-level API with links to the different available APIs"""
 
-    endpoints = ApiEndpoints
+    endpoints = RequestRegistry
 
     def __new__(cls, provider: str, network: str):
         """
@@ -62,7 +129,7 @@ def __new__(cls, provider: str, network: str):
         try:
             provider_ = parse_enumeration_from_template(provider, Provider)
 
-            api = cls.endpoints[provider_.name][network.upper()].value
+            api = cls.endpoints.resolve(provider_.name, network)
 
             if not api:
                 raise KeyError
@@ -75,8 +142,4 @@ def __new__(cls, provider: str, network: str):
     @classmethod
     def discover(cls) -> dict:
         """Display available API endpoints"""
-        api_endpoints = {}
-        for provider in cls.endpoints:
-            api_endpoints[provider.__name__] = [network.name for network in cls.endpoints[provider.__name__]]
-
-        return api_endpoints
+        return cls.endpoints.discover()

wetterdienst/core/scalar/request.py

@@ -37,7 +37,6 @@
 from wetterdienst.metadata.resolution import Frequency, Resolution, ResolutionType
 from wetterdienst.settings import Settings
 from wetterdienst.util.enumeration import parse_enumeration_from_template
-from wetterdienst.util.geo import Coordinates, derive_nearest_neighbours
 
 log = logging.getLogger(__name__)
 
@@ -702,6 +701,8 @@ def filter_by_rank(
         :param rank: number of stations_result to be returned, greater 0
         :return: pandas.DataFrame with station information for the selected stations_result
         """
+        from wetterdienst.util.geo import Coordinates, derive_nearest_neighbours
+
         rank = int(rank)
 
         if rank <= 0:

wetterdienst/ui/cli.py

@@ -16,8 +16,6 @@
 
 from wetterdienst import Provider, Wetterdienst, __appname__, __version__
 from wetterdienst.exceptions import ProviderError
-from wetterdienst.provider.dwd.radar.api import DwdRadarSites
-from wetterdienst.provider.eumetnet.opera.sites import OperaRadarSites
 from wetterdienst.ui.core import (
     get_interpolate,
     get_stations,
@@ -879,6 +877,9 @@ def radar(
     wmo_code: str,
     country_name: str,
 ):
+    from wetterdienst.provider.dwd.radar.api import DwdRadarSites
+    from wetterdienst.provider.eumetnet.opera.sites import OperaRadarSites
+
     if dwd:
         data = DwdRadarSites().all()
     else:

wetterdienst/ui/core.py

@@ -12,7 +12,6 @@
 from wetterdienst.metadata.datarange import DataRange
 from wetterdienst.metadata.period import PeriodType
 from wetterdienst.metadata.resolution import Resolution, ResolutionType
-from wetterdienst.provider.dwd.mosmix import DwdMosmixRequest, DwdMosmixType
 from wetterdienst.settings import Settings
 from wetterdienst.util.enumeration import parse_enumeration_from_template
 
@@ -64,6 +63,8 @@ def _get_stations_request(
     dropna: bool,
     use_nearby_station_until_km: float,
 ):
+    from wetterdienst.provider.dwd.mosmix import DwdMosmixRequest, DwdMosmixType
+
     # TODO: move this into Request core
     start_date, end_date = None, None
     if date:

wetterdienst/ui/explorer/app.py

@@ -17,7 +17,7 @@
 from dash import Input, Output, State, dcc, html
 from geojson import Feature, FeatureCollection, Point
 
-from wetterdienst.api import ApiEndpoints
+from wetterdienst.api import RequestRegistry
 from wetterdienst.core.scalar.result import ValuesResult
 from wetterdienst.exceptions import InvalidParameterCombination
 from wetterdienst.metadata.columns import Columns
@@ -77,7 +77,7 @@ def fetch_stations(provider: str, network: str, resolution: str, dataset: str, p
     if not (provider and network and resolution and dataset and parameter and period):
         return empty_frame
 
-    api = ApiEndpoints[provider][network].value
+    api = RequestRegistry.resolve(provider, network)
 
     if period == "ALL":
         period = [*api._period_base]
@@ -149,7 +149,7 @@ def fetch_values(
         log.warning("Querying without station_id is rejected")
         return empty_frame
 
-    api = ApiEndpoints[provider][network].value
+    api = RequestRegistry.resolve(provider, network)
 
     if period == "ALL":
         period = [*api._period_base]
@@ -437,7 +437,7 @@ def set_network_options(provider):
     if not provider:
         return []
 
-    return [{"label": network.name, "value": network.name} for network in ApiEndpoints[provider]]
+    return [{"label": network, "value": network} for network in RequestRegistry.get_network_names(provider)]
 
 
 @app.callback(
@@ -452,9 +452,9 @@ def set_resolution_options(provider, network):
     if not (provider and network):
         return []
 
-    api = ApiEndpoints[provider][network].value
+    api = RequestRegistry.resolve(provider, network)
 
-    if api == DwdMosmixRequest:
+    if isinstance(api, DwdMosmixRequest):
         return [{"label": resolution.name, "value": resolution.name} for resolution in DwdMosmixType]
     else:
         return [{"label": resolution.name, "value": resolution.name} for resolution in api._resolution_base]
@@ -473,7 +473,7 @@ def set_dataset_options(provider, network, resolution):
     if not (provider and network and resolution):
         return []
 
-    api = ApiEndpoints[provider][network].value
+    api = RequestRegistry.resolve(provider, network)
 
     if api._has_datasets and not api._unique_dataset:
         # first dataset is placeholder for unique dataset with parameters combined from all datasets
@@ -505,7 +505,7 @@ def set_parameter_options(provider, network, resolution, dataset):
     if not (provider and network and resolution and dataset):
         return [], []
 
-    api = ApiEndpoints[provider][network].value
+    api = RequestRegistry.resolve(provider, network)
 
     if api._has_datasets and not api._unique_dataset:
         if dataset == resolution:

wetterdienst/ui/explorer/layout/observations_germany.py

@@ -4,11 +4,11 @@
 import dash_leaflet as dl
 from dash import dcc, html
 
-from wetterdienst.api import ApiEndpoints
+from wetterdienst.api import RequestRegistry
 
 
 def get_providers():
-    return [{"label": provider.__name__, "value": provider.__name__} for provider in ApiEndpoints]
+    return [{"label": provider, "value": provider} for provider in RequestRegistry.get_provider_names()]
 
 
 def dashboard_layout() -> html:

wetterdienst/util/parameter.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) 2018-2021, earthobservations developers.
 # Distributed under the MIT License. See LICENSE for more info.
+import types
+
+
 class _GetAttrMeta(type):
     # https://stackoverflow.com/questions/33727217/subscriptable-objects-in-class
     def __getitem__(cls, x):
@@ -9,8 +12,9 @@ def __getitem__(cls, x):
     def __iter__(cls):
         """Getting subclasses which usually represent resolutions"""
         for attr in vars(cls):
-            if not attr.startswith("_"):
-                yield cls[attr]
+            slot = cls[attr]
+            if not attr.startswith("_") and not isinstance(slot, types.MethodType):
+                yield slot
 
 
 class DatasetTreeCore(metaclass=_GetAttrMeta):