FDSNWS: implement EIDA token authentication and add authentication to routed Clients

obspy/clients/fdsn/client.py

@@ -30,6 +30,7 @@
 
 import obspy
 from obspy import UTCDateTime, read_inventory
+from obspy.core.compatibility import urlparse
 from .header import (DEFAULT_PARAMETERS, DEFAULT_USER_AGENT, FDSNWS,
                      OPTIONAL_PARAMETERS, PARAMETER_ALIASES, URL_MAPPINGS,
                      WADL_PARAMETERS_NOT_TO_BE_PARSED, FDSNException,
@@ -138,7 +139,8 @@ def _validate_base_url(cls, base_url):
 
     def __init__(self, base_url="IRIS", major_versions=None, user=None,
                  password=None, user_agent=DEFAULT_USER_AGENT, debug=False,
-                 timeout=120, service_mappings=None, force_redirect=False):
+                 timeout=120, service_mappings=None, force_redirect=False,
+                 eida_token=None):
         """
         Initializes an FDSN Web Service client.
 
@@ -193,10 +195,18 @@ def __init__(self, base_url="IRIS", major_versions=None, user=None,
             when a redirect is discovered. This is done to improve security.
             Settings this flag to ``True`` will force all redirects to be
             followed even if credentials are given.
+        :type eida_token: str
+        :param eida_token: Token for EIDA authentication mechanism, see
+            http://geofon.gfz-potsdam.de/waveform/archive/auth/index.php. If a
+            token is provided, options ``user`` and ``password`` must be given.
+            This mechanism is only available on select EIDA nodes. The token
+            can be provided in form of the PGP message as a string, or the
+            filename of a local file with the PGP message in it.
         """
         self.debug = debug
         self.user = user
         self.timeout = timeout
+        self._force_redirect = force_redirect
 
         # Cache for the webservice versions. This makes interactive use of
         # the client more convenient.
@@ -220,23 +230,7 @@ def __init__(self, base_url="IRIS", major_versions=None, user=None,
 
         self.base_url = base_url
 
-        # Only add the authentication handler if required.
-        handlers = []
-        if user is not None and password is not None:
-            # Create an OpenerDirector for HTTP Digest Authentication
-            password_mgr = urllib_request.HTTPPasswordMgrWithDefaultRealm()
-            password_mgr.add_password(None, base_url, user, password)
-            handlers.append(urllib_request.HTTPDigestAuthHandler(password_mgr))
-
-        if (user is None and password is None) or force_redirect is True:
-            # Redirect if no credentials are given or the force_redirect
-            # flag is True.
-            handlers.append(CustomRedirectHandler())
-        else:
-            handlers.append(NoRedirectionHandler())
-
-        # Don't install globally to not mess with other codes.
-        self._url_opener = urllib_request.build_opener(*handlers)
+        self._set_opener(user, password)
 
         self.request_headers = {"User-Agent": user_agent}
         # Avoid mutable kwarg.
@@ -261,6 +255,128 @@ def __init__(self, base_url="IRIS", major_versions=None, user=None,
 
         self._discover_services()
 
+        # Use EIDA token if provided - this requires setting new url openers.
+        #
+        # This can only happen after the services have been discovered as
+        # the clients needs to know if the fdsnws implementation has support
+        # for the EIDA token system.
+        #
+        # This is a non-standard feature but we support it, given the number
+        # of EIDA nodes out there.
+        if eida_token is not None:
+            # Make sure user/pw are not also given.
+            if user is not None or password is not None:
+                msg = ("EIDA authentication token provided, but "
+                       "user and password are also given.")
+                raise FDSNException(msg)
+            self.set_eida_token(eida_token)
+
+    @property
+    def _has_eida_auth(self):
+        return self.services.get('eida-auth', False)
+
+    def set_credentials(self, user, password):
+        """
+        Set user and password resulting in subsequent web service
+        requests for waveforms being authenticated for potential access to
+        restricted data.
+
+        :type user: str
+        :param user: User name of credentials.
+        :type password: str
+        :param password: Password for given user name.
+        """
+        self._set_opener(user, password)
+
+    def set_eida_token(self, token):
+        """
+        Fetch user and password from the server using the provided token,
+        resulting in subsequent web service requests for waveforms being
+        authenticated for potential access to restricted data.
+        This only works for select EIDA nodes and relies on the auth mechanism
+        described here:
+        http://geofon.gfz-potsdam.de/waveform/archive/auth/index.php
+
+        :type token: str
+        :param token: Token for EIDA authentication mechanism, see
+            http://geofon.gfz-potsdam.de/waveform/archive/auth/index.php. If
+            the client was initialized with options ``user`` and ``password``,
+            these settings will be overridden. This mechanism is only available
+            on select EIDA nodes. The token can be provided in form of the PGP
+            message as a string, or the filename of a local file with the PGP
+            message in it.
+        """
+        user, password = self._resolve_eida_token(token)
+        self.set_credentials(user, password)
+
+    def _set_opener(self, user, password):
+        # Only add the authentication handler if required.
+        handlers = []
+        if user is not None and password is not None:
+            # Create an OpenerDirector for HTTP Digest Authentication
+            password_mgr = urllib_request.HTTPPasswordMgrWithDefaultRealm()
+            password_mgr.add_password(None, self.base_url, user, password)
+            handlers.append(urllib_request.HTTPDigestAuthHandler(password_mgr))
+
+        if (user is None and password is None) or self._force_redirect is True:
+            # Redirect if no credentials are given or the force_redirect
+            # flag is True.
+            handlers.append(CustomRedirectHandler())
+        else:
+            handlers.append(NoRedirectionHandler())
+
+        # Don't install globally to not mess with other codes.
+        self._url_opener = urllib_request.build_opener(*handlers)
+        if self.debug:
+            print('Installed new opener with handlers: {!s}'.format(handlers))
+
+    def _resolve_eida_token(self, token):
+        """
+        Use the token to get credentials.
+        """
+        if not self._has_eida_auth:
+            msg = ("EIDA token authentication requested but service at '{}' "
+                   "does not specify /dataselect/auth in the "
+                   "dataselect/application.wadl.").format(self.base_url)
+            raise FDSNException(msg)
+
+        token_file = None
+        # check if there's a local file that matches the provided string
+        if os.path.isfile(token):
+            token_file = token
+            with open(token_file, 'rb') as fh:
+                token = fh.read().decode()
+        # sanity check on the token
+        if not _validate_eida_token(token):
+            if token_file:
+                msg = ("Read EIDA token from file '{}' but it does not "
+                       "seem to contain a valid PGP message.").format(
+                            token_file)
+            else:
+                msg = ("EIDA token does not seem to be a valid PGP message. "
+                       "If you passed a filename, make sure the file "
+                       "actually exists.")
+            raise ValueError(msg)
+
+        # force https so that we don't send around tokens unsecurely
+        url = 'https://{}/fdsnws/dataselect/1/auth'.format(
+            urlparse(self.base_url).netloc + urlparse(self.base_url).path)
+        # paranoid: check again that we only send the token to https
+        if urlparse(url).scheme != "https":
+            msg = 'This should not happen, please file a bug report.'
+            raise Exception(msg)
+
+        # retrieve user/password using the token
+        if self.debug:
+            print('Downloading {} with eida token data in POST'.format(url))
+
+        # Already does the error checking with fdsnws semantics.
+        response = self._download(url=url, data=token.encode(),
+                                  use_gzip=True, return_string=True)
+
+        user, password = response.decode().split(':')
+        return user, password
+
     def get_events(self, starttime=None, endtime=None, minlatitude=None,
                    maxlatitude=None, minlongitude=None, maxlongitude=None,
                    latitude=None, longitude=None, minradius=None,
@@ -1372,7 +1488,15 @@ def run(self):
                 raise FDSNException("Timeout while requesting '%s'." % url)
 
             if "dataselect" in url:
-                self.services["dataselect"] = WADLParser(wadl).parameters
+                wadl_parser = WADLParser(wadl)
+                self.services["dataselect"] = wadl_parser.parameters
+                # check if EIDA auth endpoint is in wadl
+                # we need to attach it to the discovered services, as these are
+                # later loaded from cache and just attaching an attribute to
+                # this client won't help knowing later if EIDA auth is
+                # supported at the server. a bit ugly but can't be helped.
+                if wadl_parser._has_eida_auth:
+                    self.services["eida-auth"] = True
                 if self.debug is True:
                     print("Discovered dataselect service")
             elif "event" in url and "application.wadl" in url:
@@ -1390,7 +1514,6 @@ def run(self):
                 except ValueError:
                     msg = "Could not parse the catalogs at '%s'." % url
                     warnings.warn(msg)
-
             elif "event" in url and "contributors" in url:
                 try:
                     self.services["available_event_contributors"] = \
@@ -1776,6 +1899,17 @@ def get_bulk_string(bulk, arguments):
     return bulk
 
 
+def _validate_eida_token(token):
+    """
+    Just a basic check if the string contains something that looks like a PGP
+    message
+    """
+    if re.search(pattern='BEGIN PGP MESSAGE', string=token,
+                 flags=re.IGNORECASE):
+        return True
+    return False
+
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod(exclude_empty=True)

obspy/clients/fdsn/routing/eidaws_routing_client.py

@@ -35,7 +35,7 @@ class EIDAWSRoutingClient(BaseRoutingClient):
     """
     def __init__(self, url="http://www.orfeus-eu.org/eidaws/routing/1",
                  include_providers=None, exclude_providers=None,
-                 debug=False, timeout=120):
+                 debug=False, timeout=120, **kwargs):
         """
         Initialize an EIDAWS router client.
 
@@ -48,7 +48,8 @@ def __init__(self, url="http://www.orfeus-eu.org/eidaws/routing/1",
         """
         BaseRoutingClient.__init__(self, debug=debug, timeout=timeout,
                                    include_providers=include_providers,
-                                   exclude_providers=exclude_providers)
+                                   exclude_providers=exclude_providers,
+                                   **kwargs)
         self._url = url
 
     @_assert_filename_not_in_kwargs

obspy/clients/fdsn/routing/federator_routing_client.py

@@ -26,7 +26,7 @@
 class FederatorRoutingClient(BaseRoutingClient):
     def __init__(self, url="http://service.iris.edu/irisws/fedcatalog/1",
                  include_providers=None, exclude_providers=None,
-                 debug=False, timeout=120):
+                 debug=False, timeout=120, **kwargs):
         """
         Initialize a federated routing client.
 
@@ -39,7 +39,8 @@ def __init__(self, url="http://service.iris.edu/irisws/fedcatalog/1",
         """
         BaseRoutingClient.__init__(self, debug=debug, timeout=timeout,
                                    include_providers=include_providers,
-                                   exclude_providers=exclude_providers)
+                                   exclude_providers=exclude_providers,
+                                   **kwargs)
         self._url = url
 
         # Parameters the routing service can work with. If this becomes a

obspy/clients/fdsn/routing/routing_client.py

@@ -16,6 +16,7 @@
 from future.builtins import *  # NOQA
 
 import io
+import re
 from multiprocessing.dummy import Pool as ThreadPool
 
 import decorator
@@ -41,6 +42,8 @@ def RoutingClient(routing_type, *args, **kwargs):  # NOQA
         respectively.
 
     Remaining ``args`` and ``kwargs`` will be passed to the underlying classes.
+    For example, credentials can be supported for all underlying data centers.
+    See :meth:`BaseRoutingClient <BaseRoutingClient.__init__>` for details.
 
     >>> from obspy.clients.fdsn import RoutingClient
 
@@ -84,7 +87,19 @@ def _assert_attach_response_not_in_kwargs(f, *args, **kwargs):
 
 
 def _download_bulk(r):
-    c = client.Client(r["endpoint"], debug=r["debug"], timeout=r["timeout"])
+    # determine base url:
+    # take everything in between (optional) http[s]:// and the next /
+    base_url = re.match(
+        pattern=r'(https?://)?([^/]*)',
+        string=r["endpoint"]).group(2)
+    # get appropriate credentials info from credentials dictionary
+    credentials = r["credentials"].get(base_url, {})
+    if r["debug"] and credentials:
+        print("Fetching from '{}' using{}credentials{}".format(
+            base_url, credentials and " " or " no ",
+            credentials and ": {!s}".format(credentials.keys()) or ""))
+    c = client.Client(r["endpoint"], debug=r["debug"], timeout=r["timeout"],
+                      **credentials)
     if r["data_type"] == "waveform":
         fct = c.get_waveforms_bulk
         service = c.services["dataselect"]
@@ -112,7 +127,7 @@ def _strip_protocol(url):
 # get_events() but also others).
 class BaseRoutingClient(HTTPClient):
     def __init__(self, debug=False, timeout=120, include_providers=None,
-                 exclude_providers=None):
+                 exclude_providers=None, credentials=None):
         """
         :type routing_type: str
         :param routing_type: The type of
@@ -123,10 +138,18 @@ def __init__(self, debug=False, timeout=120, include_providers=None,
         :type include_providers: str or list of str
         :param include_providers: Get data only from these providers. Can be
             the full HTTP address of one of the shortcuts ObsPy knows about.
+        :type credentials: dict
+        :param credentials: Credentials for the individual data centers as a
+            dictionary that maps base url of FDSN web service to either
+            username/password or EIDA token, e.g.
+            ``credentials={
+            'geofon.gfz-potsdam.de': {'token': 'my_token_file.txt'},
+            'service.iris.edu': {'user': 'me', 'password': 'my_pass'}}``
         """
         HTTPClient.__init__(self, debug=debug, timeout=timeout)
         self.include_providers = include_providers
         self.exclude_providers = exclude_providers
+        self.credentials = credentials or {}
 
     @property
     def include_providers(self):
@@ -197,7 +220,8 @@ def _download_parallel(self, split, data_type, **kwargs):
                 "endpoint": k,
                 "bulk_str": v,
                 "data_type": data_type,
-                "kwargs": kwargs})
+                "kwargs": kwargs,
+                "credentials": self.credentials})
         pool = ThreadPool(processes=len(dl_requests))
         results = pool.map(_download_bulk, dl_requests)
 

obspy/clients/fdsn/tests/data/eida_token.txt

@@ -0,0 +1,13 @@
+-----BEGIN PGP MESSAGE-----
+Version: GnuPG v2.0.9 (GNU/Linux)
+
+owGbwMvMwMR4USg04vQMEQbG0+5JDOEWH+OrlXITM3OUrBSUSlKLSxzyk4oLKvXy
+i9KVdBSUyhJzMlPiS/NKIAqMDIwMdA0MgSjEwMAKjKKUajsZZVgYGJkY2FiZQOYx
+cHEKwCx5Lcv+T0W79KhOxauJLEnP7t0768ZTvzk1Z7XbnSSmqvB7/rfsRfOvSrMt
+39GjosTgveqB7gHRYz0+jPP36dj073r2s8h6Xvu0+sAtLbfO6C3xNolRDPgQ4tud
+m6J8tm573KGFFTG/+uI/hSir56Uav33xN1DBwKPs1pt8H6vFirf81J5Y3eR8wbmm
+c9eXV5VeBy/9bYrd+TXsnFkiayjrxViWdm8uUXnj4hzDE5vvTeVZGH/nclVE1v6z
+DO3n3STZvjMKsRUxVPKcnTmHcU64qiTX10bRqi1yHzuMs35Gzz0sMpXbTencLduG
+hMrp91PMe04c0otcNytOKDFe/uDva3sd9jZKcxSXb3Gs2Rxf2QgA
+=VVHX
+-----END PGP MESSAGE-----