Add support for the IRIS Federated Catalog Service

misc/docs/source/packages/obspy.clients.fdsn.rst

@@ -10,6 +10,7 @@
        :nosignatures:
 
        client.Client
+       client.FederatedClient
 
     .. comment to end block
 
@@ -20,6 +21,10 @@
        :nosignatures:
 
        client
+       routers
+       routers.fedcatalog_parser
+       routers.fedcatalog_client.FederatedClient
+       routers.routing_client
        mass_downloader
        mass_downloader.domain
        mass_downloader.mass_downloader.MassDownloader

obspy/clients/fdsn/__init__.py

@@ -3,16 +3,16 @@
 obspy.clients.fdsn - FDSN web service client for ObsPy
 ======================================================
 The obspy.clients.fdsn package contains a client to access web servers that
-implement the FDSN web service definitions (https://www.fdsn.org/webservices/).
+implement the `FDSN web service definitions`_.
 
 :copyright:
     The ObsPy Development Team (devs@obspy.org)
 :license:
     GNU Lesser General Public License, Version 3
     (https://www.gnu.org/copyleft/lesser.html)
 
-Basic Usage
------------
+Basic FDSN Client Usage
+-----------------------
 
 The first step is always to initialize a client object.
 
@@ -134,8 +134,69 @@
                                         endtime=endtime)
         inventory.plot()
 
+
+Basic FDSN FedCatalog Client Usage
+----------------------------------
+
+The
+:mod:`FDSN fedcatalog_client <obspy.clients.fdsn.routers.fedcatalog_client>`
+module provides federated
+access to multiple web servers that implement the
+`FDSN Station and Dataselect web service definitions
+<https://www.fdsn.org/webservices/>`_.
+
+The first step is always to initialize a :class:`FederatedClient` object.
+
+>>> from obspy.clients.fdsn import FederatedClient
+>>> client = FederatedClient()
+
+(1) :meth:`~obspy.clients.fdsn.routers.fedcatalog_client.
+FederatedClient.get_waveforms()`: The following
+example illustrates how to request 60 minutes of the ``"LHZ"`` channel of
+station Apirathos, Naxos, Greece (``"APE"``) of the GEOFON (``"GE"``) for a
+seismic event around 2006-01-08T11:34:54.000 (UTC). Results are returned as a
+:class:`~obspy.core.stream.Stream` object.
+
+    >>> from obspy import UTCDateTime
+    >>> t = UTCDateTime("2006-01-08T11:34:54.000")
+    >>> st = client.get_waveforms("GE", "APE", "", "LHZ", t, t + 60 * 60)
+    >>> st.plot()  # doctest: +SKIP
+
+    .. plot::
+
+        from obspy import UTCDateTime
+        from obspy.clients.fdsn import Client
+        client = Client('GFZ')
+        t = UTCDateTime("2006-01-08T11:34:54.000")
+        st = client.get_waveforms("GE", "APE", "", "LHZ", t, t + 60 * 60)
+        st.plot()
+
+(2) :meth:`~obspy.clients.fdsn.routers.fedcatalog_client.
+FederatedClient.get_stations()`: Uses the IRIS Fed Catalog web service to
+    return station metadata as an
+    :class:`~obspy.core.inventory.inventory.Inventory` object.
+
+    >>> inventory = client.get_stations(network="GE", station="A*",
+    ...                                 channel="?HZ", level="station",
+    ...                                 endtime="2016-12-31")
+    >>> print(inventory)  # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
+    Inventory created at 2...Z
+        Sending institution: SeisComP3 (GFZ)
+        Contains:
+            Networks (1):
+                GE
+            Stations (4):
+                GE.APE (GEOFON Station Apirathos, Naxos)
+                GE.APE (NOA/GEOFON Station Apeiranthos,Naxos, Greece)
+                GE.APEZ (GEOFON Station Moni Apezanon, Greece)
+                GE.ARPR (GEOFON/MedNet/KOERI Station Arapgir, Turkey)
+            Channels (0):
+    <BLANKLINE>
+
 Please see the documentation for each method for further information and
 examples.
+
+.. _FDSN web service definitions: https://www.fdsn.org/webservices/
 """
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
@@ -144,6 +205,7 @@
 
 from .client import Client  # NOQA
 from .header import URL_MAPPINGS  # NOQA
+from .routers import FederatedClient  # NOQA
 
 
 # insert supported URL mapping list dynamically in docstring

obspy/clients/fdsn/client.py

@@ -20,22 +20,12 @@
 import io
 import os
 import re
-import sys
 from socket import timeout as socket_timeout
 import textwrap
 import threading
 import warnings
 from collections import OrderedDict
 
-if sys.version_info.major == 2:
-    from urllib import urlencode
-    import urllib2 as urllib_request
-    import Queue as queue
-else:
-    from urllib.parse import urlencode
-    import urllib.request as urllib_request
-    import queue
-
 from lxml import etree
 
 import obspy
@@ -46,6 +36,15 @@
                      FDSNRedirectException, FDSNNoDataException)
 from .wadl_parser import WADLParser
 
+if PY2:
+    from urllib import urlencode
+    import urllib2 as urllib_request
+    import Queue as queue
+else:
+    from urllib.parse import urlencode
+    import urllib.request as urllib_request
+    import queue
+
 
 DEFAULT_SERVICE_VERSIONS = {'dataselect': 1, 'station': 1, 'event': 1}
 
@@ -889,7 +888,7 @@ def get_waveforms_bulk(self, bulk, quality=None, minimumlength=None,
         url = self._build_url("dataselect", "query")
 
         data_stream = self._download(url,
-                                     data=bulk.encode('ascii', 'strict'))
+                                     data=bulk)
         data_stream.seek(0, 0)
         if filename:
             self._write_to_file_object(filename, data_stream)
@@ -1034,7 +1033,7 @@ def get_stations_bulk(self, bulk, level=None, includerestricted=None,
         url = self._build_url("station", "query")
 
         data_stream = self._download(url,
-                                     data=bulk.encode('ascii', 'strict'))
+                                     data=bulk)
         data_stream.seek(0, 0)
         if filename:
             self._write_to_file_object(filename, data_stream)
@@ -1079,7 +1078,10 @@ def _get_bulk_string(self, bulk, arguments):
                 msg = ("Unrecognized input for 'bulk' argument. Please "
                        "contact developers if you think this is a bug.")
                 raise NotImplementedError(msg)
-        return bulk
+        if PY2:
+            return bulk
+        else:
+            return bulk.encode('ASCII')
 
     def _write_to_file_object(self, filename_or_object, data_stream):
         if hasattr(filename_or_object, "write"):
@@ -1296,53 +1298,7 @@ def _download(self, url, return_string=False, data=None, use_gzip=True):
             url, opener=self._url_opener, headers=self.request_headers,
             debug=self.debug, return_string=return_string, data=data,
             timeout=self.timeout, use_gzip=use_gzip)
-        # get detailed server response message
-        if code != 200:
-            try:
-                server_info = data.read()
-            except Exception:
-                server_info = None
-            else:
-                server_info = server_info.decode('ASCII', errors='ignore')
-            if server_info:
-                server_info = "\n".join(
-                    line for line in server_info.splitlines() if line)
-        # No data.
-        if code == 204:
-            raise FDSNNoDataException("No data available for request.",
-                                      server_info)
-        elif code == 400:
-            msg = ("Bad request. If you think your request was valid "
-                   "please contact the developers.")
-            raise FDSNException(msg, server_info)
-        elif code == 401:
-            raise FDSNException("Unauthorized, authentication required.",
-                                server_info)
-        elif code == 403:
-            raise FDSNException("Authentication failed.", server_info)
-        elif code == 413:
-            raise FDSNException("Request would result in too much data. "
-                                "Denied by the datacenter. Split the request "
-                                "in smaller parts", server_info)
-        # Request URI too large.
-        elif code == 414:
-            msg = ("The request URI is too large. Please contact the ObsPy "
-                   "developers.", server_info)
-            raise NotImplementedError(msg)
-        elif code == 500:
-            raise FDSNException("Service responds: Internal server error",
-                                server_info)
-        elif code == 503:
-            raise FDSNException("Service temporarily unavailable", server_info)
-        elif code is None:
-            if "timeout" in str(data).lower():
-                raise FDSNException("Timed Out")
-            else:
-                raise FDSNException("Unknown Error (%s): %s" % (
-                    (str(data.__class__.__name__), str(data))))
-        # Catch any non 200 codes.
-        elif code != 200:
-            raise FDSNException("Unknown HTTP code: %i" % code, server_info)
+        raise_on_error(code, data)
         return data
 
     def _build_url(self, service, resource_type, parameters={}):
@@ -1442,7 +1398,7 @@ def run(self):
             elif isinstance(wadl, FDSNRedirectException):
                 redirect_messages.add(str(wadl))
                 continue
-            elif wadl == "timeout":
+            elif wadl.decode('utf-8') == "timeout":
                 raise FDSNException("Timeout while requesting '%s'." % url)
 
             if "dataselect" in url:
@@ -1622,6 +1578,67 @@ def build_url(base_url, service, major_version, resource_type,
     return url
 
 
+def raise_on_error(code, data):
+    """
+    Raise an error for non-200 HTTP response codes
+
+    Note: Also used by the ~obspy.clients.fdsn.routers.fedcatalog_client
+          module.
+
+    :type code: int
+    :param code: HTTP response code
+    :type data: io.BytesIO
+    :param data: Data returned by the server
+    """
+    # get detailed server response message
+    if code != 200:
+        try:
+            server_info = data.read()
+        except Exception:
+            server_info = None
+        else:
+            server_info = server_info.decode('ASCII', errors='ignore')
+        if server_info:
+            server_info = "\n".join(
+                line for line in server_info.splitlines() if line)
+    # No data.
+    if code == 204:
+        raise FDSNNoDataException("No data available for request.",
+                                  server_info)
+    elif code == 400:
+        msg = ("Bad request. If you think your request was valid "
+               "please contact the developers.")
+        raise FDSNException(msg, server_info)
+    elif code == 401:
+        raise FDSNException("Unauthorized, authentication required.",
+                            server_info)
+    elif code == 403:
+        raise FDSNException("Authentication failed.", server_info)
+    elif code == 413:
+        raise FDSNException("Request would result in too much data. "
+                            "Denied by the datacenter. Split the request "
+                            "in smaller parts", server_info)
+    # Request URI too large.
+    elif code == 414:
+        msg = ("The request URI is too large. Please contact the ObsPy "
+               "developers.", server_info)
+        raise NotImplementedError(msg)
+    elif code == 500:
+        raise FDSNException("Service responds: Internal server error",
+                            server_info)
+    elif code == 503:
+        raise FDSNException("Service temporarily unavailable", server_info)
+    elif code is None:
+        if "timeout" in str(data).lower():
+            raise FDSNException("Timed Out")
+        else:
+            raise FDSNException("Unknown Error (%s): %s" % (
+                (str(data.__class__.__name__), str(data))))
+    # Catch any non 200 codes.
+    elif code != 200:
+        raise FDSNException("Unknown HTTP code: %i" % code, server_info)
+
+
 def download_url(url, opener, timeout=10, headers={}, debug=False,
                  return_string=True, data=None, use_gzip=True):
     """
@@ -1635,6 +1652,9 @@ def download_url(url, opener, timeout=10, headers={}, debug=False,
     specified.
 
     Performs a http GET if data=None, otherwise a http POST.
+
+    Note: Also used by the ~obspy.clients.fdsn.routers.fedcatalog_client
+          module.
     """
     if debug is True:
         print("Downloading %s %s requesting gzip compression" % (