Merge 93b8ba1 into 4bf9901

CHANGELOG.txt

@@ -4,11 +4,13 @@ master:
      arguments to not download certain pieces of data. (see #1305)
    * The mass downloader can now download stations that are part of a given
      inventory object.
+ - obspy.css
+    * Read support for NNSA KB Core format waveform data. (see #1290, #1301)
 
 1.0.x:
  - obspy.clients.fdsn:
     * Local URLs are now recognized as valid URLs. (see #1309)
-    * Some bug fixes for the mass downloader. (see #1293, #1304)
+	* Some bug fixes for the mass downloader. (see #1293, #1304)
 
 1.0.0: (doi: 10.5281/zenodo.46151)
  - General:

obspy/CONTRIBUTORS.txt

@@ -64,3 +64,4 @@ Andrew Walker
 Marcus Walther
 Joachim Wassermann
 Mark C. Williams
+Andrew Winkelman

obspy/core/util/base.py

@@ -50,7 +50,7 @@
 WAVEFORM_PREFERRED_ORDER = ['MSEED', 'SAC', 'GSE2', 'SEISAN', 'SACXY', 'GSE1',
                             'Q', 'SH_ASC', 'SLIST', 'TSPAIR', 'Y', 'PICKLE',
                             'SEGY', 'SU', 'SEG2', 'WAV', 'DATAMARK', 'CSS',
-                            'AH', 'PDAS', 'KINEMETRICS_EVT']
+                            'NNSA_KB_CORE', 'AH', 'PDAS', 'KINEMETRICS_EVT']
 EVENT_PREFERRED_ORDER = ['QUAKEML', 'NLLOC_HYP']
 # waveform plugins accepting a byteorder keyword
 WAVEFORM_ACCEPT_BYTEORDER = ['MSEED', 'Q', 'SAC', 'SEGY', 'SU']

obspy/io/css/core.py

@@ -70,12 +70,44 @@ def _is_css(filename):
     except:
         return False
     return True
+
+def _is_nnsa_kb_core(filename):
+    """
+    Checks whether a file is NNSA KB Core waveform data (header) or not.
+
+    :type filename: str
+    :param filename: NNSA KB Core file to be checked.
+    :rtype: bool
+    :return: ``True`` if a NNSA KB Core waveform header file.
+    """
+    # Fixed file format.
+    # Tests:
+    #  - the length of each line (287 chars)
+    #  - two epochal time fields
+    #    (for position of dot and if they convert to UTCDateTime)
+    #  - supported data type descriptor
+    try:
+        with open(filename, "rb") as fh:
+            lines = fh.readlines()
+            # check for empty file
+            if not lines:
+                return False
+            # check every line
+            for line in lines:
+                assert(len(line.rstrip(b"\n\r")) == 287)
+                assert(line[27] == b".")
+                UTCDateTime(float(line[16:33]))
+                assert(line[73] == b".")
+                UTCDateTime(float(line[62:79]))
+                assert(line[144:146] in DTYPE)
+    except:
+        return False
+    return True
 
 
 def _read_css(filename, **kwargs):
     """
     Reads a CSS waveform file and returns a Stream object.
-
     .. warning::
         This function should NOT be called directly, it registers via the
         ObsPy :func:`~obspy.core.stream.read` function, call this instead.
@@ -119,3 +151,51 @@ def _read_css(filename, **kwargs):
         tr = Trace(data, header=header)
         traces.append(tr)
     return Stream(traces=traces)
+
+def _read_nnsa_kb_core(filename, **kwargs):
+    """
+    Reads a NNSA KB Core waveform file and returns a Stream object.
+    .. warning::
+        This function should NOT be called directly, it registers via the
+        ObsPy :func:`~obspy.core.stream.read` function, call this instead.
+
+    :type filename: str
+    :param filename: NNSA KB Core file to be read.
+    :rtype: :class:`~obspy.core.stream.Stream`
+    :returns: Stream with Traces specified by given file.
+    """
+    # read metafile with info on single traces
+    with open(filename, "rb") as fh:
+        lines = fh.readlines()
+    basedir = os.path.dirname(filename)
+    traces = []
+    # read single traces
+    for line in lines:
+        npts = int(line[80:88])
+        dirname = line[149:213].strip().decode()
+        filename = line[214:246].strip().decode()
+        filename = os.path.join(basedir, dirname, filename)
+        offset = int(line[247:257])
+        dtype = DTYPE[line[144:146]]
+        if isinstance(dtype, tuple):
+            read_fmt = np.dtype(dtype[0])
+            fmt = dtype[1]
+        else:
+            read_fmt = np.dtype(dtype)
+            fmt = read_fmt
+        with open(filename, "rb") as fh:
+            fh.seek(offset)
+            data = fh.read(read_fmt.itemsize * npts)
+            data = from_buffer(data, dtype=read_fmt)
+            data = np.require(data, dtype=fmt)
+        header = {}
+        header['station'] = line[0:6].strip().decode()
+        header['channel'] = line[7:15].strip().decode()
+        header['starttime'] = UTCDateTime(float(line[16:33]))
+        header['sampling_rate'] = float(line[89:100])
+        header['calib'] = float(line[101:117])
+        header['calper'] = float(line[118:134])
+        tr = Trace(data, header=header)
+        traces.append(tr)
+    return Stream(traces=traces)
+

obspy/io/css/tests/data/README

@@ -31,3 +31,8 @@ Test data for CSS support of ObsPy:
 - Field 16: waveform directory name
 - Field 17: waveform file name
 - for meaning of other field see page 12 (15 in pdf) in ../../docs/SchemaReferenceMan.pdf
+
+'test_nnsa_kb.wfdisc'
+
+- similar layout to CSS: fields are the same but columns are different
+- for details, see page 27 (70 in pdf) in ../../docs/NNSA_KB_CORE_SchemaMan.pdf

obspy/io/css/tests/data/test2.wfdisc

@@ -0,0 +1,6 @@
+TESTbe HHZ       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - s4 - ./                                                               201101311155.10.be.w                      0         0 2011/01/31         
+TESTbe HHE       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - s4 - ./                                                               201101311155.10.be.w                  19200         0 2011/01/31         
+TESTbe HHN       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - s4 - ./                                                               201101311155.10.be.w                  38400         0 2011/01/31         
+TESTle HHZ       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - i4 - ./                                                               201101311155.10.le.w                      0         0 2011/01/31         
+TESTle HHE       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - i4 - ./                                                               201101311155.10.le.w                  19200         0 2011/01/31         
+TESTle HHN       1296474900.00000         1        1  2011031  1296474959.98800  4800     80.0                    1.0              1.0 3ESPC  - i4 - ./                                                               201101311155.10.le.w                  38400         0 2011/01/31         

obspy/io/css/tests/test_core.py

@@ -16,7 +16,7 @@
 from obspy import read
 from obspy.core import Stream, Trace, UTCDateTime
 from obspy.core.util import NamedTemporaryFile
-from obspy.io.css.core import _is_css, _read_css
+from obspy.io.css.core import _is_css, _read_css, _is_nnsa_kb_core, _read_nnsa_kb_core
 
 
 class CoreTestCase(unittest.TestCase):
@@ -26,7 +26,8 @@ class CoreTestCase(unittest.TestCase):
     def setUp(self):
         # directory where the test files are located
         self.path = os.path.join(os.path.dirname(__file__), 'data')
-        self.filename = os.path.join(self.path, 'test.wfdisc')
+        self.filename = os.path.join(self.path, 'test.wfdisc')  # CSS
+        self.filename2 = os.path.join(self.path, 'test2.wfdisc')  # NNSA_KB_CORE
         # set up stream for validation
         header = {}
         header['station'] = 'TEST'
@@ -65,16 +66,29 @@ def test_is_css(self):
             fh = open(tempfile, "wb")
             fh.close()
             assert(not _is_css(tempfile))
+
+    def test_is_nnsa_kb_core(self):
+        """
+        Read files via obspy.core.stream.read function.
+        """
+        # 1
+        assert(_is_nnsa_kb_core(self.filename2))
+        # check that empty files are not recognized as NNSA_KB_CORE
+        with NamedTemporaryFile() as tf:
+            tempfile = tf.name
+            fh = open(tempfile, "wb")
+            fh.close()
+            assert(not _is_nnsa_kb_core(tempfile))
 
-    def test_read_via_obspy(self):
+    def test_css_read_via_obspy(self):
         """
         Read files via obspy.core.stream.read function.
         """
         # 1
         st = read(self.filename)
         self.assertEqual(st, self.st_result)
 
-    def test_read_via_module(self):
+    def test_css_read_via_module(self):
         """
         Read files via obspy.io.css.core._read_css function.
         """
@@ -84,6 +98,25 @@ def test_read_via_module(self):
         for tr in self.st_result:
             tr.stats.pop('_format')
         self.assertEqual(st, self.st_result)
+
+    def test_nnsa_kb_core_read_via_obspy(self):
+        """
+        Read files via obspy.core.stream.read function.
+        """
+        # 1
+        st = read(self.filename2)
+        self.assertEqual(st, self.st_result)
+
+    def test_nnsa_kb_core_read_via_module(self):
+        """
+        Read files via obspy.io.css.core._read_nnsa_kb_core function.
+        """
+        # 1
+        st = _read_nnsa_kb_core(self.filename2)
+        # _format entry is not present when using low-level function
+        for tr in self.st_result:
+            tr.stats.pop('_format')
+        self.assertEqual(st, self.st_result)
 
 
 def suite():

setup.py

@@ -148,6 +148,7 @@
         'GSE1 = obspy.io.gse2.core',
         'GSE2 = obspy.io.gse2.core',
         'MSEED = obspy.io.mseed.core',
+        'NNSA_KB_CORE = obspy.io.css.core',
         'PDAS = obspy.io.pdas.core',
         'SAC = obspy.io.sac.core',
         'SACXY = obspy.io.sac.core',
@@ -181,6 +182,10 @@
         'isFormat = obspy.io.css.core:_is_css',
         'readFormat = obspy.io.css.core:_read_css',
         ],
+    'obspy.plugin.waveform.NNSA_KB_CORE': [
+        'isFormat = obspy.io.css.core:_is_nnsa_kb_core',
+        'readFormat = obspy.io.css.core:_read_nnsa_kb_core',
+        ],
     'obspy.plugin.waveform.DATAMARK': [
         'isFormat = obspy.io.datamark.core:_is_datamark',
         'readFormat = obspy.io.datamark.core:_read_datamark',