Merge pull request #1746 from krischer/raise-exception-when-reading-v…

…ery-large-mseed-file

Raise exception when attempting to read very large mini-SEED files

CHANGELOG.txt

@@ -104,6 +104,8 @@ master: (doi: 10.5281/zenodo.165135)
      InternalMSEEDReadingWarning now called InternalMSEEDWarning as both
      can now also be raised in non-reading contexts (see #1658).
    * Should no-longer segfault with arbitrarily truncated files (see #1728).
+   * Will now raise an exception when attempting to directly read mini-SEED
+     files larger than 2048 MiB (#1746).
  - obspy.io.nlloc:
    * Set preferred origin of event (see #1570)
  - obspy.io.nordic:

obspy/io/mseed/__init__.py

@@ -168,6 +168,10 @@ class ObsPyMSEEDFilesizeTooSmallError(ObsPyMSEEDReadingError):
     pass
 
 
+class ObsPyMSEEDFilesizeTooLargeError(ObsPyMSEEDReadingError):
+    pass
+
+
 __all__ = ['InternalMSEEDError', 'InternalMSEEDWarning', 'ObsPyMSEEDError',
            'ObsPyMSEEDFilesizeTooSmallError']
 

obspy/io/mseed/core.py

@@ -17,7 +17,8 @@
 
 from obspy import Stream, Trace, UTCDateTime
 from obspy.core.util import NATIVE_BYTEORDER
-from . import util, InternalMSEEDError, ObsPyMSEEDFilesizeTooSmallError
+from . import (util, InternalMSEEDError, ObsPyMSEEDFilesizeTooSmallError,
+               ObsPyMSEEDFilesizeTooLargeError)
 from .headers import (DATATYPES, ENCODINGS, HPTERROR, HPTMODULUS, SAMPLETYPE,
                       SEED_CONTROL_HEADERS, UNSUPPORTED_ENCODINGS,
                       VALID_CONTROL_HEADERS, VALID_RECORD_LENGTHS, Selections,
@@ -271,6 +272,13 @@ def _read_mseed(mseed_object, starttime=None, endtime=None, headonly=False,
         msg = "The smallest possible mini-SEED record is made up of 128 " \
               "bytes. The passed buffer or file contains only %i." % length
         raise ObsPyMSEEDFilesizeTooSmallError(msg)
+    elif length > 2 ** 31:
+        msg = ("ObsPy can currently not directly read mini-SEED files that "
+               "are larger than 2^31 bytes (2048 MiB). To still read it, "
+               "please read the file in chunks as documented here: "
+               "https://github.com/obspy/obspy/pull/1419"
+               "#issuecomment-221582369")
+        raise ObsPyMSEEDFilesizeTooLargeError(msg)
 
     info = util.get_record_information(mseed_object, endian=bo)
 

obspy/io/mseed/tests/test_mseed_special_issues.py

@@ -18,11 +18,12 @@
 import numpy as np
 
 from obspy import Stream, Trace, UTCDateTime, read
-from obspy.core.compatibility import from_buffer
+from obspy.core.compatibility import from_buffer, mock
 from obspy.core.util import NamedTemporaryFile
 from obspy.core.util.attribdict import AttribDict
-from obspy.io.mseed import InternalMSEEDError, \
-    InternalMSEEDWarning, ObsPyMSEEDFilesizeTooSmallError
+from obspy.io.mseed import (InternalMSEEDError, InternalMSEEDWarning,
+                            ObsPyMSEEDFilesizeTooSmallError,
+                            ObsPyMSEEDFilesizeTooLargeError)
 from obspy.io.mseed import util
 from obspy.io.mseed.core import _read_mseed, _write_mseed
 from obspy.io.mseed.headers import clibmseed
@@ -1143,6 +1144,25 @@ def test_reading_less_than_128_bytes(self):
             "The smallest possible mini-SEED record is made up of 128 bytes. "
             "The passed buffer or file contains only 127.")
 
+    @mock.patch("os.path.getsize")
+    def test_reading_file_larger_than_2048_mib(self, getsize_mock):
+        """
+        ObsPy can currently not directly read files that are larger than
+        2^31 bytes. This raises an exception with a description of how to
+        get around it.
+        """
+        getsize_mock.return_value = 2 ** 31 + 1
+        filename = os.path.join(self.path, 'data',
+                                'BW.BGLD.__.EHE.D.2008.001.first_10_records')
+        with self.assertRaises(ObsPyMSEEDFilesizeTooLargeError) as e:
+            _read_mseed(filename)
+        self.assertEqual(
+            e.exception.args[0],
+            "ObsPy can currently not directly read mini-SEED files that are "
+            "larger than 2^31 bytes (2048 MiB). To still read it, please "
+            "read the file in chunks as documented here: "
+            "https://github.com/obspy/obspy/pull/1419#issuecomment-221582369")
+
 
 def suite():
     return unittest.makeSuite(MSEEDSpecialIssueTestCase, 'test')