Merge branch 'pre-master' of github.com:loerum/mipp into pre-master

doc/source/api.rst

@@ -0,0 +1,147 @@
+The :mod:`mipp` API
+===================
+
+MIPP
+----
+
+.. automodule:: mipp
+   :members:
+   :undoc-members:
+
+Metadata
+~~~~~~~~
+
+.. automodule:: mipp.mda
+   :members:
+   :undoc-members:
+
+Configuration
+~~~~~~~~~~~~~
+
+.. automodule:: mipp.cfg
+   :members:
+   :undoc-members:
+
+Logging
+~~~~~~~
+
+.. automodule:: mipp.log
+   :members:
+   :undoc-members:
+
+
+XRIT input layer
+----------------
+
+MSG
+~~~
+
+.. automodule:: mipp.xrit.MSG
+   :members:
+   :undoc-members:
+
+
+GOMS
+~~~~
+
+.. automodule:: mipp.xrit.GOMS
+   :members:
+   :undoc-members:
+
+MTP
+~~~
+
+.. automodule:: mipp.xrit.MTP
+   :members:
+   :undoc-members:
+
+SGS
+~~~
+
+.. automodule:: mipp.xrit.SGS
+   :members:
+   :undoc-members:
+
+_xrit
+~~~~~
+
+.. automodule:: mipp.xrit._xrit
+   :members:
+   :undoc-members:
+
+bin_reader
+~~~~~~~~~~
+
+.. automodule:: mipp.xrit.bin_reader
+   :members:
+   :undoc-members:
+
+loader
+~~~~~~
+
+.. automodule:: mipp.xrit.loader
+   :members:
+   :undoc-members:
+
+convert
+~~~~~~~
+
+.. automodule:: mipp.xrit.convert
+   :members:
+   :undoc-members:
+
+sat
+~~~
+
+.. automodule:: mipp.xrit.sat
+   :members:
+   :undoc-members:
+
+Metadata
+~~~~~~~~
+
+.. automodule:: mipp.xrit.mda
+   :members:
+   :undoc-members:
+
+
+XSAR input layer
+----------------
+
+Cosmo Sky-med
+~~~~~~~~~~~~~
+
+.. automodule:: mipp.xsar.CSK
+   :members:
+   :undoc-members:
+
+
+Radarsat-2
+~~~~~~~~~~
+
+.. automodule:: mipp.xsar.RS2
+   :members:
+   :undoc-members:
+
+
+Terra-SAR X
+~~~~~~~~~~~
+
+.. automodule:: mipp.xsar.TSX
+   :members:
+   :undoc-members:
+
+sat
+~~~
+
+.. automodule:: mipp.xsar.sat
+   :members:
+   :undoc-members:
+
+Metadata
+~~~~~~~~
+
+.. automodule:: mipp.xsar.mda
+   :members:
+   :undoc-members:
+

doc/source/conf.py

@@ -17,6 +17,33 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.append(os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../../mipp'))
+sys.path.insert(0, os.path.abspath('../../mipp/xrit'))
+
+class Mock(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        return Mock()
+
+    @classmethod
+    def __getattr__(cls, name):
+        if name in ('__file__', '__path__'):
+            return '/dev/null'
+        elif name[0] == name[0].upper():
+            mockType = type(name, (), {})
+            mockType.__module__ = __name__
+            return mockType
+        else:
+            return Mock()
+
+
+MOCK_MODULES = ['numpy', ]
+for mod_name in MOCK_MODULES:
+    sys.modules[mod_name] = Mock()
+
 
 # -- General configuration -----------------------------------------------------
 
@@ -59,6 +86,10 @@
 # Else, today_fmt is used as the format for a strftime call.
 #today_fmt = '%B %d, %Y'
 
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 

doc/source/index.rst

@@ -13,6 +13,7 @@ This is a presentation of:
 
    introduction
    calibration
+   api
 
 Indices and tables
 ==================

doc/source/introduction.rst

@@ -4,13 +4,16 @@
 
 ``mipp`` is a Meteorological Ingest-Processing Package (http://github.com/loerum/mipp).
 
- It's a Python libray and it's main task is to convert satellite level-1.5 data into a 
- format understood by ``mpop`` (http://github.com/mraspaud/mpop). 
+ It's a Python library and it's main task is to convert low level satellite
+ data into a format understood by ``mpop``
+ (http://github.com/mraspaud/mpop). The primary purpose is to support
+ Geostationary satellite data (level 1.5) but there is also support for the
+ reading of some polar orbiting SAR data (see below).
 
  A more sophisticated interface to satellite data objects is supported by ``mpop``.
 
-In the start, it will handle **MET7**, **GEOS11**, **GOES12** and **MTSAT1R**,
-"eumetcasted" FSD data::
+Currently it handles data from all current Meteosat Second Generation (MSG)
+satellites, Meteosat 7, GOES 11-15, MTSAT's, and GOMS, all as retrieved via EUMETCast::
 
   L-000-MTP___-MET7________-00_7_057E-PRO______-201002261600-__
   L-000-MTP___-MET7________-00_7_057E-000001___-201002261600-C_
@@ -25,11 +28,26 @@ In the start, it will handle **MET7**, **GEOS11**, **GOES12** and **MTSAT1R**,
   ...
   ...
 
+In addition ``mipp`` handles Synthetic Apperture Radar (SAR) data from
+Terrscan-X, Cosmo-Sky Med, and Radarsat 2.
 
 ``mipp`` will:
-  * decompress XRIT files (if Eumetsat's ``xRITDecompress`` is available).
-  * decode/strip-off (according to [CGMS]_, [MTP]_, [SGS]_) XRIT headers and collect meta-data.
-  * catenate image data into a numpy-array.
+
+  * Decompress XRIT files (if Eumetsat's ``xRITDecompress`` is
+    available). Software to uncompress HRIT/XRIT can be obtained from EUMETSAT
+    (register and download the `Public Wavelet Transform Decompression Library
+    Software`_). Please be sure to set the environment variable
+    ``XRIT_DECOMPRESS_PATH`` to point to the full path to the decompression
+    software, e.g. ``/usr/bin/xRITDecompress``. Also you can specify where the
+    decompressed files should be stored after decompression, using the
+    environment variable ``XRIT_DECOMPRESS_OUTDIR``. If this variable is not
+    set the decompressed files will be found in the same directory as the
+    compressed ones.
+
+
+  * Decode/strip-off (according to [CGMS]_, [MTP]_, [SGS]_) XRIT headers and collect meta-data.
+
+  * Catenate image data into a numpy-array.
 
     * if needed, convert 10 bit data to 16 bit
     * if a region is defined (by a slice or center, size), only read what is specified.
@@ -166,8 +184,31 @@ Usage
     image_data.tofile(fp)
     fp.close()
 
+
+Examples of the usage of some lower level tools
+-----------------------------------------------
+
+Here an example how to get the observation times (embedded in the 'Image
+Segment Line Quality' record) of each scanline in a segment:
+
+.. code-block:: python
+
+    import mipp.xrit.MSG
+
+    segfile = "/local_disk/data/MSG/HRIT/H-000-MSG3__-MSG3________-WV_062___-000002___-201311211300-__"
+    lineq = mipp.xrit.MSG.get_scanline_quality(segfile)
+    print lineq[0]
+   
+    (465, datetime.datetime(2013, 11, 21, 13, 1, 48, 924000), 1, 1, 0)
+    
+
 A script, process_fsd
 ---------------------
+
+The script is intended for work on other geostationary data than the MSG
+(Meteosat) data, the so-called Foreign Satellite Data (FSD). That is e.g. GOES,
+MTSAT and COMS.
+
 .. code-block:: text
 
     process_fsd --check-satellite <prologue-file>
@@ -190,8 +231,13 @@ A script, process_fsd
 
 ==============================
 
+ .. _Public Wavelet Transform Decompression Library Software: http://www.eumetsat.int/website/home/Data/DataDelivery/SupportSoftwareandTools/index.html
  .. [CGMS] LRIT/HRIT Global Specification; CGMS 03; Issue 2.6; 12 August 1999 
     "MSG Ground Segment LRIT/HRIT Mission Specific Implementation"
     EUM/MSG/SPE/057; Issue 6; 21 June 2006 
  .. [MTP] "The Meteosat Archive; Format Guide No. 1; Basic Imagery: OpenMTP Format"; EUM FG 1; Rev 2.1; April 2000
  .. [SGS] "MSG Ground Segment LRIT/HRIT Mission Specific Implementation"; EUM/MSG/SPE/057; Issue 6; 21 June 2006
+
+
+
+

mipp/xrit/MSG.py

@@ -793,7 +793,7 @@ def read_epiheader(fp):
     return ftr
 
 def read_metadata(prologue, image_files, epilogue):
-    """ Selected items from the Meteosat-9 prolog file.
+    """ Selected items from the MSG prologue file.
     """
     segment_size = 464 # number of lines in a segment
 
@@ -859,6 +859,29 @@ def read_metadata(prologue, image_files, epilogue):
 
     return md
 
+def read_obstimes(epilogue):
+    """Get the start and end full disk scan times from the Epilogue file."""
+    epi = _xrit.read_epilogue(epilogue)
+    fpt = StringIO(epi.data)
+    ftr = read_epiheader(fpt)
+    return (ftr['ForwardScanStart'], ftr['ForwardScanEnd'])
+
+def read_scanline_quality(segment_filename):
+    """Get the line quality data from the segment file. This is from the
+    ImageSegmentLineQuality class of the _xrit module, which maps the Header
+    Type 129 - Image Segment Line Quality as decribed in the 'MSG Ground
+    Segment LRIT/HRIT Mission Specific Implementation, EUM/MSG/SPE/057'
+    document, see reference below.
+    
+    This record contains the scan line mean acquisition times
+    
+    http://www.eumetsat.int/website/wcm/idc/idcplg?IdcService=GET_FILE&dDocName=PDF_TEN_05057_SPE_MSG_LRIT_HRI&RevisionSelectionMethod=LatestReleased&Rendition=Web
+
+    """
+    imgdata = _xrit.read_imagedata(segment_filename)
+    return imgdata.image_quality.line_quality
+
+
 if __name__ == '__main__':
     p = _xrit.read_prologue(sys.argv[1])
     e = _xrit.read_epilogue(sys.argv[-1])

mipp/xrit/_xrit.py

@@ -24,15 +24,17 @@
            'list']
 
 def decompress(infile, outdir='.'):
-    """Will decompress a XRIT data file and return the path to the decompressed file.
-    It expect to find Eumetsat's xRITDecompress through the environment variable XRIT_DECOMPRESS_PATH 
+    """Will decompress an XRIT data file and return the path to the
+    decompressed file. It expect to find Eumetsat's xRITDecompress through the
+    environment variable XRIT_DECOMPRESS_PATH
     """
     from subprocess import Popen, PIPE
     cmd = os.environ.get('XRIT_DECOMPRESS_PATH', None)
     if not cmd:
         raise IOError("XRIT_DECOMPRESS_PATH is not defined" +
                       " (complete path to xRITDecompress)")
-
+
+    infile = os.path.abspath(infile)
     cwd = os.getcwd()
     os.chdir(outdir)
 
@@ -362,7 +364,7 @@ def read_mpef(file_name):
     if s.file_type == 144:
         return s
     else:
-        raise mipp.DecodeError("this is no 'MPEF (type=124)' file: '%s'"%file_name)
+        raise mipp.DecodeError("this is no 'MPEF (type=144)' file: '%s'"%file_name)
 
 # Backward compatible
 read_mpef_clm = read_mpef