mig to nosetests // travis prep

.gitignore

@@ -8,3 +8,4 @@ gh-pages
 .coverage
 *swp
 test/data
+htmlcov/

.travis.yml

@@ -4,12 +4,15 @@ env:
     global:
         - OMP_NUM_THREADS=4
 
+cache:
+  pip: true
+  directories: $TRAVIS_BUILD_DIR/kite/test/data
+
 matrix:
     include:
         - os: linux
           sudo: required
           python: 2.7
-          cache: pip
           virtualenv:
               system_site_packages: true
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
-description-file = README.md
-license-file = LICENSE.md
+description-file=README.md
+license-file=LICENSE.md
 
 [bdist_wheel]
 universal=1
@@ -25,3 +25,7 @@ exclude_lines =
     raise
     if __name__ == .__main__.:
     logger.error
+omit =
+    */spool/*
+    */talpa/*
+    */qt_utils.py

src/covariance.py

@@ -239,7 +239,7 @@ def noise_data(self):
         if self._noise_data is not None:
             return self._noise_data
         elif self.config.noise_coord is not None:
-            self._log.info('Selecting noise_data from config...')
+            self._log.debug('Selecting noise_data from config...')
             llE, llN = self.scene.frame.mapENMatrix(
                 *self.config.noise_coord[:2])
             sE, sN = self.scene.frame.mapENMatrix(
@@ -248,7 +248,7 @@ def noise_data(self):
             slice_N = slice(llN, llN + sN)
             self.noise_data = self.scene.displacement[slice_N, slice_E]
         else:
-            self._log.info('Selecting noise_data from Quadtree...')
+            self._log.debug('Selecting noise_data from Quadtree...')
             node = self.selectNoiseNode()
             self.noise_data = node.displacement
             self.noise_coord = [node.llE, node.llN,

src/model_scene.py

@@ -98,15 +98,15 @@ def max_horizontal_displacement(self):
     def addSource(self, source):
         if source not in self.sources:
             self.sources.append(source)
-        self._log.info('%s added' % source.__class__.__name__)
+        self._log.debug('%s added' % source.__class__.__name__)
         source.evParametersChanged.subscribe(self._clearModel)
 
         self._clearModel()
 
     def removeSource(self, source):
         source.evParametersChanged.unsubscribe(self._clearModel)
         self.sources.remove(source)
-        self._log.info('%s removed' % source.__class__.__name__)
+        self._log.debug('%s removed' % source.__class__.__name__)
         del source
 
         self._clearModel()
@@ -189,7 +189,7 @@ def getKiteScene(self):
         :rtype: :class:`Scene`
         '''
         from .scene import Scene, SceneConfig
-        self._log.info('Creating kite.Scene from ModelScene')
+        self._log.debug('Creating kite.Scene from ModelScene')
 
         config = SceneConfig()
         config.frame = self.frame.config
@@ -211,7 +211,7 @@ def _clearModel(self):
     def save(self, filename):
         _file, ext = op.splitext(filename)
         filename = filename if ext in ['.yml'] else filename + '.yml'
-        self._log.info('Saving model scene to %s' % filename)
+        self._log.debug('Saving model scene to %s' % filename)
         for source in self.sources:
             source.regularize()
         self.config.dump(filename='%s' % filename,
@@ -221,7 +221,7 @@ def save(self, filename):
     def load(cls, filename):
         config = guts.load(filename=filename)
         model_scene = cls(config=config)
-        model_scene._log.info('Loading config from %s' % filename)
+        model_scene._log.debug('Loading config from %s' % filename)
         for source in model_scene.sources:
             model_scene.addSource(source)
         return model_scene

src/scene.py

@@ -633,7 +633,7 @@ def save(self, filename=None):
         filename = _file if ext in ['.yml', '.npz'] else filename
 
         components = ['displacement', 'theta', 'phi']
-        self._log.info('Saving scene data to %s.npz' % filename)
+        self._log.debug('Saving scene data to %s.npz' % filename)
 
         num.savez('%s.npz' % (filename),
                   *[getattr(self, arr) for arr in components])
@@ -642,7 +642,7 @@ def save(self, filename=None):
     def saveConfig(self, filename):
         _file, ext = op.splitext(filename)
         filename = filename if ext in ['.yml'] else filename + '.yml'
-        self._log.info('Saving scene config to %s' % filename)
+        self._log.debug('Saving scene config to %s' % filename)
         self.config.dump(filename='%s' % filename,
                          header='kite.Scene YAML Config')
 
@@ -660,7 +660,7 @@ def _load(self, filename):
         components = ['displacement', 'theta', 'phi']
 
         basename = op.splitext(filename)[0]
-        scene._log.info('Loading from %s[.npz,.yml]' % basename)
+        scene._log.debug('Loading from %s[.npz,.yml]' % basename)
         try:
             data = num.load('%s.npz' % basename)
             for i, comp in enumerate(components):
@@ -680,7 +680,7 @@ def _load(self, filename):
     load = staticmethod(_load)
 
     def load_config(self, filename):
-        self._log.info('Loading config from %s' % filename)
+        self._log.debug('Loading config from %s' % filename)
         self.config = guts.load(filename=filename)
         self.meta = self.config.meta
 
@@ -706,7 +706,7 @@ def _import_data(self, path, **kwargs):
         for mod in scene_io.__all__:
             module = eval('scene_io.%s(scene)' % mod)
             if module.validate(path, **kwargs):
-                scene._log.info('Importing %s using %s module' %
+                scene._log.debug('Importing %s using %s module' %
                                 (path, mod))
                 data = module.read(path, **kwargs)
                 break

src/scene_io.py

@@ -144,8 +144,9 @@ def read(self, filename, **kwargs):
         if utm_zone is None:
             utm_zone = 33
             utm_zone_letter = 'N'
-            self._log.warning('Defaulting to UTM Zone %d%s' %
-                              (utm_zone, utm_zone_letter))
+            self._log.warning(
+                'utm_zone not defined. Defaulting to UTM Zone %d%s!' %
+                (utm_zone, utm_zone_letter))
         if not (num.all(utm_e) or num.all(utm_n)):
             self._log.warning(
                 'Could not find referencing UTM vectors in .mat file')
@@ -264,7 +265,7 @@ def _getAngle(self, filename, pattern):
             return 0.
 
         filename = phi_files[0]
-        self._log.info('Found %s in %s' % (pattern, filename))
+        self._log.debug('Found %s in %s' % (pattern, filename))
         return num.memmap(filename, mode='r', dtype='>f4')
 
     def read(self, filename, **kwargs):
@@ -314,7 +315,7 @@ def read(self, filename, **kwargs):
         c = self.container
 
         if radar_frequency is not None:
-            self._log.info('Scaling radian displacement by radar_frequency')
+            self._log.debug('Scaling radian displacement by radar_frequency')
             wavelength = 299792458. / radar_frequency
             displ = (displ / (4.*num.pi)) * wavelength
             c['meta']['wavelength'] = wavelength
@@ -328,7 +329,7 @@ def read(self, filename, **kwargs):
         c['par_file'] = par_file
 
         if par['DEM_projection'] == 'UTM':
-            self._log.info('Parameter file provides UTM reference')
+            self._log.debug('Parameter file provides UTM reference')
             import utm
             c['displacement'] = num.transpose(displ/100)
             c['theta'] = num.transpose(theta)
@@ -365,7 +366,7 @@ def read(self, filename, **kwargs):
             c['frame']['dLon'] =\
                 (urlon - c['frame']['llLon']) / displ.shape[1]
         else:
-            self._log.info('Parameter file provides Lat/Lon reference')
+            self._log.debug('Parameter file provides Lat/Lon reference')
             c['frame']['llLat'] = par['corner_lat'] + par['post_lat'] * nrows
             c['frame']['llLon'] = par['corner_lon']
             c['frame']['dLon'] = par['post_lon']
@@ -454,7 +455,8 @@ def read(self, filename, **kwargs):
         look_ref2 = par['LOOK_REF2']
         look_ref3 = par['LOOK_REF3']
         look_ref4 = par['LOOK_REF4']
-        look = num.mean(num.array([look_ref1,look_ref2,look_ref3,look_ref4]))
+        look = num.mean(
+            num.array([look_ref1, look_ref2, look_ref3, look_ref4]))
 
         data = num.memmap(filename, dtype='<f4')
         data = data.reshape(nlines, nrows*2)

test/common.py

@@ -7,11 +7,8 @@
 
 op = os.path
 
-data_dir = op.join(
-    op.dirname(op.abspath(__file__)),
-    'data')
-
-data_uri = 'http://data.pyrocko.org/testing/'
+data_uri = 'http://data.pyrocko.org/testing/kite/'
+data_dir = op.join(op.dirname(op.abspath(__file__)), 'data')
 
 logger = logging.getLogger('kite.testing')
 
@@ -33,20 +30,21 @@ def _dir_content(url):
         return files
 
     def _file_size(url):
-        r = requests.head(url)
+        r = requests.head(url, headers={'Accept-Encoding': 'identity'})
         r.raise_for_status()
+        print r.headers
         return int(r.headers['content-length'])
 
     def _download_file(url, fn_local):
-        print fn_local
+        print url, fn_local
         if op.exists(fn_local):
             if os.stat(fn_local).st_size == _file_size(url):
                 return fn_local
 
         logger.info('Downloading %s...' % url)
         fsize = _file_size(url)
 
-        r = requests.get(url)
+        r = requests.get(url, stream=True)
         r.raise_for_status()
 
         dl_bytes = 0
@@ -56,6 +54,7 @@ def _download_file(url, fn_local):
                 f.write(d)
 
         if dl_bytes != fsize:
+            print url, dl_bytes, fsize
             raise DownloadError('Download incomplete!')
         logger.info('Download completed!')
         return fn_local
@@ -112,3 +111,10 @@ def __str__(self):
         if len(self.results) == 0:
             rstr.append('None ran!')
         return '\n'.join(rstr)
+
+
+def setLogLevel(level):
+    print('SETTING LOGLEVEL!')
+    level = getattr(logging, level, 'DEBUG')
+    logging.basicConfig(
+        level=level)

test/test_covariance.py

@@ -3,18 +3,18 @@
 import numpy as num
 import matplotlib.pyplot as plt
 
-from kite import Scene, TestScene
+from kite import Scene
 from . import common
 
 benchmark = common.Benchmark()
+common.setLogLevel('DEBUG')
 
 
 class TestCovariance(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        file = common.get_test_data(
-            '20110214_20110401_ml4_sm.unw.geo_ig_dsc_ionnocorr.mat')
+        file = common.get_test_data('myanmar_alos_dsc_ionocorr.mat')
         cls.sc = Scene.import_data(file)
 
     def test_covariance(self):
@@ -32,26 +32,27 @@ def test_covariance(self):
                 num.testing.assert_allclose(c1, c2,
                                             rtol=200, atol=2e3, verbose=True)
 
+    def test_synthetic_noise(self):
+        self.sc.covariance.syntheticNoise()
+        self.sc.covariance.variance
+
     @benchmark
-    @unittest.skip('Skip!')
-    def testCovariancParallel(self):
+    def test_covariance_parallel(self):
         cov = self.sc.covariance
-        cov._calcCovarianceMatrix(method='full', nthreads=12)
+        cov._calcCovarianceMatrix(method='full', nthreads=0)
 
     @benchmark
-    @unittest.skip('Skip!')
-    def testCovariancSingle(self):
+    def _test_covariance_single_thread(self):
         cov = self.sc.covariance
         cov._calcCovarianceMatrix(method='full', nthreads=1)
 
     @benchmark
-    @unittest.skip('Skip!')
-    def testCovariancFocal(self):
+    def test_covariance_focal(self):
         cov = self.sc.covariance
         cov._calcCovarianceMatrix(method='focal')
 
     @unittest.skip('Skip!')
-    def testCovarianceVisual(self):
+    def _test_covariance_visual(self):
         cov = self.sc.covariance
         cov.epsilon = .02
         cov.subsampling = 10

test/test_io.py

@@ -3,20 +3,25 @@
 import unittest
 import tempfile
 import shutil
-import os
 import numpy as num
+from os import path as op
 
 from . import common
 from kite import Scene
 
+
+# format (dl_dir, load_file)
 filenames = {
-    'matlab': 'myanmar_alos_dsc_ionocorr.mat',
-    'gmtsar': 'gmtsar/',
-    # 'roi_pac': None,
-    # 'gamma': None,
-    # 'isce': None,
+    'matlab': ('myanmar_alos_dsc_ionocorr.mat', None),
+    'gmtsar': ('gmtsar/', 'gmtsar/unwrap_ll.los_ll.grd'),
+    'roi_pac': ('roi_pac/',
+                'roi_pac/geo_20160113-20160206_atmo_2rlks_c10_cut.unw'),
+    'gamma': ('gamma/', 'gamma/asc'),
+    'isce': ('isce/', 'isce/filt_topophase.unw.geo')
 }
 
+common.setLogLevel('DEBUG')
+
 
 class SceneIOTest(unittest.TestCase):
 
@@ -29,15 +34,20 @@ def tearDownClass(cls):
         shutil.rmtree(cls.tmp_dir)
 
 
-def _create_test_func(fmt, filename):
+def _create_test_func(fmt, dl_path, filename):
+
     def f(self):
-        fn = common.get_test_data(filename)
-        if isinstance(fn, list):
-            fn = fn[-1]
+        common.get_test_data(dl_path)
+
+        if filename is None:
+            load_path = dl_path
+        else:
+            load_path = filename
+        load_path = op.join(common.data_dir, load_path)
 
-        fn_save = os.path.join(self.tmp_dir, 'kite-%s' % fmt)
+        fn_save = op.join(self.tmp_dir, 'kite-%s' % fmt)
 
-        sc1 = Scene.import_data(fn)
+        sc1 = Scene.import_data(load_path)
         sc1.save(fn_save)
 
         sc2 = Scene.load(fn_save)
@@ -51,8 +61,8 @@ def f(self):
     return f
 
 
-for fmt, filename in filenames.iteritems():
+for fmt, fns in filenames.iteritems():
     setattr(
         SceneIOTest,
         'test_import_%s' % fmt,
-        _create_test_func(fmt, filename))
+        _create_test_func(fmt, *fns))