update time request handling

PlotDASC.py

@@ -4,7 +4,7 @@
 
 This program by default projects HiST auroral tomography system FOV onto PFRR DASC.
 """
-import dasciutils.io as dui
+import dasciutils as du
 from dascutils.plots import histdasc,moviedasc
 
 def plotdasc(img,wavelength,odir,cadence,rows,cols):
@@ -28,6 +28,6 @@ def plotdasc(img,wavelength,odir,cadence,rows,cols):
 
 
 
-    img = dui.load(p.indir, p.azfn, p.elfn, p.wavelength, p.minmax, p.tlim)
+    img = du.load(p.indir, p.azfn, p.elfn, p.wavelength, p.minmax, p.tlim)
 
     plotdasc(img, wwl, p.odir, p.cadence, None, None)

PlotDASC_HiST.py

@@ -4,7 +4,7 @@
 
 This program by default projects HiST auroral tomography system FOV onto PFRR DASC.
 """
-from dascutils.io import readallDasc
+from dascutils import load
 from dascutils.plots import histdasc,moviedasc
 #
 from themisasi.fov import mergefov
@@ -35,7 +35,7 @@ def plothstfovondasc(imgs,odir,cadence,rows,cols):
     try:
         plothstfovondasc(img,p.wavelength,p.odir,p.cadence,rows,cols)
     except NameError:
-        imgs = readallDasc(p.indir,p.azfn,p.elfn,p.wavelength,p.minmax,p.tlim)
+        imgs = load(p.indir,p.azfn,p.elfn,p.wavelength,p.minmax,p.tlim)
         rows,cols = mergefov(ocalfn,wlla,waz,wel,None,None,p.ncal,p.projalt,site='DASC')
 
         plothstfovondasc(img,p.wavelength,p.odir,p.cadence,rows,cols)

dascutils/__init__.py

@@ -4,6 +4,8 @@
 from dateutil.parser import parse
 from datetime import datetime
 from urllib.parse import urlparse
+#
+from .io import load # noqa: F401
 
 
 def totimestamp(t):
@@ -20,7 +22,7 @@ def totimestamp(t):
         t = totimestamp(parse(t))
     elif isinstance(t,(float,int)):
         t = float(t)
-        assert 1e9 < t < 3e9, f'did you really mean {datetime.fromtimestamp(t)}'
+        assert 1e9 < t < 3e9, f'did you  mean {datetime.fromtimestamp(t)}'
     else: # assume it's an iterable 1-D vector
         t = list(map(totimestamp,t))
 

dascutils/io.py

@@ -14,7 +14,7 @@
 import xarray
 
 
-def load(flist:list, azfn:Path=None, elfn:Path=None, minmax:tuple=None, treq:list=None) -> xarray.Dataset:
+def load(flist:list, azfn:Path=None, elfn:Path=None, treq:list=None) -> xarray.Dataset:
     """
     reads FITS images and spatial az/el calibration for allsky camera
     Bdecl is in degrees, from IGRF model
@@ -24,42 +24,47 @@ def load(flist:list, azfn:Path=None, elfn:Path=None, minmax:tuple=None, treq:lis
     if not flist:
         raise FileNotFoundError('no files of this wavelength')
 
-    if isinstance(flist,(str,Path)):
-        flist = [flist]
-
+    flist = np.atleast_1d(flist)
 #%% read one file mode
     if treq is not None:
-        expstart = []
+        if isinstance(treq,str):
+            treq = parse(treq)
+        elif isinstance(treq,(np.ndarray,tuple,list)) and isinstance(treq[0],str):
+            treq = list(map(parse,treq))
+        treq = np.atleast_1d(treq)
 
+        time = []
         for i,fn in enumerate(flist):
             try:
                 with fits.open(fn, mode='readonly') as h:
-                    expstart.append(parse(h[0].header['OBSDATE'] + ' ' + h[0].header['OBSSTART']))
-            except IOError: #many corrupted files, accounted for by preallocated vectors
+                    time.append(parse(h[0].header['OBSDATE'] + 'T' + h[0].header['OBSSTART']))
+            except OSError: #many corrupted files, accounted for by preallocated vectors
                 pass
 
-        expstart = np.array(expstart)
+        time = np.array(time)
 
-        if isinstance(treq,float) or len(treq) == 1: # single frame
-            fi = np.nanargmin(abs(expstart-treq)) #index number in flist desired
-        elif len(treq)==2: #frames within bounds
+        if treq.size == 1: # single frame
+            fi = abs(time-treq).argmin() #index number in flist desired
+        elif treq.size == 2: #frames within bounds
             if treq[0] is not None:
-                fi = (treq[0] <= expstart)
+                fi = (treq[0] <= time)
                 if treq[1] is not None:
-                    fi &= (expstart < treq[1])
+                    fi &= (time < treq[1])
             elif treq[1] is not None:
-                fi = (expstart < treq[1])
+                fi = (time < treq[1])
             else:
                 fi = slice(None)
         else:
             fi = slice(None)
 
-        flist = flist[fi]
-        if len(flist)==0:
-            raise FileNotFoundError('no files found within time limits')
+        if isinstance(fi,np.ndarray) and not fi.any():
+            raise ValueError(f'no valid data found in {treq}')
 
+        flist = flist[fi]
         if isinstance(flist,Path): # so that we can iterate
             flist = [flist]
+        if len(flist)==0:
+            raise FileNotFoundError('no files found within time limits')
 #%% iterate over image files
     time = []; img= [];  wavelen = []
 
@@ -102,9 +107,6 @@ def load(flist:list, azfn:Path=None, elfn:Path=None, minmax:tuple=None, treq:lis
     time = np.array(time)
     wavelen = np.array(wavelen)
     wavelengths = np.unique(wavelen)
-# %% deal with corrupted data
-    if minmax is not None:
-        img[(img<minmax[0]) | (img>minmax[1])] = 1 #instead of 0 for lognorm
 
     ds = {}
     for w in wavelengths:

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-install_requires = ['python-dateutil','pytz','numpy','astropy']
+install_requires = ['python-dateutil','pytz','numpy','astropy','xarray']
 tests_require=['pytest','nose','coveralls']
 # %%
 from setuptools import setup, find_packages
@@ -10,7 +10,7 @@
       url='https://github.com/scivision/dascutils',
       description='Utilities for UAF Digital All-Sky Camera: reading and plotting',
       long_description=open('README.rst').read(),
-      version = '1.0.0',
+      version = '1.1.0',
       classifiers=[
       'Development Status :: 4 - Beta',
       'Environment :: Console',

tests/test_all.py

@@ -3,10 +3,10 @@
 import numpy as np
 import xarray
 import tempfile
+from datetime import datetime
 from numpy.testing import assert_allclose
 #
 import dascutils as du
-import dascutils.io as dui
 
 R = Path(__file__).parent
 
@@ -26,11 +26,18 @@ def test_timestamp():
                                    (1325579522.0, 1325579532.0))
 
 def test_readdasc():
-    data = dui.load(fn,azfn,elfn)
+    data = du.load(fn,azfn,elfn)
     assert isinstance(data, xarray.Dataset)
     assert 428 in data.data_vars
     assert data[428].shape == (1,512,512)
     assert data.sensorloc == {'lat': 65.126, 'lon': -147.479, 'alt_m': 200.0}
+    assert data.time.values == datetime(2015, 10, 7, 8, 23, 5, 930000)
+
+    data = du.load(fn,azfn,elfn,'2012-01-03T08:32:02')
+    assert data[428].shape == (1,512,512)
+
+    data = du.load(fn,azfn,elfn,('2012-01-03T08:32:02','2016-01-04'))
+    assert data[428].shape == (1,512,512)
 
 
 def test_getdasc():