Merge branch 'master' of https://github.com/legacysurvey/legacypipe

py/legacyanalysis/create_testcase.py

@@ -57,6 +57,9 @@ def main():
         outccds.delete_column(c)
     outccds.image_hdu[:] = 1
 
+    # Convert to list to avoid truncating filenames
+    outccds.image_filename = [fn for fn in outccds.image_filename]
+
     for iccd,ccd in enumerate(C):
 
         assert(ccd.camera.strip() == 'decam')
@@ -84,6 +87,22 @@ def main():
         crpix2 = tim.hdr['CRPIX2']
         tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
         tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
+
+        # Add image extension to filename
+        outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
+        outim.wtfn  = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname)
+        outim.dqfn  = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname)
+        # fitsio doesn't compress .fz by default
+        outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
+        outim.wtfn  = outim.wtfn .replace('.fits.fz', '.fits')
+        outim.dqfn  = outim.dqfn .replace('.fits.fz', '.fits')
+
+        outccds.image_filename[iccd] = outim.imgfn
+
+        print('Changed output filenames to:')
+        print(outim.imgfn)
+        print(outim.wtfn)
+        print(outim.dqfn)
 
         fitsio.write(outim.imgfn, None, header=tim.primhdr, clobber=True)
         fitsio.write(outim.imgfn, tim.getImage(), header=tim.hdr,
@@ -113,7 +132,6 @@ def main():
         fitsio.write(outim.dqfn, tim.dq, header=tim.hdr,
                      extname=ccd.ccdname)
 
-
         print('PSF filename:', outim.psffn)
         trymakedirs(outim.psffn, dir=True)
         psfex.writeto(outim.psffn)

py/legacypipe/common.py

@@ -67,7 +67,7 @@ class LegacyEllipseWithPriors(EllipseWithPriors):
 from tractor.ellipses import EllipseE
 class SimpleGalaxy(ExpGalaxy):
     shape = EllipseE(0.45, 0., 0.)
-    
+
     def __init__(self, *args):
         super(SimpleGalaxy, self).__init__(*args)
         self.shape = SimpleGalaxy.shape

py/legacypipe/merge-zeropoints.py

@@ -4,7 +4,99 @@
 import os
 from astrometry.util.fits import fits_table, merge_tables
 
+def decals_dr3_dedup():
+    SN = fits_table('survey-ccds-nondecals.fits.gz')
+    SD = fits_table('survey-ccds-decals.fits.gz')
+    sne = np.unique(SN.expnum)
+    sde = np.unique(SD.expnum)
+    isec = set(sne).intersection(sde)
+    print(len(isec), 'exposures in common between "decals" and "nondecals"')
+    # These are two versions of CP reductions in CPDES82 and CPHETDEX.
+    I = np.flatnonzero(np.array([e in isec for e in SD.expnum]))
+    print(len(I), 'rows in "decals"')
 
+    I = np.flatnonzero(np.array([e not in isec for e in SD.expnum]))
+    SD.cut(I)
+    print(len(SD), 'rows remaining')
+
+    # Now, also move "decals" filenames containing "CPDES82" to "nondecals".
+    I = np.flatnonzero(np.array(['CPDES82' in fn for fn in SD.image_filename]))
+    keep = np.ones(len(SD), bool)
+    keep[I] = False
+
+    print('Merging:')
+    SN.about()
+    SD[I].about()
+
+    SN2 = merge_tables((SN, SD[I]), columns='fillzero')
+    SD2 = SD[keep]
+
+    print('Moved CPDES82: now', len(SN2), 'non-decals and', len(SD2), 'decals')
+
+    SN2.writeto('survey-ccds-nondecals.fits')
+    SD2.writeto('survey-ccds-decals.fits')
+
+    SE = fits_table('survey-ccds-extra.fits.gz')
+    SN = fits_table('survey-ccds-nondecals.fits')
+    SD = fits_table('survey-ccds-decals.fits')
+
+    sne = np.unique(SN.expnum)
+    sde = np.unique(SD.expnum)
+    see = np.unique(SE.expnum)
+
+    i1 = set(sne).intersection(sde)
+    i2 = set(sne).intersection(see)
+    i3 = set(sde).intersection(see)
+    print('Intersections:', len(i1), len(i2), len(i3))
+
+
+
+def decals_dr3_extra():
+    # /global/homes/a/arjundey/ZeroPoints/decals-zpt-dr3-all.fits
+    T = fits_table('/global/cscratch1/sd/desiproc/zeropoints/decals-zpt-dr3-all.fits')
+
+    S1 = fits_table('survey-ccds-nondecals.fits.gz')
+    S2 = fits_table('survey-ccds-decals.fits.gz')
+    S = merge_tables([S1,S2])
+
+    gotchips = set(zip(S.expnum, S.ccdname))
+
+    got = np.array([(e,c) in gotchips for e,c in zip(T.expnum, T.ccdname)])
+    print('Found', sum(got), 'of', len(T), 'dr3-all CCDs in existing surveys tables of size', len(S))
+
+    I = np.flatnonzero(np.logical_not(got))
+    T.cut(I)
+    print(len(T), 'remaining')
+    #print('Directories:', np.unique([os.path.basename(os.path.dirname(fn.strip())) for fn in T.filename]))
+    print('Filenames:', np.unique(T.filename))
+
+    T.writeto('extras.fits')
+
+    basedir = os.environ['LEGACY_SURVEY_DIR']
+    cam = 'decam'
+    image_basedir = os.path.join(basedir, 'images')
+    TT = []
+
+    for fn,dirnms in [
+        ('extras.fits',
+         ['CP20140810_?_v2',
+          'CP20141227', 'CP20150108', 'CP20150326',
+          'CP20150407', 'CP20151010', 'CP20151028', 'CP20151126',
+          'CP20151226', 'CP20160107', 'CP20160225',
+          'COSMOS', 'CPDES82',
+          'NonDECaLS/*',
+         ]),
+        ]:
+        T = normalize_zeropoints(fn, dirnms, image_basedir, cam)
+        TT.append(T)
+    T = merge_tables(TT)
+    outfn = 'survey-ccds-extra.fits'
+    T.writeto(outfn)
+    print('Wrote', outfn)
+
+
+
+
 def decals_dr3():
     basedir = os.environ['LEGACY_SURVEY_DIR']
     cam = 'decam'
@@ -30,6 +122,16 @@ def decals_dr3():
     T.writeto(outfn)
     print('Wrote', outfn)
 
+    nd = np.array(['NonDECaLS' in fn for fn in T.image_filename])
+    I = np.flatnonzero(nd)
+    T[I].writeto('survey-ccds-nondecals.fits')
+
+    I = np.flatnonzero(np.logical_not(nd))
+    T[I].writeto('survey-ccds-decals.fits')
+
+    for fn in ['survey-ccds-nondecals.fits', 'survey-ccds-decals.fits']:
+        os.system('gzip --best ' + fn)
+
 
 def normalize_zeropoints(fn, dirnms, image_basedir, cam, T=None):
     if T is None:
@@ -148,7 +250,9 @@ def normalize_zeropoints(fn, dirnms, image_basedir, cam, T=None):
 if __name__ == '__main__':
     import sys
 
-    decals_dr3()
+    #decals_dr3()
+    #decals_dr3_extra()
+    decals_dr3_dedup()
     sys.exit(0)
 
     # Mosaicz tests