Add parquet sourceTable_visit catalogs to hsc repo

.gitattributes

@@ -1,3 +1,4 @@
 *.fits filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.fz filter=lfs diff=lfs merge=lfs -text
+*.parq filter=lfs diff=lfs merge=lfs -text

README.md

@@ -65,8 +65,9 @@ config.skyMap.pixelScale=0.263
 hsc
 ---
 
-Source catalogs, metadata, and zeroed+compressed images taken from the `w_2020_14`processing run of the HSC RC dataset, available at `lsst-dev:/datasets/hsc/repo/rerun/RC/w_2020_14/DM-24359-sfm`. 
+Source catalogs, metadata, and zeroed+compressed images taken from the `w_2020_14`processing run of the HSC RC dataset, available at `lsst-dev:/datasets/hsc/repo/rerun/RC/w_2020_14/DM-24359-sfm`.
 The included `scripts/compress_jointcal_hsc_test_data.py` file copies the necessary data from the butler repo, removes pixel-level data from the calexps and compresses the source catalogs, and extracts the relevant portions of the sqlite3 registry.
+Finally, the src catalogs have been converted to sourceTable_visit parquet tables for quick ingest.  The included `scripts/make_sourcetables_jointcal_hsc_test_data.sh` file runs the appropriate transformation tasks and cleans up temporary files.
 
 Git LFS
 -------

hsc/config/transformSourceTable.py

@@ -0,0 +1,5 @@
+import lsst.pipe.tasks.postprocess
+assert type(config)==lsst.pipe.tasks.postprocess.TransformSourceTableConfig, 'config is of type %s.%s instead of lsst.pipe.tasks.postprocess.TransformSourceTableConfig' % (type(config).__module__, type(config).__name__)
+# Path to YAML file specifying functors to be computed
+config.functorFile='./scripts/policy/Source_hsc.yaml'
+

hsc/config/writeSourceTable.py

@@ -0,0 +1,8 @@
+import lsst.pipe.tasks.postprocess
+assert type(config)==lsst.pipe.tasks.postprocess.WriteSourceTableConfig, 'config is of type %s.%s instead of lsst.pipe.tasks.postprocess.WriteSourceTableConfig' % (type(config).__module__, type(config).__name__)
+# Add local photoCalib columns from the calexp.photoCalib? Should only set True if generating Source Tables from older src tables which do not already have local calib columns
+config.doApplyExternalPhotoCalib=True
+
+# Add local WCS columns from the calexp.wcs? Should only set True if generating Source Tables from older src tables which do not already have local calib columns
+config.doApplyExternalSkyWcs=True
+

scripts/make_sourcetables_jointcal_hsc_test_data.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env sh
+
+# Convert all source catalogs in the hsc/ repo to sourceTable_visit using
+# default conversions.
+# Should be run from the testdata_jointcal base path
+# Note that --no-versions is specified because of the different version of the stack
+# used to generate the source tables (w_2020_22) vs the original data (w_2020_14).
+# This is necessary because we need obs_subaru from w_2020_18 or later
+# (including DM-24379) in order to do the sourceTable conversions, and from
+# w_2020_21 or later (including DM-25020) which includes the correct local background
+# measurement.  Finally, the default Source.yaml file needs to be modified to remove
+# the sky_source column which did not exist for the w_2020_14 RC2 reprocessing
+
+writeSourceTable.py ./hsc --output ./hsc --id filter=HSC-R^HSC-I --config doApplyExternalPhotoCalib=True --config doApplyExternalSkyWcs=True --no-versions
+
+transformSourceTable.py ./hsc --output ./hsc --id filter=HSC-R^HSC-I --no-versions --config functorFile=./scripts/policy/Source_hsc.yaml
+
+consolidateSourceTable.py ./hsc --output ./hsc --id visit=34648^34670^34674^34690^34714^35892^36140^36192^36236^36260 --no-versions
+
+# Finally, clean out SRC and sourceTable per-ccd parquet files that we do not need
+
+rm hsc/0????/HSC-?/output/*-???.parq

scripts/policy/Source_hsc.yaml

@@ -0,0 +1,271 @@
+funcs:
+    sourceId: # the index of deepCoadd_obj IS the objectId
+        functor: Index
+    coord_ra:
+        # reference position required by db. Not in DPDD
+        functor: CoordColumn
+        args: coord_ra
+    coord_dec:
+        # Reference position required by db. Not in DPDD
+        functor: CoordColumn
+        args: coord_dec
+    ccdVisitId:
+        functor: Column
+        args: ccdVisitId
+    # objectId: not avaliable
+    # ssObjectId: not avaliable
+    parentSourceId:
+        functor: Column
+        args: parent
+    #htmId20: not avaliable
+    x:
+        functor: Column
+        args: slot_Centroid_x
+    y:
+        functor: Column
+        args: slot_Centroid_y
+    xErr:
+        functor: Column
+        args: slot_Centroid_xErr
+    yErr:
+        functor: Column
+        args: slot_Centroid_yErr
+    # x_y_Cov: not available
+    ra:
+        functor: RAColumn
+    # raErr: not available yet DM-15180
+    decl:
+        functor: DecColumn
+    # declErr: not available yet DM-15180
+    # ra_decl_Cov: not available yet
+    # One calibrated Calib flux is important:
+    ApFlux:
+        functor: LocalNanojansky
+        args:
+            - slot_CalibFlux_instFlux
+            - slot_CalibFlux_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+    ApFluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - slot_CalibFlux_instFlux
+            - slot_CalibFlux_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+    # TODO: When DM-25019 is complete, these should be
+    # changed to use the local value of the background
+    # model, rather than the residual of the background
+    sky:
+        functor: LocalNanojansky
+        args:
+            - base_LocalBackground_instFlux
+            - base_LocalBackground_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+    skyErr:
+        functor: LocalNanojanskyErr
+        args:
+            - base_LocalBackground_instFlux
+            - base_LocalBackground_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+    PsFlux:
+        functor: LocalNanojansky
+        args:
+            - slot_PsfFlux_instFlux
+            - slot_PsfFlux_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+    PsFluxErr:
+        functor: LocalNanojanskyErr
+        args:
+            - slot_PsfFlux_instFlux
+            - slot_PsfFlux_instFluxErr
+            - base_LocalPhotoCalib
+            - base_LocalPhotoCalibErr
+
+    # These PS columns makes sense anymore as named:
+    # psX
+    # psXSigma
+    # psY
+    # psYSigma
+    # psFlux_psX_Cov
+    # psFlux_psY_Cov
+    # psX_Y_Cov
+    # psLnL
+    # psChi2
+    # psN
+    # psRa
+    # psRaSigma
+    # psDecl
+    # psDeclSigma
+    # psFlux_psRa_Cov
+    # psFlux_psDecl_Cov
+
+
+    Ixx:
+        functor: Column
+        args: slot_Shape_xx
+    Iyy:
+        functor: Column
+        args: slot_Shape_yy
+    Ixy:
+        functor: Column
+        args: slot_Shape_xy
+    # DPDD should include Psf Shape
+    IxxPsf:
+        functor: Column
+        args: slot_PsfShape_xx
+    IyyPsf:
+        functor: Column
+        args: slot_PsfShape_yy
+    IxyPsf:
+        functor: Column
+        args: slot_PsfShape_xy
+    # apNann: Replaced by raw Aperture instFluxes in flags section below
+    # apMeanSb:  Replaced by raw Aperture instFluxes in flags section below
+    # apMeanSbErr:  Replaced by raw Aperture instFluxes in flags section below
+    extendedness:
+        functor: Column
+        args: base_ClassificationExtendedness_value
+flags:
+   - base_LocalPhotoCalib
+   - base_LocalPhotoCalib_flag
+   - base_LocalPhotoCalibErr
+   - base_LocalWcs_flag
+   - base_LocalWcs_CDMatrix_2_1
+   - base_LocalWcs_CDMatrix_1_1
+   - base_LocalWcs_CDMatrix_1_2
+   - base_LocalWcs_CDMatrix_2_2
+   - base_Blendedness_abs
+   - base_Blendedness_flag
+   - base_Blendedness_flag_noCentroid
+   - base_Blendedness_flag_noShape
+   - base_CircularApertureFlux_12_0_flag
+   - base_CircularApertureFlux_12_0_flag_apertureTruncated
+   - base_CircularApertureFlux_12_0_flag_sincCoeffsTruncated
+   - base_CircularApertureFlux_12_0_instFlux
+   - base_CircularApertureFlux_12_0_instFluxErr
+   - base_CircularApertureFlux_17_0_flag
+   - base_CircularApertureFlux_17_0_flag_apertureTruncated
+   - base_CircularApertureFlux_17_0_instFlux
+   - base_CircularApertureFlux_17_0_instFluxErr
+   - base_CircularApertureFlux_25_0_flag
+   - base_CircularApertureFlux_25_0_flag_apertureTruncated
+   - base_CircularApertureFlux_25_0_instFlux
+   - base_CircularApertureFlux_25_0_instFluxErr
+   - base_CircularApertureFlux_35_0_flag
+   - base_CircularApertureFlux_35_0_flag_apertureTruncated
+   - base_CircularApertureFlux_35_0_instFlux
+   - base_CircularApertureFlux_35_0_instFluxErr
+   - base_CircularApertureFlux_3_0_flag
+   - base_CircularApertureFlux_3_0_flag_apertureTruncated
+   - base_CircularApertureFlux_3_0_flag_sincCoeffsTruncated
+   - base_CircularApertureFlux_3_0_instFlux
+   - base_CircularApertureFlux_3_0_instFluxErr
+   - base_CircularApertureFlux_4_5_flag
+   - base_CircularApertureFlux_4_5_flag_apertureTruncated
+   - base_CircularApertureFlux_4_5_flag_sincCoeffsTruncated
+   - base_CircularApertureFlux_4_5_instFlux
+   - base_CircularApertureFlux_4_5_instFluxErr
+   - base_CircularApertureFlux_50_0_flag
+   - base_CircularApertureFlux_50_0_flag_apertureTruncated
+   - base_CircularApertureFlux_50_0_instFlux
+   - base_CircularApertureFlux_50_0_instFluxErr
+   - base_CircularApertureFlux_6_0_flag
+   - base_CircularApertureFlux_6_0_flag_apertureTruncated
+   - base_CircularApertureFlux_6_0_flag_sincCoeffsTruncated
+   - base_CircularApertureFlux_6_0_instFlux
+   - base_CircularApertureFlux_6_0_instFluxErr
+   - base_CircularApertureFlux_70_0_flag
+   - base_CircularApertureFlux_70_0_flag_apertureTruncated
+   - base_CircularApertureFlux_70_0_instFlux
+   - base_CircularApertureFlux_70_0_instFluxErr
+   - base_CircularApertureFlux_9_0_flag
+   - base_CircularApertureFlux_9_0_flag_apertureTruncated
+   - base_CircularApertureFlux_9_0_flag_sincCoeffsTruncated
+   - base_CircularApertureFlux_9_0_instFlux
+   - base_CircularApertureFlux_9_0_instFluxErr
+   - base_ClassificationExtendedness_flag
+   - base_FootprintArea_value
+   - base_Jacobian_flag
+   - base_Jacobian_value
+   - base_LocalBackground_instFlux
+   - base_LocalBackground_instFluxErr
+   - base_LocalBackground_flag
+   - base_LocalBackground_flag_noGoodPixels
+   - base_LocalBackground_flag_noPsf
+   - base_PixelFlags_flag
+   - base_PixelFlags_flag_bad
+   - base_PixelFlags_flag_cr
+   - base_PixelFlags_flag_crCenter
+   - base_PixelFlags_flag_edge
+   - base_PixelFlags_flag_interpolated
+   - base_PixelFlags_flag_interpolatedCenter
+   - base_PixelFlags_flag_offimage
+   - base_PixelFlags_flag_saturated
+   - base_PixelFlags_flag_saturatedCenter
+   - base_PixelFlags_flag_suspect
+   - base_PixelFlags_flag_suspectCenter
+   - base_PsfFlux_apCorr
+   - base_PsfFlux_apCorrErr
+   - base_PsfFlux_area
+   - base_PsfFlux_flag
+   - base_PsfFlux_flag_apCorr
+   - base_PsfFlux_flag_edge
+   - base_PsfFlux_flag_noGoodPixels
+   - base_SdssCentroid_flag
+   - base_SdssCentroid_flag_almostNoSecondDerivative
+   - base_SdssCentroid_flag_badError
+   - base_SdssCentroid_flag_edge
+   - base_SdssCentroid_flag_noSecondDerivative
+   - base_SdssCentroid_flag_notAtMaximum
+   - base_SdssCentroid_flag_resetToPeak
+   - base_Variance_flag
+   - base_Variance_flag_emptyFootprint
+   - base_Variance_value
+   - calib_astrometry_used
+   - calib_detected
+   - calib_photometry_reserved
+   - calib_photometry_used
+   - calib_psf_candidate
+   - calib_psf_reserved
+   - calib_psf_used
+   - deblend_deblendedAsPsf
+   - deblend_hasStrayFlux
+   - deblend_masked
+   - deblend_nChild
+   - deblend_parentTooBig
+   - deblend_patchedTemplate
+   - deblend_rampedTemplate
+   - deblend_skipped
+   - deblend_tooManyPeaks
+   - ext_shapeHSM_HsmPsfMoments_flag
+   - ext_shapeHSM_HsmPsfMoments_flag_no_pixels
+   - ext_shapeHSM_HsmPsfMoments_flag_not_contained
+   - ext_shapeHSM_HsmPsfMoments_flag_parent_source
+   - ext_shapeHSM_HsmShapeRegauss_flag
+   - ext_shapeHSM_HsmShapeRegauss_flag_galsim
+   - ext_shapeHSM_HsmShapeRegauss_flag_no_pixels
+   - ext_shapeHSM_HsmShapeRegauss_flag_not_contained
+   - ext_shapeHSM_HsmShapeRegauss_flag_parent_source
+
+flag_rename_rules:
+    # Taken from db-meas-forced
+    - ['ext_photometryKron_', '']
+    - ['base_PixelFlags_flag', 'PixelFlags']
+    - ['base_Classification', '']
+    - ['base_Sdss', '']
+    - ['base_CircularApertureFlux', 'ApFlux']
+    - ['ext_shapeHSM_', '']
+    - ['ext_convolved_', '']
+    - ['undeblended_base', 'undeblended']
+    - ['undeblended_ext_photometryKron', 'undeblended']
+    - ['ext_photometryKron_', '']
+    - ['base_', '']
+    - ['slot_', '']
+    - ['calib_', 'Calib_']
+    - ['deblend_', 'Deblend_']
+    - ['blendedness', 'Blendedness']
+