-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add parquet sourceTable_visit catalogs to hsc repo
- Loading branch information
Showing
16 changed files
with
346 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
*.fits filter=lfs diff=lfs merge=lfs -text | ||
*.gz filter=lfs diff=lfs merge=lfs -text | ||
*.fz filter=lfs diff=lfs merge=lfs -text | ||
*.parq filter=lfs diff=lfs merge=lfs -text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
import lsst.pipe.tasks.postprocess | ||
assert type(config)==lsst.pipe.tasks.postprocess.TransformSourceTableConfig, 'config is of type %s.%s instead of lsst.pipe.tasks.postprocess.TransformSourceTableConfig' % (type(config).__module__, type(config).__name__) | ||
# Path to YAML file specifying functors to be computed | ||
config.functorFile='./scripts/policy/Source_hsc.yaml' | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
import lsst.pipe.tasks.postprocess | ||
assert type(config)==lsst.pipe.tasks.postprocess.WriteSourceTableConfig, 'config is of type %s.%s instead of lsst.pipe.tasks.postprocess.WriteSourceTableConfig' % (type(config).__module__, type(config).__name__) | ||
# Add local photoCalib columns from the calexp.photoCalib? Should only set True if generating Source Tables from older src tables which do not already have local calib columns | ||
config.doApplyExternalPhotoCalib=True | ||
|
||
# Add local WCS columns from the calexp.wcs? Should only set True if generating Source Tables from older src tables which do not already have local calib columns | ||
config.doApplyExternalSkyWcs=True | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#!/usr/bin/env sh | ||
|
||
# Convert all source catalogs in the hsc/ repo to sourceTable_visit using | ||
# default conversions. | ||
# Should be run from the testdata_jointcal base path | ||
# Note that --no-versions is specified because of the different version of the stack | ||
# used to generate the source tables (w_2020_22) vs the original data (w_2020_14). | ||
# This is necessary because we need obs_subaru from w_2020_18 or later | ||
# (including DM-24379) in order to do the sourceTable conversions, and from | ||
# w_2020_21 or later (including DM-25020) which includes the correct local background | ||
# measurement. Finally, the default Source.yaml file needs to be modified to remove | ||
# the sky_source column which did not exist for the w_2020_14 RC2 reprocessing | ||
|
||
writeSourceTable.py ./hsc --output ./hsc --id filter=HSC-R^HSC-I --config doApplyExternalPhotoCalib=True --config doApplyExternalSkyWcs=True --no-versions | ||
|
||
transformSourceTable.py ./hsc --output ./hsc --id filter=HSC-R^HSC-I --no-versions --config functorFile=./scripts/policy/Source_hsc.yaml | ||
|
||
consolidateSourceTable.py ./hsc --output ./hsc --id visit=34648^34670^34674^34690^34714^35892^36140^36192^36236^36260 --no-versions | ||
|
||
# Finally, clean out SRC and sourceTable per-ccd parquet files that we do not need | ||
|
||
rm hsc/0????/HSC-?/output/*-???.parq |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,278 @@ | ||
funcs: | ||
sourceId: # the index of deepCoadd_obj IS the objectId | ||
functor: Index | ||
coord_ra: | ||
# reference position required by db. Not in DPDD | ||
functor: CoordColumn | ||
args: coord_ra | ||
coord_dec: | ||
# Reference position required by db. Not in DPDD | ||
functor: CoordColumn | ||
args: coord_dec | ||
ccdVisitId: | ||
functor: Column | ||
args: ccdVisitId | ||
# objectId: not avaliable | ||
# ssObjectId: not avaliable | ||
parentSourceId: | ||
functor: Column | ||
args: parent | ||
#htmId20: not avaliable | ||
x: | ||
functor: Column | ||
args: slot_Centroid_x | ||
y: | ||
functor: Column | ||
args: slot_Centroid_y | ||
xErr: | ||
functor: Column | ||
args: slot_Centroid_xErr | ||
yErr: | ||
functor: Column | ||
args: slot_Centroid_yErr | ||
# x_y_Cov: not available | ||
ra: | ||
functor: RAColumn | ||
# raErr: not available yet DM-15180 | ||
decl: | ||
functor: DecColumn | ||
# declErr: not available yet DM-15180 | ||
# ra_decl_Cov: not available yet | ||
# One calibrated Calib flux is important: | ||
ApFlux: | ||
functor: LocalNanojansky | ||
args: | ||
- slot_CalibFlux_instFlux | ||
- slot_CalibFlux_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
ApFluxErr: | ||
functor: LocalNanojanskyErr | ||
args: | ||
- slot_CalibFlux_instFlux | ||
- slot_CalibFlux_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
# TODO: When DM-25019 is complete, these should be | ||
# changed to use the local value of the background | ||
# model, rather than the residual of the background | ||
sky: | ||
functor: LocalNanojansky | ||
args: | ||
- base_LocalBackground_instFlux | ||
- base_LocalBackground_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
skyErr: | ||
functor: LocalNanojanskyErr | ||
args: | ||
- base_LocalBackground_instFlux | ||
- base_LocalBackground_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
PsFlux: | ||
functor: LocalNanojansky | ||
args: | ||
- slot_PsfFlux_instFlux | ||
- slot_PsfFlux_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
PsFluxErr: | ||
functor: LocalNanojanskyErr | ||
args: | ||
- slot_PsfFlux_instFlux | ||
- slot_PsfFlux_instFluxErr | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalibErr | ||
|
||
# These PS columns makes sense anymore as named: | ||
# psX | ||
# psXSigma | ||
# psY | ||
# psYSigma | ||
# psFlux_psX_Cov | ||
# psFlux_psY_Cov | ||
# psX_Y_Cov | ||
# psLnL | ||
# psChi2 | ||
# psN | ||
# psRa | ||
# psRaSigma | ||
# psDecl | ||
# psDeclSigma | ||
# psFlux_psRa_Cov | ||
# psFlux_psDecl_Cov | ||
|
||
|
||
Ixx: | ||
functor: Column | ||
args: slot_Shape_xx | ||
Iyy: | ||
functor: Column | ||
args: slot_Shape_yy | ||
Ixy: | ||
functor: Column | ||
args: slot_Shape_xy | ||
# DPDD should include Psf Shape | ||
IxxPsf: | ||
functor: Column | ||
args: slot_PsfShape_xx | ||
IyyPsf: | ||
functor: Column | ||
args: slot_PsfShape_yy | ||
IxyPsf: | ||
functor: Column | ||
args: slot_PsfShape_xy | ||
# apNann: Replaced by raw Aperture instFluxes in flags section below | ||
# apMeanSb: Replaced by raw Aperture instFluxes in flags section below | ||
# apMeanSbErr: Replaced by raw Aperture instFluxes in flags section below | ||
extendedness: | ||
functor: Column | ||
args: base_ClassificationExtendedness_value | ||
# The RC2 processing from w_2020_14 did not include calexp sky sources or | ||
# the associated flag. As of w_2020_22, the sky_source flag is expected | ||
# in sourceTable_visit, so this will add the flag column and set them | ||
# all to False (because none of the sources are sky sources). | ||
sky_source: | ||
functor: CustomFunctor | ||
args: base_PixelFlags_flag & ~base_PixelFlags_flag | ||
flags: | ||
- base_LocalPhotoCalib | ||
- base_LocalPhotoCalib_flag | ||
- base_LocalPhotoCalibErr | ||
- base_LocalWcs_flag | ||
- base_LocalWcs_CDMatrix_2_1 | ||
- base_LocalWcs_CDMatrix_1_1 | ||
- base_LocalWcs_CDMatrix_1_2 | ||
- base_LocalWcs_CDMatrix_2_2 | ||
- base_Blendedness_abs | ||
- base_Blendedness_flag | ||
- base_Blendedness_flag_noCentroid | ||
- base_Blendedness_flag_noShape | ||
- base_CircularApertureFlux_12_0_flag | ||
- base_CircularApertureFlux_12_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_12_0_flag_sincCoeffsTruncated | ||
- base_CircularApertureFlux_12_0_instFlux | ||
- base_CircularApertureFlux_12_0_instFluxErr | ||
- base_CircularApertureFlux_17_0_flag | ||
- base_CircularApertureFlux_17_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_17_0_instFlux | ||
- base_CircularApertureFlux_17_0_instFluxErr | ||
- base_CircularApertureFlux_25_0_flag | ||
- base_CircularApertureFlux_25_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_25_0_instFlux | ||
- base_CircularApertureFlux_25_0_instFluxErr | ||
- base_CircularApertureFlux_35_0_flag | ||
- base_CircularApertureFlux_35_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_35_0_instFlux | ||
- base_CircularApertureFlux_35_0_instFluxErr | ||
- base_CircularApertureFlux_3_0_flag | ||
- base_CircularApertureFlux_3_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_3_0_flag_sincCoeffsTruncated | ||
- base_CircularApertureFlux_3_0_instFlux | ||
- base_CircularApertureFlux_3_0_instFluxErr | ||
- base_CircularApertureFlux_4_5_flag | ||
- base_CircularApertureFlux_4_5_flag_apertureTruncated | ||
- base_CircularApertureFlux_4_5_flag_sincCoeffsTruncated | ||
- base_CircularApertureFlux_4_5_instFlux | ||
- base_CircularApertureFlux_4_5_instFluxErr | ||
- base_CircularApertureFlux_50_0_flag | ||
- base_CircularApertureFlux_50_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_50_0_instFlux | ||
- base_CircularApertureFlux_50_0_instFluxErr | ||
- base_CircularApertureFlux_6_0_flag | ||
- base_CircularApertureFlux_6_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_6_0_flag_sincCoeffsTruncated | ||
- base_CircularApertureFlux_6_0_instFlux | ||
- base_CircularApertureFlux_6_0_instFluxErr | ||
- base_CircularApertureFlux_70_0_flag | ||
- base_CircularApertureFlux_70_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_70_0_instFlux | ||
- base_CircularApertureFlux_70_0_instFluxErr | ||
- base_CircularApertureFlux_9_0_flag | ||
- base_CircularApertureFlux_9_0_flag_apertureTruncated | ||
- base_CircularApertureFlux_9_0_flag_sincCoeffsTruncated | ||
- base_CircularApertureFlux_9_0_instFlux | ||
- base_CircularApertureFlux_9_0_instFluxErr | ||
- base_ClassificationExtendedness_flag | ||
- base_FootprintArea_value | ||
- base_Jacobian_flag | ||
- base_Jacobian_value | ||
- base_LocalBackground_instFlux | ||
- base_LocalBackground_instFluxErr | ||
- base_LocalBackground_flag | ||
- base_LocalBackground_flag_noGoodPixels | ||
- base_LocalBackground_flag_noPsf | ||
- base_PixelFlags_flag | ||
- base_PixelFlags_flag_bad | ||
- base_PixelFlags_flag_cr | ||
- base_PixelFlags_flag_crCenter | ||
- base_PixelFlags_flag_edge | ||
- base_PixelFlags_flag_interpolated | ||
- base_PixelFlags_flag_interpolatedCenter | ||
- base_PixelFlags_flag_offimage | ||
- base_PixelFlags_flag_saturated | ||
- base_PixelFlags_flag_saturatedCenter | ||
- base_PixelFlags_flag_suspect | ||
- base_PixelFlags_flag_suspectCenter | ||
- base_PsfFlux_apCorr | ||
- base_PsfFlux_apCorrErr | ||
- base_PsfFlux_area | ||
- base_PsfFlux_flag | ||
- base_PsfFlux_flag_apCorr | ||
- base_PsfFlux_flag_edge | ||
- base_PsfFlux_flag_noGoodPixels | ||
- base_SdssCentroid_flag | ||
- base_SdssCentroid_flag_almostNoSecondDerivative | ||
- base_SdssCentroid_flag_badError | ||
- base_SdssCentroid_flag_edge | ||
- base_SdssCentroid_flag_noSecondDerivative | ||
- base_SdssCentroid_flag_notAtMaximum | ||
- base_SdssCentroid_flag_resetToPeak | ||
- base_Variance_flag | ||
- base_Variance_flag_emptyFootprint | ||
- base_Variance_value | ||
- calib_astrometry_used | ||
- calib_detected | ||
- calib_photometry_reserved | ||
- calib_photometry_used | ||
- calib_psf_candidate | ||
- calib_psf_reserved | ||
- calib_psf_used | ||
- deblend_deblendedAsPsf | ||
- deblend_hasStrayFlux | ||
- deblend_masked | ||
- deblend_nChild | ||
- deblend_parentTooBig | ||
- deblend_patchedTemplate | ||
- deblend_rampedTemplate | ||
- deblend_skipped | ||
- deblend_tooManyPeaks | ||
- ext_shapeHSM_HsmPsfMoments_flag | ||
- ext_shapeHSM_HsmPsfMoments_flag_no_pixels | ||
- ext_shapeHSM_HsmPsfMoments_flag_not_contained | ||
- ext_shapeHSM_HsmPsfMoments_flag_parent_source | ||
- ext_shapeHSM_HsmShapeRegauss_flag | ||
- ext_shapeHSM_HsmShapeRegauss_flag_galsim | ||
- ext_shapeHSM_HsmShapeRegauss_flag_no_pixels | ||
- ext_shapeHSM_HsmShapeRegauss_flag_not_contained | ||
- ext_shapeHSM_HsmShapeRegauss_flag_parent_source | ||
|
||
flag_rename_rules: | ||
# Taken from db-meas-forced | ||
- ['ext_photometryKron_', ''] | ||
- ['base_PixelFlags_flag', 'PixelFlags'] | ||
- ['base_Classification', ''] | ||
- ['base_Sdss', ''] | ||
- ['base_CircularApertureFlux', 'ApFlux'] | ||
- ['ext_shapeHSM_', ''] | ||
- ['ext_convolved_', ''] | ||
- ['undeblended_base', 'undeblended'] | ||
- ['undeblended_ext_photometryKron', 'undeblended'] | ||
- ['ext_photometryKron_', ''] | ||
- ['base_', ''] | ||
- ['slot_', ''] | ||
- ['calib_', 'Calib_'] | ||
- ['deblend_', 'Deblend_'] | ||
- ['blendedness', 'Blendedness'] | ||
|