Merge pull request #4 from lsst-dm/tickets/DM-31462

DM-31462: Update task ordering and configs for running fgcmBuildStarsTable

FGCM/all_steps.sh

@@ -3,11 +3,11 @@ DM=29946
 WORKDIR=/datasets/hsc/repo/rerun/RC/${WEEK}/DM-${DM}/
 LOGDIR=$WORKDIR/logs/FGCM
 mkdir -p $LOGDIR
-. /software/lsstsw/stack/loadLSST.bash 
+. /software/lsstsw/stack/loadLSST.bash
 setup lsst_distrib -t $WEEK
-fgcmMakeLut.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmMakeLutRC2.py --clobber-config > $LOGDIR/fgcmMakeLut.log 2>&1 
-srun fgcmBuildStars.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmBuildStarsRC2.py --id ccd=40 filter=HSC-G^HSC-R^HSC-I^HSC-Z^HSC-Y^NB0921 --clobber-config > $LOGDIR/fgcmBuildStars.log 2>&1 
-fgcmFitCycle.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmFitCycleRC2_cycle00_config.py --clobber-config > $LOGDIR/fgcmFitCycle_0.log 2>&1 
+fgcmMakeLut.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmMakeLutRC2.py --clobber-config > $LOGDIR/fgcmMakeLut.log 2>&1
+srun fgcmBuildStarsTable.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmBuildStarsTableRC2.py --id filter=HSC-G^HSC-R^HSC-I^HSC-Z^HSC-Y^NB0921 --clobber-config > $LOGDIR/fgcmBuildStarsTable.log 2>&1
+fgcmFitCycle.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmFitCycleRC2_cycle00_config.py --clobber-config > $LOGDIR/fgcmFitCycle_0.log 2>&1
 fgcmFitCycle.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmFitCycleRC2_cycle01_config.py --clobber-config > $LOGDIR/fgcmFitCycle_1.log 2>&1
 fgcmFitCycle.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmFitCycleRC2_cycle02_config.py --clobber-config > $LOGDIR/fgcmFitCycle_2.log 2>&1
 fgcmFitCycle.py /datasets/hsc/repo --rerun RC/$WEEK/DM-${DM}-sfm:RC/$WEEK/DM-${DM}/FGCM --configfile fgcmFitCycleRC2_cycle03_config.py --clobber-config > $LOGDIR/fgcmFitCycle_3.log 2>&1

FGCM/fgcmBuildStarsTableRC2.py

@@ -2,6 +2,8 @@
 HSC-specific overrides for FgcmBuildStars
 """
 
+config.randomSeed = 23456
+
 # Minimum number of observations per band for a star to be considered for calibration
 config.minPerBand = 1
 # Match radius to associate stars from src catalogs (arcseconds)

config/RC2templates.yaml

@@ -442,6 +442,7 @@ task_list_all_visits:
 tasks_to_execute_ordered:
   - makeSkyMap
   - singleFrameDriver
+  - consolidateSourceTable
   - jointcal
   - skyCorrection
 
@@ -466,5 +467,4 @@ tasks_to_execute_ordered:
   - validateDrp
   - transformObjectCatalog
   - consolidateObjectTable
-  - consolidateSourceTable
   #- fakesDriver

notes_RC2.txt

@@ -52,28 +52,28 @@ SQUASH_USER=$USER
 SQUASH_password=*****
 
 # You should be ready to start running from the run directory (in your screen with the above setup) just run:
-source 01_makeSkyMap.sh 
+source 01_makeSkyMap.sh
 
 # The steps mostly need to be run sequentially. To automate I did:
 alias swait='while [ `squeue | grep $USER | wc -l` -gt 0 ]; do sleep 300; echo "Waiting".; done;'
 # swait waits until all jobs are done
-source 01_makeSkyMap.sh; swait source 02_singleFrameDriver.sh; swait source 03_jointcal.sh; swait source 04_skyCorrection.sh
+source 01_makeSkyMap.sh; swait source 02_singleFrameDriver.sh; swait source 03_consolidateSourceTable.sh; swait source 04_jointcal.sh; swait source 05_skyCorrection.sh
 # Will thus complete the first four steps (roughly 24 hours of processing)
 
-# After 03_jointcal.sh you need to run FGCM. In a separate screen (e.g. screen -S FGCM)
+# After 04_jointcal.sh you need to run FGCM. In a separate screen (e.g. screen -S FGCM)
 cd /THIS/DIRECTORY/FGCM/
 # Edit all_steps.sh to change WEEK and DM to the current week and DM
 source all_steps.sh
-# FGCM will take about 24 hours. You can run 04_skyCorrection.sh simultaneously. Check logs in
+# FGCM will take about 24 hours. You can run 05_skyCorrection.sh simultaneously. Check logs in
 # /datasets/hsc/repo/rerun/RC/${WEEK}/DM-${DM}/logs/FGCM
- 
-#After FGCM, just go to your RC2 screen and 
-source 05_visitAnalysis.sh
-#etc. Step 5-10 will take ~5 days
+
+#After FGCM, just go to your RC2 screen and
+source 06_visitAnalysis.sh
+#etc. Step 6-11 will take ~5 days
 
 # The remaining steps (post forced photometry) take ~1 day but require more log reading
 
-# Before running the steps that dispatch results to Squash (steps 15, 16 and 18), make sure following environment variables
+# Before running the steps that dispatch results to Squash (steps 16, 17 and 19), make sure following environment variables
 # are set (see https://pipelines.lsst.io/modules/lsst.verify/scripts/dispatch_verify.py.html)
 export DATASET="HSC RC2"
 export DATASET_REPO_URL="https://jira.lsstcorp.org/browse/DM-30424"
@@ -82,7 +82,7 @@ export RUN_ID_URL="https://jira.lsstcorp.org/browse/DM-30424"
 export VERSION_TAG="w_2021_22"
 
 # You need to regularly check logs. There's no good way to do this. Many scripts write to slurm logs so
-# tail sfm* 
+# tail sfm*
 # shows you the last few lines of all single frame processing jobs and will usually show you if they are okay.
 # similarly for tail coadd*, tail sky*, tail jointcal*, tail mt*
 # The names and locations of logs are listed in either the .sh script or its associated .sl or .mp files
@@ -100,4 +100,4 @@ ln -s /datasets/hsc/repo/rerun/RC/$WEEK/DM-$DM/plots /home/$USER/public_html/${W
 # Then run the script, which will copy logs from your run directory to the repo
 python  copy_log.py
 
-# Congratulations, you have made it through an arduous RC2. Now look at notes_RC2_fakes.txt  
+# Congratulations, you have made it through an arduous RC2. Now look at notes_RC2_fakes.txt