-
Notifications
You must be signed in to change notification settings - Fork 0
/
notes_RC2.txt
103 lines (85 loc) · 4.39 KB
/
notes_RC2.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# All this should be performed in a screen session (e.g. screen -S RC2)
# on lsst-condorprod-sub01 (which you can log into through lsst-login
# You can log out of a screen session with ctrl-a-d and then log back in with screen -r RC2
# Set week and DM. The DM is detrermined when you make a "RC2 reprocessing with $WEEK" JIRA Ticket
# Assign to self
# Team- Data Facility
# Watchers- yusra, emorganson
# Epic Link- LOE: Periodic reprocessing of test data sets
WEEK=w_2021_18
DM=29946
# GOODWEEK and GOODDM are the WEEK and DM from the last successful run
GOODWEEK=w_2021_14
GOODDM=29519
# Basic Setup, confirm the week's stack you want is ready
source /software/lsstsw/stack/loadLSST.bash
eups list lsst_distrib
setup lsst_distrib $WEEK
# Seeting up the gen2 RC2 scripts
mkdir -p /home/$USER/HSC/RC2/rerun_scripts/
./gen2-RC2-generator --DMticket DM-$DM --week $WEEK --rerun2 RC/${GOODWEEK}/DM-${GOODDM} --filepath /home/$USER/HSC/RC2/rerun_scripts/{DMticket}
cd /home/$USER/HSC/RC2/rerun_scripts/DM-$DM
source cloner.sh
scl enable devtoolset-8 bash
# scl kills your setup and variables, so after reassigning week IDENTICALLY to above, we setup our stack
WEEK=w_2021_18
source /software/lsstsw/stack/loadLSST.bash
setup lsst_distrib $WEEK
source builder.sh
CWD=$(pwd)
cd git/qa_explorer
setup -j -r .
scons
cd $CWD
cd git/pipe_analysis
setup -j -r .
scons
cd $CWD
# Squash is used to send in some data in steps
# dispatch_verify_colorAnalysis dispatch_verify_coaddAnalysis dispatch_verify_matchedVisit
# You need to get a squash account (directions here):
# https://confluence.lsstcorp.org/display/DM/RC2+Reprocessing+recipe in point 6
#
SQUASH_USER=$USER
SQUASH_password=*****
# You should be ready to start running from the run directory (in your screen with the above setup) just run:
source 01_makeSkyMap.sh
# The steps mostly need to be run sequentially. To automate I did:
alias swait='while [ `squeue | grep $USER | wc -l` -gt 0 ]; do sleep 300; echo "Waiting".; done;'
# swait waits until all jobs are done
source 01_makeSkyMap.sh; swait source 02_singleFrameDriver.sh; swait source 03_consolidateSourceTable.sh; swait source 04_jointcal.sh; swait source 05_skyCorrection.sh
# Will thus complete the first four steps (roughly 24 hours of processing)
# After 04_jointcal.sh you need to run FGCM. In a separate screen (e.g. screen -S FGCM)
cd /THIS/DIRECTORY/FGCM/
# Edit all_steps.sh to change WEEK and DM to the current week and DM
source all_steps.sh
# FGCM will take about 24 hours. You can run 05_skyCorrection.sh simultaneously. Check logs in
# /datasets/hsc/repo/rerun/RC/${WEEK}/DM-${DM}/logs/FGCM
#After FGCM, just go to your RC2 screen and
source 06_visitAnalysis.sh
#etc. Step 6-11 will take ~5 days
# The remaining steps (post forced photometry) take ~1 day but require more log reading
# Before running the steps that dispatch results to Squash (steps 16, 17 and 19), make sure following environment variables
# are set (see https://pipelines.lsst.io/modules/lsst.verify/scripts/dispatch_verify.py.html)
export DATASET="HSC RC2"
export DATASET_REPO_URL="https://jira.lsstcorp.org/browse/DM-30424"
export RUN_ID="DM-30424"
export RUN_ID_URL="https://jira.lsstcorp.org/browse/DM-30424"
export VERSION_TAG="w_2021_22"
# You need to regularly check logs. There's no good way to do this. Many scripts write to slurm logs so
# tail sfm*
# shows you the last few lines of all single frame processing jobs and will usually show you if they are okay.
# similarly for tail coadd*, tail sky*, tail jointcal*, tail mt*
# The names and locations of logs are listed in either the .sh script or its associated .sl or .mp files
# I there are errors in the logs, ping the slack channel dm-hsc-reprocessing
# FGCM issues go to Eli Rykoff @eli
# Most analysis issues (everything except single frame, sky, coadd, multiband and forced phot) go to Lauren MacCarthur @laurenam
# Yusra AlSayyad @yusra is a good resource for everything else
#After all processing is done
ln -s /datasets/hsc/repo/rerun/RC/$WEEK/DM-$DM/plots /home/$USER/public_html/${WEEK}_qaplots
# makes plots at https://lsst.ncsa.illinois.edu/~$USER
# Slack Simon @ksk and tell him you did the Squash stuff
# Finally, edit WEEK and DM in copy_log.py, and replace the absolute path in variable INDIR with your username.
# Then run the script, which will copy logs from your run directory to the repo
python copy_log.py
# Congratulations, you have made it through an arduous RC2. Now look at notes_RC2_fakes.txt