Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-43033: Update to fgcm 3.10.5 with reserved refstar updates. #41

Merged
merged 9 commits into from
Mar 6, 2024
2 changes: 1 addition & 1 deletion fgcm/fgcmChisq.py
Original file line number Diff line number Diff line change
Expand Up @@ -868,7 +868,7 @@ def _chisqWorker(self, goodStarsAndObs):
# Get good observations of reference stars
# This must be two steps because we first need the indices to
# avoid out-of-bounds
mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR']
mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR'] | objFlagDict['REFSTAR_RESERVED']
goodRefObsGO, = np.where((objRefIDIndex[obsObjIDIndexGO] >= 0) &
((objFlag[obsObjIDIndexGO] & mask) == 0))

Expand Down
2 changes: 1 addition & 1 deletion fgcm/fgcmGray.py
Original file line number Diff line number Diff line change
Expand Up @@ -1270,7 +1270,7 @@ def computeExposureReferenceOffsets(self):
objRefIDIndex = snmm.getArray(self.fgcmStars.objRefIDIndexHandle)
refMag = snmm.getArray(self.fgcmStars.refMagHandle)

goodStars = self.fgcmStars.getGoodStarIndices(checkMinObs=True, removeRefstarOutliers=True, removeRefstarBadcols=True)
goodStars = self.fgcmStars.getGoodStarIndices(checkMinObs=True, removeRefstarOutliers=True, removeRefstarBadcols=True, removeRefstarReserved=True)
_, goodObs = self.fgcmStars.getGoodObsIndices(goodStars, expFlag=self.fgcmPars.expFlag, checkBadMag=True)

# Add in the gray values
Expand Down
3 changes: 2 additions & 1 deletion fgcm/fgcmSigmaRef.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ def computeSigmaRef(self):

# Select only stars that have reference magnitudes
# and that are not flagged as outliers
# We allow all the reference colors in to get color range statistics.
# We allow all the reference colors including reserved in
# to get color range statistics.
mask = objFlagDict['REFSTAR_OUTLIER']
use, = np.where((objRefIDIndex[goodStars] >= 0) &
((objFlag[goodStars] & mask) == 0))
Expand Down
27 changes: 20 additions & 7 deletions fgcm/fgcmStars.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,10 @@ def loadStars(self, fgcmPars,
if test.size > 0:
self.fgcmLog.info('Flagging %d stars as reference star outliers from previous cycles.' %
(test.size))
test, = np.where((flagFlag[a] & objFlagDict['REFSTAR_RESERVED']) > 0)
if test.size > 0:
self.fgcmLog.info('Flagging %d stars as reference star reserved from previous cycles.' %

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This sentence isn't very clear. Something like 'Flagging %d stars as reference stars that were reserved from previous cycles.' (if I am correctly interpreting the original sentence) would be better.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changing it at this point will have to wait for the next version of fgcm, unless I go through a whole rigmarole. But thank you for pointing this out.

(test.size))

objFlag[b] = flagFlag[a]
else:
Expand Down Expand Up @@ -699,15 +703,17 @@ def prepStars(self, fgcmPars):
if (refMatches.size/objRefIDIndex.size > self.refStarMaxFracUse):
self.fgcmLog.info("Fraction of reference star matches is greater than "
"refStarMaxFracUse (%.3f); down-sampling." % (self.refStarMaxFracUse))
nTarget = int(self.config.refStarMaxFracUse*objRefIDIndex.size)
objFlag = snmm.getArray(self.objFlagHandle)

nTarget = int(self.refStarMaxFracUse*objRefIDIndex.size)
nMatch = refMatches.size
nToRemove = nMatch - nTarget

remove = np.random.choice(refMatches.size,
size=nToRemove,
replace=False)
refMag[objRefIDIndex[refMatches[remove]], :] = 99.0
objRefIDIndex[refMatches[remove]] = -1
# Flag these as REFSTAR_RESERVED
objFlag[refMatches[remove]] |= objFlagDict['REFSTAR_RESERVED']

# Compute the fraction of stars that are reference stars
for i, band in enumerate(self.bands):
Expand Down Expand Up @@ -1041,7 +1047,8 @@ def computeNTotalStats(self, fgcmPars):
1)

def getGoodStarIndices(self, includeReserve=False, onlyReserve=False, checkMinObs=False,
checkHasColor=False, removeRefstarOutliers=False, removeRefstarBadcols=False):
checkHasColor=False, removeRefstarOutliers=False, removeRefstarBadcols=False,
removeRefstarReserved=False):
"""
Get the good star indices.

Expand All @@ -1059,6 +1066,8 @@ def getGoodStarIndices(self, includeReserve=False, onlyReserve=False, checkMinOb
Remove reference star outliers.
removeRefstarBadcols : `bool`, optional
Remove reference stars with "bad colors".
removeRefstarReserved : `bool`, optional
Remove reference stars that are "reserved"?

returns
-------
Expand All @@ -1077,6 +1086,8 @@ def getGoodStarIndices(self, includeReserve=False, onlyReserve=False, checkMinOb
mask |= objFlagDict['REFSTAR_OUTLIER']
if removeRefstarBadcols:
mask |= objFlagDict['REFSTAR_BAD_COLOR']
if removeRefstarReserved:
mask |= objFlagDict['REFSTAR_RESERVED']

if includeReserve or onlyReserve:
mask &= ~objFlagDict['RESERVED']
Expand Down Expand Up @@ -1413,7 +1424,7 @@ def computeAbsOffset(self):

goodStars = self.getGoodStarIndices(includeReserve=False, checkMinObs=True)

mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR']
mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR'] | objFlagDict['REFSTAR_RESERVED']
use, = np.where((objRefIDIndex[goodStars] >= 0) &
((objFlag[goodStars] & mask) == 0))
goodRefStars = goodStars[use]
Expand Down Expand Up @@ -1528,7 +1539,7 @@ def computeEGray(self, goodObs, ignoreRef=True, onlyObsErr=False):
refMagErr = snmm.getArray(self.refMagErrHandle)

# Only use _good_ (non-outlier) reference stars in here.
mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR']
mask = objFlagDict['REFSTAR_OUTLIER'] | objFlagDict['REFSTAR_BAD_COLOR'] | objFlagDict['REFSTAR_RESERVED']
goodRefObsGO, = np.where((objRefIDIndex[obsObjIDIndex[goodObs]] >= 0) &
((objFlag[obsObjIDIndex[goodObs]] & mask) == 0))

Expand Down Expand Up @@ -2028,7 +2039,8 @@ def getFlagStarIndices(self):
# everything else should be recomputed based on the good exposures, calibrations, etc
flagMask = (objFlagDict['VARIABLE'] |
objFlagDict['RESERVED'] |
objFlagDict['REFSTAR_OUTLIER'])
objFlagDict['REFSTAR_OUTLIER'] |
objFlagDict['REFSTAR_RESERVED'])

flagged,=np.where((objFlag & flagMask) > 0)

Expand Down Expand Up @@ -2147,6 +2159,7 @@ def plotRefStarColorTermResiduals(self, fgcmPars):
refMag = snmm.getArray(self.refMagHandle)

for mode in ['all', 'cut']:
# We leave in the "RESERVED" stars for these plots.
if (mode == 'all'):
goodStars = self.getGoodStarIndices(includeReserve=True,
checkMinObs=True,
Expand Down
3 changes: 2 additions & 1 deletion fgcm/fgcmUtilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
'RESERVED':2**4,
'REFSTAR_OUTLIER': 2**5,
'BAD_QUANTITY': 2**6,
'REFSTAR_BAD_COLOR': 2**7}
'REFSTAR_BAD_COLOR': 2**7,
'REFSTAR_RESERVED': 2**8}

# Dictionary of observation flags
obsFlagDict = {'NO_EXPOSURE': 2**0,
Expand Down