Skip to content

Commit

Permalink
Merge branch 'tickets/DM-27437' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@plazas
plazas committed Nov 10, 2020
2 parents 6d87fe0 + 2e5aafa commit b30456c
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 72 deletions.
21 changes: 0 additions & 21 deletions python/lsst/cp/pipe/astierCovPtcFit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,26 +272,6 @@ def subtractDistantOffset(self, maxLag=8, startLag=5, polDegree=1):

return

def setMaxMu(self, maxMu):
"""Select signal level based on max average signal in ADU"""
# mus are sorted at construction
index = self.mu < maxMu
k = index.sum()
self.mu = self.mu[:k]
self.cov = self.cov[:k, ...]
self.vcov = self.vcov[:k, ...]
self.sqrtW = self.sqrtW[:k, ...]

return

def setMaxMuElectrons(self, maxMuEl):
"""Select signal level based on max average signal in electrons"""
g = self.getGain()
kill = (self.mu*g > maxMuEl)
self.sqrtW[kill, :, :] = 0

return

def copy(self):
"""Make a copy of params"""
cop = copy.deepcopy(self)
Expand Down Expand Up @@ -604,7 +584,6 @@ def fitFullModel(self, pInit=None, nSigma=5.0, maxFitIter=3):
if ierr not in [1, 2, 3, 4]:
raise RuntimeError("Minimization failed: " + mesg)
self.covParams = paramsCov

return params

def ndof(self):
Expand Down
30 changes: 4 additions & 26 deletions python/lsst/cp/pipe/astierCovPtcUtils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,6 @@ class LoadParams:
r: `int`, optional
Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
maxMu: `float`, optional
Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
maxMuElectrons: `float`, optional
Maximum signal in electrons.
subtractDistantValue: `bool`, optional
Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
Expand All @@ -263,8 +257,6 @@ class LoadParams:
"""
def __init__(self):
self.r = 8
self.maxMu = 1e9
self.maxMuElectrons = 1e9
self.subtractDistantValue = False
self.start = 5
self.offsetDegree = 1
Expand Down Expand Up @@ -303,24 +295,15 @@ def loadData(tupleName, params):
c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
else:
c = CovFit(ntext, params.r)
thisMaxMu = params.maxMu
# Tune the maxMuElectrons cut
for iter in range(3):
cc = c.copy()
cc.setMaxMu(thisMaxMu)
cc.initFit() # allows to get a crude gain.
gain = cc.getGain()
if (thisMaxMu*gain < params.maxMuElectrons):
thisMaxMu = params.maxMuElectrons/gain
continue
cc.setMaxMuElectrons(params.maxMuElectrons)
break

cc = c.copy()
cc.initFit() # allows to get a crude gain.
covFitList[ext] = cc

return covFitList


def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):
def fitData(tupleName, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):
"""Fit data to models in Astier+19.
Parameters
Expand All @@ -338,9 +321,6 @@ def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):
r: `int`, optional
Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
maxMu: `float`, optional
Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
nSigmaFullFit : `float`, optional
Sigma cut to get rid of outliers in full model fit.
Expand Down Expand Up @@ -370,9 +350,7 @@ def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):

lparams = LoadParams()
lparams.subtractDistantValue = False
lparams.maxMu = maxMu
lparams.r = r

covFitList = loadData(tupleName, lparams)
covFitNoBList = {} # [None]*(exts[-1]+1)
for ext, c in covFitList.items():
Expand Down
22 changes: 14 additions & 8 deletions python/lsst/cp/pipe/makeBrighterFatterKernel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,15 +108,21 @@ class MakeBrighterFatterKernelTaskConfig(pexConfig.Config):
doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
default='ccd',
)
minMeanSignal = pexConfig.Field(
dtype=float,
doc="Minimum value of mean signal (in ADU) to consider.",
default=0,
minMeanSignal = pexConfig.DictField(
keytype=str,
itemtype=float,
doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
" The same cut is applied to all amps if this dictionary is of the form"
" {'ALL_AMPS': value}",
default={'ALL_AMPS': 0.0},
)
maxMeanSignal = pexConfig.Field(
dtype=float,
doc="Maximum value to of mean signal (in ADU) to consider.",
default=9e6,
maxMeanSignal = pexConfig.DictField(
keytype=str,
itemtype=float,
doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
" The same cut is applied to all amps if this dictionary is of the form"
" {'ALL_AMPS': value}",
default={'ALL_AMPS': 1e6},
)
maxIterRegression = pexConfig.Field(
dtype=int,
Expand Down
50 changes: 33 additions & 17 deletions python/lsst/cp/pipe/ptc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
)
sigmaClipFullFitCovariancesAstier = pexConfig.Field(
dtype=float,
doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
doc="Sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
default=5.0,
)
maxIterFullFitCovariancesAstier = pexConfig.Field(
Expand Down Expand Up @@ -100,15 +100,21 @@ class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
doc="Bin the image by this factor in both dimensions.",
default=1,
)
minMeanSignal = pexConfig.Field(
dtype=float,
doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
default=0,
minMeanSignal = pexConfig.DictField(
keytype=str,
itemtype=float,
doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
" The same cut is applied to all amps if this dictionary is of the form"
" {'ALL_AMPS': value}",
default={'ALL_AMPS': 0.0},
)
maxMeanSignal = pexConfig.Field(
dtype=float,
doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
default=9e6,
maxMeanSignal = pexConfig.DictField(
keytype=str,
itemtype=float,
doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
" The same cut is applied to all amps if this dictionary is of the form"
" {'ALL_AMPS': value}",
default={'ALL_AMPS': 1e6},
)
initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
dtype=float,
Expand Down Expand Up @@ -293,6 +299,19 @@ def runDataRef(self, dataRefList):
for ampName in ampNames:
datasetPtc.inputExpIdPairs[ampName] = expIds

maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
for ampName in ampNames:
if 'ALL_AMPS' in self.config.maxMeanSignal:
maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
elif ampName in self.config.maxMeanSignal:
maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]

if 'ALL_AMPS' in self.config.minMeanSignal:
minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
elif ampName in self.config.minMeanSignal:
minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]

tupleRecords = []
allTags = []
for expTime, (exp1, exp2) in expPairs.items():
Expand All @@ -317,7 +336,7 @@ def runDataRef(self, dataRefList):
nAmpsNan += 1
continue
tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
if (muDiff <= self.config.minMeanSignal) or (muDiff >= self.config.maxMeanSignal):
if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
continue

tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
Expand Down Expand Up @@ -452,7 +471,7 @@ def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
the class `PhotonTransferCurveDatase`.
"""

covFits, covFitsNoB = fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
covFits, covFitsNoB = fitData(covariancesWithTagsArray,
r=self.config.maximumRangeCovariancesAstier,
nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
Expand Down Expand Up @@ -922,14 +941,11 @@ def errFunc(p, x, y):
varVecOriginal = np.array(dataset.rawVars[ampName])
varVecOriginal = self._makeZeroSafe(varVecOriginal)

mask = ((meanVecOriginal >= self.config.minMeanSignal) &
(meanVecOriginal <= self.config.maxMeanSignal))

goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
self.config.initialNonLinearityExclusionThresholdPositive,
self.config.initialNonLinearityExclusionThresholdNegative)
if not (mask.any() and goodPoints.any()):
msg = (f"\nSERIOUS: All points in either mask: {mask} or goodPoints: {goodPoints} are bad."
if not (goodPoints.any()):
msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
f"Setting {ampName} to BAD.")
self.log.warn(msg)
# The first and second parameters of initial fit are discarded (bias and gain)
Expand All @@ -950,7 +966,7 @@ def errFunc(p, x, y):
dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
continue

mask = mask & goodPoints
mask = goodPoints

if ptcFitType == 'EXPAPPROXIMATION':
ptcFunc = funcAstier
Expand Down

0 comments on commit b30456c

Please sign in to comment.