Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

jp-1690 calspec3 cube_build clean up #5347

Merged
merged 5 commits into from
Sep 21, 2020
Merged

jp-1690 calspec3 cube_build clean up #5347

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
78d41af
if all data flagged as DO_NOT_USE skip file
jemorrison Sep 17, 2020
7507145
remove print statement
jemorrison Sep 17, 2020
2fd7050
remove un-needed statements
jemorrison Sep 18, 2020
ef748ff
updates after review
jemorrison Sep 18, 2020
dce22b9
update change log
jemorrison Sep 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
26 changes: 18 additions & 8 deletions jwst/cube_build/blot_cube_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,21 +41,23 @@ def __init__(self, median_model, input_models):
# Pull out the needed information from the Median IFUCube
self.median_skycube = median_model
self.instrument = median_model.meta.instrument.name
self.detector = median_model.meta.instrument.detector

# basic information about the type of data
self.grating = None
self.filter = None
self.subchannel = None
self.channel = None
self.par_median = None

if self.instrument == 'MIRI':
self.channel = median_model.meta.instrument.channel
self.subchannel = median_model.meta.instrument.band.lower()
self.par_median = self.channel

elif self.instrument == 'NIRSPEC':
self.grating = median_model.meta.instrument.grating
self.filter = median_model.meta.instrument.filter

self.par_median = self.grating
# ________________________________________________________________
# set up x,y,z of Median Cube
# Median cube shoud have linear wavelength
Expand All @@ -80,23 +82,31 @@ def __init__(self, median_model, input_models):
self.cube_wave = self.cube_wave[good_data]
self.cube_flux = self.cube_flux[good_data]

# initialize blotted images to be original input images
self.input_models = input_models

# initialize blotted images to be original input images valid for the Median Image
# read channel (MIRI) or grating (NIRSpec) value that the Median Image covers
# only use input models in this range
self.input_models = []
for model in input_models:
if self.instrument == 'MIRI':
par = model.meta.instrument.channel
if self.instrument == 'NIRSPEC':
par = model.meta.instrument.grating

found = par.find(self.par_median)
if found > -1:
self.input_models.append(model)
# **********************************************************************

def blot_info(self):
""" Prints the basic paramters of the blot image and median sky cube
"""
log.info('Information on Blotting')
log.info('Working with instrument %s %s', self.instrument,
self.detector)
log.info('Working with instrument %s ', self.instrument)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor nit, but could remove unused white space after %s

log.info('shape of sky cube %f %f %f',
self.median_skycube.data.shape[2],
self.median_skycube.data.shape[1],
self.median_skycube.data.shape[0])

log.info('Instrument %s ', self.instrument)
if self.instrument == 'MIRI':
log.info('Channel %s', self.channel)
log.info('Sub-channel %s', self.subchannel)
Expand Down
4 changes: 0 additions & 4 deletions jwst/cube_build/cube_build_step.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,6 @@ def process(self, input):
if not self.single:
self.log.info(f'Number of IFU cubes produced by this run = {num_cubes}')

if self.single:
self.log.info(f'Number of single IFU cubes produced by a this run = {num_cubes}')

# ModelContainer of ifucubes
cube_container = datamodels.ModelContainer()

Expand Down Expand Up @@ -357,7 +354,6 @@ def process(self, input):
for cube in cube_container:
footprint = cube.meta.wcs.footprint(axis_type="spatial")
update_s_region_keyword(cube, footprint)

if status_cube ==1:
self.skip = True

Expand Down
155 changes: 86 additions & 69 deletions jwst/cube_build/ifu_cube.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,10 @@ def build_ifucube(self):
# loop over the files that cover the spectral range the cube is for
for k in range(nfiles):
ifile = self.master_table.FileMap[self.instrument][this_par1][this_par2][k]
# set up ifile_ref to be first file used to copy in basic header info
# to ifucube meta data
if i == 0 and k ==0:
ifile_ref = ifile

log.debug(f"Working on Band defined by: {this_par1} {this_par2}")
# --------------------------------------------------------------------------------
Expand All @@ -638,18 +642,28 @@ def build_ifucube(self):

coord1, coord2, wave, flux, err, slice_no, rois_pixel, roiw_pixel, weight_pixel,\
softrad_pixel, scalerad_pixel, alpha_det, beta_det = pixelresult
# check that there is valid data returned
# If all the data is flagged as DO_NOT_USE - not common then log warning and skip data
nn = wave.size
no_data = False
if nn == 0:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is nn used anywhere else? If not, why not just consolidate these lines into:

if wave.size == 0:
    no_data = True
else:
    no_data = False

no_data = True

t1 = time.time()
log.info("Time to transform pixels to output frame = %.1f s" % (t1 - t0,))
log.debug("Time to transform pixels to output frame = %.1f s" % (t1 - t0,))

# If setting the DQ plane of the IFU
if self.skip_dqflagging:
if self.skip_dqflagging or no_data:
log.info("Skipping setting DQ flagging")
else:
t0 = time.time()
roiw_ave = np.mean(roiw_pixel)
self.map_fov_to_dqplane(this_par1, coord1, coord2, wave, roiw_ave, slice_no)
t1 = time.time()
log.info("Time to set initial dq values = %.1f s" % (t1 - t0,))
log.debug("Time to set initial dq values = %.1f s" % (t1 - t0,))

if no_data:
log.warning(f'No valid data found on file {ifile.meta.filename}')
if self.weighting == 'msm' or self.weighting == 'emsm':
t0 = time.time()
cube_cloud.match_det2cube_msm(self.naxis1, self.naxis2, self.naxis3,
Expand Down Expand Up @@ -792,7 +806,7 @@ def build_ifucube(self):
t1 = time.time()
log.info("Time to find Cube Flux = %.1f s" % (t1 - t0,))
# result consist of ifu_cube and status
result = self.setup_final_ifucube_model(0)
result = self.setup_final_ifucube_model(ifile_ref)
# _______________________________________________________________________
# shove Flux and iflux in the final IFU cube

Expand All @@ -811,69 +825,73 @@ def build_ifucube_single(self):
"""
# loop over input models
single_ifucube_container = datamodels.ModelContainer()
n = len(self.input_models)

log.info("Number of Single IFU cubes to create = %i" % n)
this_par1 = self.list_par1[0] # only one channel is used in this approach
# this_par2 = None # not important for this type of mapping
cube_debug = None

for j in range(n):
log.debug("Working on next Single IFU Cube = %i" % (j + 1))
t0 = time.time()
# for each new data model create a new spaxel
total_num = self.naxis1 * self.naxis2 * self.naxis3
self.spaxel_flux = np.zeros(total_num)
self.spaxel_weight = np.zeros(total_num)
self.spaxel_iflux = np.zeros(total_num)
self.spaxel_dq = np.zeros((self.naxis3, self.naxis2 * self.naxis1), dtype=np.uint32)
self.spaxel_var = np.zeros(total_num)

subtract_background = False

pixelresult = self.map_detector_to_outputframe(this_par1,
subtract_background,
self.input_models[j])

coord1, coord2, wave, flux, err, slice_no, rois_pixel, roiw_pixel, weight_pixel, \
softrad_pixel, scalerad_pixel, alpha_det, beta_det = pixelresult

cube_cloud.match_det2cube_msm(self.naxis1,
self.naxis2,
self.naxis3,
self.cdelt1, self.cdelt2,
self.cdelt3_normal,
self.xcenters,
self.ycenters,
self.zcoord,
self.spaxel_flux,
self.spaxel_weight,
self.spaxel_iflux,
self.spaxel_var,
flux,
err,
coord1, coord2, wave,
self.weighting,
rois_pixel, roiw_pixel,
weight_pixel,
softrad_pixel,
scalerad_pixel,
cube_debug,
self.debug_file)
number_bands = len(self.list_par1)
this_par1 = self.list_par1[0] # single IFUcube only have a single channel
j = 0
for i in range(number_bands):
this_par2 = self.list_par2[i]
nfiles = len(self.master_table.FileMap[self.instrument][this_par1][this_par2])
# ________________________________________________________________________________
# loop over the files that cover the spectral range the cube is for
for k in range(nfiles):
ifile = self.master_table.FileMap[self.instrument][this_par1][this_par2][k]
cube_debug = None
log.debug("Working on next Single IFU Cube = %i" % (j + 1))
t0 = time.time()
# for each new data model create a new spaxel
total_num = self.naxis1 * self.naxis2 * self.naxis3
self.spaxel_flux = np.zeros(total_num)
self.spaxel_weight = np.zeros(total_num)
self.spaxel_iflux = np.zeros(total_num)
self.spaxel_dq = np.zeros((self.naxis3, self.naxis2 * self.naxis1), dtype=np.uint32)
self.spaxel_var = np.zeros(total_num)

subtract_background = False

pixelresult = self.map_detector_to_outputframe(this_par1,
subtract_background,
ifile)

coord1, coord2, wave, flux, err, slice_no, rois_pixel, roiw_pixel, weight_pixel, \
softrad_pixel, scalerad_pixel, alpha_det, beta_det = pixelresult

cube_cloud.match_det2cube_msm(self.naxis1,
self.naxis2,
self.naxis3,
self.cdelt1, self.cdelt2,
self.cdelt3_normal,
self.xcenters,
self.ycenters,
self.zcoord,
self.spaxel_flux,
self.spaxel_weight,
self.spaxel_iflux,
self.spaxel_var,
flux,
err,
coord1, coord2, wave,
self.weighting,
rois_pixel, roiw_pixel,
weight_pixel,
softrad_pixel,
scalerad_pixel,
cube_debug,
self.debug_file)
# _______________________________________________________________________
# shove Flux and iflux in the final ifucube
self.find_spaxel_flux()
self.find_spaxel_flux()

# determine Cube Spaxel flux
status = 0
result = self.setup_final_ifucube_model(j)
ifucube_model, status = result
t1 = time.time()
log.debug("Time to Create Single ifucube = %.1f s" % (t1 - t0,))
single_ifucube_container.append(ifucube_model)
if status !=0:
log.debug("Possible problem with single ifu cube, no valid data in cube" )

status = 0
result = self.setup_final_ifucube_model(ifile)
ifucube_model, status = result
t1 = time.time()
log.debug("Time to Create Single ifucube = %.1f s" % (t1 - t0,))
single_ifucube_container.append(ifucube_model)
if status !=0:
log.debug("Possible problem with single ifu cube, no valid data in cube" )
j = j + 1
return single_ifucube_container
# **************************************************************************

Expand Down Expand Up @@ -1216,7 +1234,7 @@ def setup_ifucube_wcs(self):
lambda_max = []

self.num_bands = len(self.list_par1)
log.info('Number of bands in cube: %i', self.num_bands)
log.debug('Number of bands in cube: %i', self.num_bands)

for i in range(self.num_bands):
this_a = parameter1[i]
Expand Down Expand Up @@ -2052,7 +2070,7 @@ def set_final_dq_flags(self):

# ********************************************************************************

def setup_final_ifucube_model(self, j):
def setup_final_ifucube_model(self, model_ref):

""" Set up the final meta WCS info of IFUCube along with other fits keywords

Expand Down Expand Up @@ -2151,7 +2169,7 @@ def setup_final_ifucube_model(self, j):
weightmap=idata,
wavetable=alldata)

ifucube_model.update(self.input_models[j])
ifucube_model.update(model_ref)
ifucube_model.meta.filename = self.output_name

ifucube_model.data = temp_flux
Expand All @@ -2167,9 +2185,9 @@ def setup_final_ifucube_model(self, j):
ifucube_model.meta.model_type = saved_model_type
# ______________________________________________________________________
if self.output_type == 'single':
with datamodels.open(self.input_models[j]) as input:
with datamodels.open(model_ref) as input:
# define the cubename for each single
filename = self.input_filenames[j]
filename = input.meta.filename
indx = filename.rfind('.fits')
self.output_name_base = filename[:indx]
self.output_file = None
Expand All @@ -2187,7 +2205,6 @@ def setup_final_ifucube_model(self, j):
outchannel = "".join(set(outchannel))
outchannel = "".join(sorted(outchannel))
ifucube_model.meta.instrument.channel = outchannel
log.info(f'IFUChannel {ifucube_model.meta.instrument.channel}')
# ______________________________________________________________________
ifucube_model.meta.wcsinfo.crval1 = self.crval1
ifucube_model.meta.wcsinfo.crval2 = self.crval2
Expand Down Expand Up @@ -2253,7 +2270,7 @@ def setup_final_ifucube_model(self, j):
# if non-linear wavelengths then this will be None
ifucube_model.meta.ifu.weight_power = self.weight_power

with datamodels.open(self.input_models[j]) as input:
with datamodels.open(model_ref) as input:
ifucube_model.meta.bunit_data = input.meta.bunit_data
ifucube_model.meta.bunit_err = input.meta.bunit_err

Expand Down
2 changes: 1 addition & 1 deletion jwst/outlier_detection/outlier_detection_ifu.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def do_detection(self):
)

if save_intermediate_results:
log.info("Writing out resampled IFU cubes...")
log.info("Writing out (single) IFU cube {}".format(model.meta.filename))
model.save(model.meta.filename)

# Initialize intermediate products used in the outlier detection
Expand Down