Skip to content

Commit

Permalink
fixed filtername collisions in reading/writing db and pp_distill
Browse files Browse the repository at this point in the history
  • Loading branch information
@mommermi
mommermi committed Nov 19, 2018
1 parent d771bcc commit e520e64
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 27 deletions.
38 changes: 31 additions & 7 deletions catalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ def __init__(self, catalogname, display=False):
self.history = '' # catalog history
self.magsys = '' # [AB|Vega|instrumental]
self.display = display
self.filtername = None

# data access functions

Expand Down Expand Up @@ -150,22 +151,26 @@ def add_field(self, field_name, field_array, field_type=None):
else:
return self.data.add_column(Column(field_array, name=field_name))

def add_fields(self, field_names, field_arrays, field_types):
def add_fields(self, field_names, field_arrays, field_types=None):
"""
add fields to self.data
input: field_names, field_arrays, field_types
output: number of added fields
"""

assert len(field_names) == len(field_arrays) == len(field_types)
assert len(field_names) == len(field_arrays)

if self.data is None:
self.data = Table()

for i in range(len(field_names)):
self.data.add_column(Column(np.array(field_arrays[i]),
name=field_names[i],
format=field_types[i]))
if field_types is None:
self.data.add_column(Column(np.array(field_arrays[i]),
name=field_names[i]))
else:
self.data.add_column(Column(np.array(field_arrays[i]),
name=field_names[i],
format=field_types[i]))

return len(field_arrays)

Expand Down Expand Up @@ -870,9 +875,10 @@ def write_database(self, filename):
# create header and write to database
header = Table([[self.catalogname], [self.origin], [self.history],
[self.magsys], [self.obstime[0]], [self.obstime[1]],
[self.obj]],
[self.obj], [self.filtername]],
names=['name', 'origin', 'description',
'magsys', 'obstime', 'exptime', 'obj'])
'magsys', 'obstime', 'exptime', 'obj',
'filtername'])
header.to_pandas().to_sql('header', db_conn, index=False)

# write data to database
Expand Down Expand Up @@ -921,11 +927,29 @@ def read_database(self, filename):
self.obstime[0] = header['obstime'][0]
self.obstime[1] = header['exptime'][0]
self.obj = header['obj'][0]
self.filtername = header['filtername'][0]

# read in data table
self.data = Table.from_pandas(read_sql('SELECT * FROM data',
db_conn))

# rename Johnson filternames
for filtername in ['B', 'V', 'R', 'I']:
if '_'+filtername+'Johnsonmag' in list(self.data.columns):
self.data.rename_column(
'_{:s}Johnsonmag'.format(filtername),
'_{:s}mag'.format(filtername))
self.data.rename_column(
'_e_{:s}Johnsonmag'.format(filtername),
'_e_{:s}mag'.format(filtername))
elif filtername+'Johnsonmag' in list(self.data.columns):
self.data.rename_column(
'{:s}Johnsonmag'.format(filtername),
'{:s}mag'.format(filtername))
self.data.rename_column(
'e_{:s}Johnsonmag'.format(filtername),
'e_{:s}mag'.format(filtername))

return self.shape[0]

# filter transformations
Expand Down
18 changes: 12 additions & 6 deletions pp_calibrate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,12 +417,17 @@ def fchi2(zp): return np.sum([(zp-residuals)**2/residuals_sig])
cat.data.rename_column('ra_deg_1', 'ra_deg')
cat.data.rename_column('dec_deg_1', 'dec_deg')

if filterkey not in cat.fields:
cat.add_fields([filterkey, efilterkey],
[cat['MAG_'+_pp_conf.photmode] + clipping_steps[idx][0],
np.sqrt(cat['MAGERR_'+_pp_conf.photmode]**2 +
clipping_steps[idx][1]**2)],
['F', 'F'])
# remove columns for filterkey for matched sources
if filterkey in cat.fields:
cat.data.remove_column(filterkey)
cat.data.remove_column(efilterkey)

cat.add_fields([filterkey, efilterkey],
[cat['MAG_'+_pp_conf.photmode] +
clipping_steps[idx][0],
np.sqrt(cat['MAGERR_'+_pp_conf.photmode]**2 +
clipping_steps[idx][1]**2)],
['F', 'F'])

# add ref_cat identifier to catalog
cat.origin = cat.origin.strip() + ";" + ref_cat.catalogname + ";"\
Expand Down Expand Up @@ -486,6 +491,7 @@ def calibrate(filenames, minstars, manfilter, manualcatalog,
filternames[filtername] = [filename]
ldac_filename = filename[:filename.find('.fit')]+'.ldac'
cat = catalog(filename)
cat.filtername = filtername
if display:
print(cat.read_ldac(ldac_filename, filename, maxflag=maxflag,
object_keyword=obsparam['object'],
Expand Down
31 changes: 17 additions & 14 deletions pp_distill.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -513,8 +513,8 @@ def distill(catalogs, man_targetname, offset, fixed_targets_file, posfile,
print('#-----------------------')

if display:
print(len(objects)/len(catalogs),
'potential target(s) per frame identified.')
print('{:d} potential target(s) per frame identified.'.format(
int(len(objects)/len(catalogs))))

# extract source data for identified targets

Expand Down Expand Up @@ -553,32 +553,35 @@ def distill(catalogs, man_targetname, offset, fixed_targets_file, posfile,

target_cat.add_fields(['ident', 'ra_deg', 'dec_deg'],
[[obj['ident'] for obj in objects_thiscat],
[obj['ra_deg'] for obj in objects_thiscat],
[obj['dec_deg'] for obj in objects_thiscat]],
['20A', 'D', 'D'])
[obj['ra_deg']
for obj in objects_thiscat],
[obj['dec_deg'] for obj in objects_thiscat]])

# identify filtername
filtername = cat.filtername

# identify magnitudes
mag_keys = ['MAG_APER', 'MAGERR_APER']
for key in cat.fields:
if 'mag' in key:
if filtername+'mag' in key:
mag_keys.append(key)

# build field lists for observed catalogs
match_keys_other_catalog, extract_other_catalog = [], []

for key in ['ra_deg', 'dec_deg', 'XWIN_IMAGE', 'YWIN_IMAGE',
'FLAGS', 'FWHM_WORLD']:
if key in cat.fields:
match_keys_other_catalog.append(key)
extract_other_catalog.append(key)

match = target_cat.match_with(cat,
match_keys_this_catalog=(
'ra_deg', 'dec_deg'),
match_keys_other_catalog=match_keys_other_catalog,
extract_this_catalog=[
'ra_deg', 'dec_deg', 'ident'],
extract_other_catalog=extract_other_catalog+mag_keys,
tolerance=None)
match = target_cat.match_with(
cat,
match_keys_this_catalog=('ra_deg', 'dec_deg'),
match_keys_other_catalog=match_keys_other_catalog,
extract_this_catalog=['ra_deg', 'dec_deg', 'ident'],
extract_other_catalog=extract_other_catalog+mag_keys,
tolerance=None)

for i in range(len(match[0][0])):
# derive calibrated magnitudes, if available
Expand Down

0 comments on commit e520e64

Please sign in to comment.