-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add script to make test repos and add v0, v1 test repos
The version 0 is "implicit", as all of our version 0 catalogs are: it has no format_version in the config. The version 1 is "explicit", and has nJy flux units. Exclude new test data from flake8. Add tests of persisted version=0 and version=1 refcats.
- Loading branch information
Showing
13 changed files
with
361 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
#!/usr/bin/env python | ||
"""Write a small trivial reference catalog, to test reading catalog versions. | ||
Takes no arguments, writes a `versionX/` directory containing the new test | ||
repo to the current directory. Requires that obs_test be setup. | ||
This code was used to write the version0/ and version1/ test repos in | ||
`meas_algorithms/tests/data/` (after suitably configuring the code to produce | ||
a "version 0" catalog that looks like most of our existing catalogs (e.g. | ||
fluxSigma instead of fluxErr, no units for fluxes). | ||
""" | ||
import os.path | ||
|
||
import numpy as np | ||
import astropy | ||
|
||
from lsst.meas.algorithms import IngestIndexedReferenceTask | ||
from lsst.meas.algorithms.ingestIndexReferenceTask import LATEST_FORMAT_VERSION | ||
import lsst.utils | ||
|
||
|
||
def make_skyCatalog(outPath, size=100): | ||
"""Write a text file containing an on-sky catalog to be ingested.""" | ||
np.random.seed(123) | ||
epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd") | ||
ident = np.arange(1, size+1, dtype=int) | ||
# ra/dec in degrees, centered on (10,20) | ||
ra = 10 + np.random.random(size) | ||
dec = 20 + np.random.random(size) | ||
ra_err = np.ones(size)*0.1 # arcsec | ||
dec_err = np.ones(size)*0.1 # arcsec | ||
a_mag = 16. + np.random.random(size)*4. | ||
a_mag_err = 0.01 + np.random.random(size)*0.2 | ||
b_mag = 17. + np.random.random(size)*5. | ||
b_mag_err = 0.02 + np.random.random(size)*0.3 | ||
is_photometric = np.random.randint(2, size=size) | ||
is_resolved = np.random.randint(2, size=size) | ||
is_variable = np.random.randint(2, size=size) | ||
# compute proper motion and PM error in arcseconds/year | ||
# and let the ingest task scale them to radians | ||
pm_amt_arcsec = (3.0*lsst.geom.arcseconds).asArcseconds() | ||
pm_dir_rad = (45*lsst.geom.degrees).asRadians() | ||
pm_ra = np.ones(size)*pm_amt_arcsec*np.cos(pm_dir_rad) | ||
pm_dec = np.ones(size)*pm_amt_arcsec*np.sin(pm_dir_rad) | ||
properMotionErr = 1e-3*lsst.geom.arcseconds | ||
pm_ra_err = np.ones(size)*properMotionErr.asArcseconds()*abs(np.cos(pm_dir_rad)) | ||
pm_dec_err = np.ones(size)*properMotionErr.asArcseconds()*abs(np.sin(pm_dir_rad)) | ||
unixtime = np.ones(size)*epoch.unix | ||
|
||
dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float), | ||
('ra_err', float), ('dec_err', float), ('a', float), | ||
('a_err', float), ('b', float), ('b_err', float), ('is_phot', int), | ||
('is_res', int), ('is_var', int), ('pm_ra', float), | ||
('pm_dec', float), ('pm_ra_err', float), | ||
('pm_dec_err', float), ('unixtime', float)]) | ||
|
||
arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err, | ||
is_photometric, is_resolved, is_variable, | ||
pm_ra, pm_dec, pm_ra_err, pm_dec_err, unixtime)), dtype=dtype) | ||
# write the data with full precision; this is not realistic for | ||
# real catalogs, but simplifies tests based on round tripped data | ||
saveKwargs = dict( | ||
header="id,ra_icrs,dec_icrs,ra_err,dec_err," | ||
"a,a_err,b,b_err,is_phot,is_res,is_var," | ||
"pm_ra,pm_dec,pm_ra_err,pm_dec_err,unixtime", | ||
fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g", | ||
"%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", | ||
"%.15g", "%.15g", "%.15g", "%.15g", "%.15g"] | ||
) | ||
|
||
np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs) | ||
return outPath+"/ref.txt" | ||
|
||
|
||
inPath = os.path.join(lsst.utils.getPackageDir('meas_algorithms'), 'tests/data') | ||
outPath = os.path.join(inPath, "version%s" % LATEST_FORMAT_VERSION) | ||
|
||
skyCatalogFile = make_skyCatalog(inPath) | ||
|
||
config = IngestIndexedReferenceTask.ConfigClass() | ||
config.ra_name = 'ra_icrs' | ||
config.dec_name = 'dec_icrs' | ||
config.mag_column_list = ['a', 'b'] | ||
config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} | ||
config.dataset_config.indexer.active.depth = 4 | ||
|
||
OBS_TEST_DIR = lsst.utils.getPackageDir('obs_test') | ||
obsTestPath = os.path.join(OBS_TEST_DIR, "data", "input") | ||
IngestIndexedReferenceTask.parseAndRun( | ||
args=[obsTestPath, "--output", outPath, skyCatalogFile], | ||
config=config) | ||
|
||
# cleanup files and make the mapper independent of obs_test's path | ||
os.remove(skyCatalogFile) | ||
os.remove(os.path.join(outPath, 'repositoryCfg.yaml')) | ||
with open(os.path.join(outPath, '_mapper'), 'w') as mapperFile: | ||
mapperFile.write('lsst.obs.test.TestMapper\n') | ||
|
||
print("Wrote new catalog to:", outPath) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
lsst.obs.test.TestMapper |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
import lsst.meas.algorithms.ingestIndexReferenceTask | ||
assert type(config)==lsst.meas.algorithms.ingestIndexReferenceTask.IngestIndexedReferenceConfig, 'config is of type %s.%s instead of lsst.meas.algorithms.ingestIndexReferenceTask.IngestIndexedReferenceConfig' % (type(config).__module__, type(config).__name__) | ||
import lsst.meas.algorithms.indexerRegistry | ||
import lsst.meas.algorithms.readTextCatalogTask | ||
# String to pass to the butler to retrieve persisted files. | ||
config.dataset_config.ref_dataset_name='cal_ref_cat' | ||
|
||
# Depth of the HTM tree to make. Default is depth=7 which gives ~ 0.3 sq. deg. per trixel. | ||
config.dataset_config.indexer['HTM'].depth=4 | ||
|
||
config.dataset_config.indexer.name='HTM' | ||
# Number of lines to skip when reading the text reference file. | ||
config.file_reader.header_lines=0 | ||
|
||
# An ordered list of column names to use in ingesting the catalog. With an empty list, column names will be discovered from the first line after the skipped header lines. | ||
config.file_reader.colnames=[] | ||
|
||
# Delimiter to use when reading text reference files. Comma is default. | ||
config.file_reader.delimiter=',' | ||
|
||
# Name of RA column | ||
config.ra_name='ra_icrs' | ||
|
||
# Name of Dec column | ||
config.dec_name='dec_icrs' | ||
|
||
# Name of RA error column | ||
config.ra_err_name=None | ||
|
||
# Name of Dec error column | ||
config.dec_err_name=None | ||
|
||
# The values in the reference catalog are assumed to be in AB magnitudes. List of column names to use for photometric information. At least one entry is required. | ||
config.mag_column_list=['a', 'b'] | ||
|
||
# A map of magnitude column name (key) to magnitude error column (value). | ||
config.mag_err_column_map={'a': 'a_err', 'b': 'b_err'} | ||
|
||
# Name of column stating if satisfactory for photometric calibration (optional). | ||
config.is_photometric_name=None | ||
|
||
# Name of column stating if the object is resolved (optional). | ||
config.is_resolved_name=None | ||
|
||
# Name of column stating if the object is measured to be variable (optional). | ||
config.is_variable_name=None | ||
|
||
# Name of column to use as an identifier (optional). | ||
config.id_name=None | ||
|
||
# Name of proper motion RA column | ||
config.pm_ra_name=None | ||
|
||
# Name of proper motion Dec column | ||
config.pm_dec_name=None | ||
|
||
# Name of proper motion RA error column | ||
config.pm_ra_err_name=None | ||
|
||
# Name of proper motion Dec error column | ||
config.pm_dec_err_name=None | ||
|
||
# Scale factor by which to multiply proper motion values to obtain units of milliarcsec/year | ||
config.pm_scale=1.0 | ||
|
||
# Name of parallax column | ||
config.parallax_name=None | ||
|
||
# Name of parallax error column | ||
config.parallax_err_name=None | ||
|
||
# Scale factor by which to multiply parallax values to obtain units of milliarcsec | ||
config.parallax_scale=1.0 | ||
|
||
# Name of epoch column | ||
config.epoch_name=None | ||
|
||
# Format of epoch column: any value accepted by astropy.time.Time, e.g. 'iso' or 'unix' | ||
config.epoch_format=None | ||
|
||
# Scale of epoch column: any value accepted by astropy.time.Time, e.g. 'utc' | ||
config.epoch_scale=None | ||
|
||
# Extra columns to add to the reference catalog. | ||
config.extra_col_names=[] | ||
|
Oops, something went wrong.