diff --git a/python/lsst/sims/catUtils/baseCatalogModels/GalaxyModels.py b/python/lsst/sims/catUtils/baseCatalogModels/GalaxyModels.py index 164e50ede..185f6d4f4 100644 --- a/python/lsst/sims/catUtils/baseCatalogModels/GalaxyModels.py +++ b/python/lsst/sims/catUtils/baseCatalogModels/GalaxyModels.py @@ -1,11 +1,12 @@ import warnings import numpy from .BaseCatalogModels import BaseCatalogObj -from lsst.sims.catalogs.generation.db import ChunkIterator +from lsst.sims.catalogs.generation.db import ChunkIterator, CompoundCatalogDBObject from lsst.sims.utils import ObservationMetaData __all__ = ["GalaxyObj", "GalaxyTileObj", "GalaxyBulgeObj", - "GalaxyDiskObj", "GalaxyAgnObj", "ImageAgnObj", "LensGalaxyObj"] + "GalaxyDiskObj", "GalaxyAgnObj", "ImageAgnObj", "LensGalaxyObj", + "GalaxyTileCompoundObj"] class GalaxyObj(BaseCatalogObj): """ @@ -218,8 +219,17 @@ def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, const #We know that galtileid comes back with the query, but we don't want #to add it to the query since it's generated on the fly. - while 'galtileid' in colnames: - colnames.remove('galtileid') + # + # 25 August 2015 + # The code below has been modified to remove all column names + # that contain 'galtileid.' This is to accommodate the + # CompoundInstanceCatalog and CompoundDBObject classes, which + # mangle column names such that they include the objid of the + # specific CatalogDBObject that is asking for them. + for name in colnames: + if 'galtileid' in name: + colnames.remove(name) + mappedcolnames = ["%s as %s"%(self.columnMap[x], x) for x in colnames] mappedcolnames = ",".join(mappedcolnames) @@ -253,6 +263,27 @@ def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, const return ChunkIterator(self, query, chunk_size) + +class GalaxyTileCompoundObj(CompoundCatalogDBObject, GalaxyTileObj): + """ + This is a daughter class of CompoundCatalogDBObject that specifically + inherits from GalaxyTileObj. This is necessary because GalaxyTileObj + implements a custom query_columns that executes a stored procedure + on fatboy (as opposed to the generic query_columns implemented in + CatalogDBObject, which converts self.columns into a SQL + query in the most naive way possible). + """ + + _table_restriction = ['galaxy'] + + def _final_pass(self, results): + for name in results.dtype.fields: + if 'raJ2000' in name or 'decJ2000' in name: + results[name] = numpy.radians(results[name]) + + return results + + class GalaxyBulgeObj(GalaxyTileObj): objid = 'galaxyBulge' componentSubset = 'BULGE' diff --git a/python/lsst/sims/catUtils/baseCatalogModels/StarModels.py b/python/lsst/sims/catUtils/baseCatalogModels/StarModels.py index 041498fef..68acc5a82 100644 --- a/python/lsst/sims/catUtils/baseCatalogModels/StarModels.py +++ b/python/lsst/sims/catUtils/baseCatalogModels/StarModels.py @@ -1,5 +1,6 @@ from .BaseCatalogModels import BaseCatalogObj from lsst.sims.utils import ObservationMetaData +from lsst.sims.catalogs.generation.db import CompoundCatalogDBObject __all__ = ["StarBase", "StarObj", "MsStarObj", "WdStarObj", "RRLyStarObj", "BhbStarObj", "EbStarObj", "CepheidStarObj", "EasterEggStarObj", @@ -239,4 +240,3 @@ class DwarfGalStarObj(StarBase): ('radialVelocity', 'vrad'), ('variabilityParameters', 'varParamStr', str, 256), ('sedFilename', 'sedfilename', unicode, 40)] - diff --git a/tests/scratchSpace/.gitignore b/tests/scratchSpace/.gitignore new file mode 100644 index 000000000..aab56acea --- /dev/null +++ b/tests/scratchSpace/.gitignore @@ -0,0 +1,5 @@ +*.py +*.pyc +*.txt +*.dat +*.sav diff --git a/tests/testCompoundCatalogs.py b/tests/testCompoundCatalogs.py new file mode 100644 index 000000000..6715ccb3d --- /dev/null +++ b/tests/testCompoundCatalogs.py @@ -0,0 +1,192 @@ +import unittest +import os +import lsst.utils.tests as utilsTests +from lsst.utils import getPackageDir + +from lsst.sims.utils import ObservationMetaData +from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj, GalaxyDiskObj, \ + GalaxyAgnObj, GalaxyTileCompoundObj, \ + StarObj + +from lsst.sims.catalogs.measures.instance import InstanceCatalog, CompoundInstanceCatalog + +class BulgeDiskCatalog(InstanceCatalog): + cannot_be_null = ['sedFilename'] + column_outputs = ['galtileid', 'raJ2000', 'decJ2000', + 'componentra', 'componentdec', + 'magNorm', 'sedFilename', + 'majorAxis', 'minorAxis', + 'positionAngle', + 'halfLightRadius', + 'internalExtinctionModel', + 'internalAv', 'internalRv', + ] + + +class AgnCatalog(InstanceCatalog): + cannot_be_null = ['sedFilename'] + column_outputs = ['galtileid', 'raJ2000', 'decJ2000', + 'componentra','componentdec', + 'magNorm', 'sedFilename', + 'variabilityParameters', + ] + + +class StarCatalog(InstanceCatalog): + cannot_be_null = ['sedFilename'] + column_outputs = ['id', 'raJ2000', 'decJ2000', + 'glon', 'glat', 'magNorm', + 'properMotionRa', 'properMotionDec', + 'parallax', 'galacticAv', 'radialVelocity', + 'variabilityParameters', 'sedFilename'] + +class CompoundCatalogTest(unittest.TestCase): + + def setUp(self): + self.baseDir = os.path.join(getPackageDir('sims_catUtils'), \ + 'tests', 'scratchSpace') + + @unittest.expectedFailure + def testGalaxyCatalog(self): + """ + Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks, + and agns using both the 'old fashioned way' (one catalog at a time), and + using CompoundInstanceCatalog + """ + controlFileName = os.path.join(self.baseDir, 'gal_compound_control.txt') + testFileName = os.path.join(self.baseDir, 'gal_compound_test.txt') + + if os.path.exists(controlFileName): + os.unlink(controlFileName) + if os.path.exists(testFileName): + os.unlink(testFileName) + + obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0, + boundType='circle', boundLength=0.05) + + dbBulge = GalaxyBulgeObj() + dbDisk = GalaxyDiskObj() + dbAgn = GalaxyAgnObj() + + catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) + catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) + catAgn = AgnCatalog(dbAgn, obs_metadata=obs) + + catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000) + catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) + catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) + + + # You need to reinstantiate the catalogs because the process of writing them + # above stripped galtileid from their _active_columns, which is the only way + # CompoundInstanceCatalog can know that it needs to worry about galtileid + catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) + catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) + catAgn = AgnCatalog(dbAgn, obs_metadata=obs) + + totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn], + obs_metadata=obs, + compoundDBclass=GalaxyTileCompoundObj) + + totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000) + + controlFile = open(controlFileName, 'r') + control = controlFile.readlines() + controlFile.close() + + testFile = open(testFileName, 'r') + test = testFile.readlines() + testFile.close() + + for line in control: + self.assertTrue(line in test) + + for line in test: + self.assertTrue(line in control) + + if os.path.exists(controlFileName): + os.unlink(controlFileName) + + if os.path.exists(testFileName): + os.unlink(testFileName) + + @unittest.expectedFailure + def testGalaxyAndStarCatalog(self): + """ + Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks, + agns, and stars using both the 'old fashioned way' (one catalog at a time), and + using CompoundInstanceCatalog + """ + controlFileName = os.path.join(self.baseDir, 'galStar_compound_control.txt') + testFileName = os.path.join(self.baseDir, 'galStar_compound_test.txt') + + if os.path.exists(controlFileName): + os.unlink(controlFileName) + if os.path.exists(testFileName): + os.unlink(testFileName) + + obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0, + boundType='circle', boundLength=0.05) + + dbBulge = GalaxyBulgeObj() + dbDisk = GalaxyDiskObj() + dbAgn = GalaxyAgnObj() + dbStar = StarObj() + + catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) + catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) + catAgn = AgnCatalog(dbAgn, obs_metadata=obs) + catStar = StarCatalog(dbStar, obs_metadata=obs) + + catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000) + catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) + catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) + catStar.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) + + + # You need to reinstantiate the catalogs because the process of writing them + # above stripped galtileid from their _active_columns, which is the only way + # CompoundInstanceCatalog can know that it needs to worry about galtileid + catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) + catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) + catAgn = AgnCatalog(dbAgn, obs_metadata=obs) + catStar = StarCatalog(dbStar, obs_metadata=obs) + + totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn, catStar], + obs_metadata=obs, + compoundDBclass=GalaxyTileCompoundObj) + + totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000) + + controlFile = open(controlFileName, 'r') + control = controlFile.readlines() + controlFile.close() + + testFile = open(testFileName, 'r') + test = testFile.readlines() + testFile.close() + + for line in control: + self.assertTrue(line in test) + + for line in test: + self.assertTrue(line in control) + + if os.path.exists(controlFileName): + os.unlink(controlFileName) + + if os.path.exists(testFileName): + os.unlink(testFileName) + + +def suite(): + utilsTests.init() + suites = [] + suites += unittest.makeSuite(CompoundCatalogTest) + + return unittest.TestSuite(suites) + +def run(shouldExit = False): + utilsTests.run(suite(), shouldExit) +if __name__ == "__main__": + run(True) diff --git a/tests/testPhoSimCatalogs.py b/tests/testPhoSimCatalogs.py index 082f20c66..b059b9bdb 100644 --- a/tests/testPhoSimCatalogs.py +++ b/tests/testPhoSimCatalogs.py @@ -13,7 +13,7 @@ import json import lsst.utils from collections import OrderedDict -from lsst.sims.catalogs.measures.instance import compound +from lsst.sims.catalogs.measures.instance import compound, CompoundInstanceCatalog from lsst.sims.catUtils.utils import testStarsDBObj, testGalaxyDiskDBObj, \ testGalaxyBulgeDBObj, testGalaxyAgnDBObj from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogSersic2D, PhoSimCatalogPoint, \ @@ -48,7 +48,9 @@ def testCatalog(self): testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) - catName = 'phoSimTestCatalog.txt' + catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), + 'tests','scratchSpace','phoSimTestCatalog.txt') + testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') @@ -69,6 +71,46 @@ def testCatalog(self): if os.path.exists(catName): os.unlink(catName) + + def testCompoundCatalog(self): + """ + This test writes a PhoSim input catalog and compares it, one line at a time + to a previously written catalog that should be identical. + + This test uses CompoundInstanceCatalog + """ + testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) + testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) + testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) + testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) + + compoundCatalog = CompoundInstanceCatalog([testBulge, testDisk, testAgn, testStar], + obs_metadata=self.obs_metadata) + + self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3) + + catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', + 'phoSimTestCatalog_compound.txt') + + compoundCatalog.write_catalog(catName) + + testFile = open(catName,'r') + testLines = testFile.readlines() + testFile.close() + controlLines = self.baseLineFile.readlines() + for line in testLines: + msg = '%s not in controlLines' % line + self.assertTrue(line in controlLines, msg=msg) + + for line in controlLines: + msg = '%s not in testLines' + self.assertTrue(line in testLines, msg=msg) + + if os.path.exists(catName): + os.unlink(catName) + + + def suite(): utilsTests.init() suites = []