Merge branch 'feature/SIM-1373/compoundClasses'

python/lsst/sims/catUtils/baseCatalogModels/GalaxyModels.py

@@ -1,11 +1,12 @@
 import warnings
 import numpy
 from .BaseCatalogModels import BaseCatalogObj
-from lsst.sims.catalogs.generation.db import ChunkIterator
+from lsst.sims.catalogs.generation.db import ChunkIterator, CompoundCatalogDBObject
 from lsst.sims.utils import ObservationMetaData
 
 __all__ = ["GalaxyObj", "GalaxyTileObj", "GalaxyBulgeObj",
-           "GalaxyDiskObj", "GalaxyAgnObj", "ImageAgnObj", "LensGalaxyObj"]
+           "GalaxyDiskObj", "GalaxyAgnObj", "ImageAgnObj", "LensGalaxyObj",
+           "GalaxyTileCompoundObj"]
 
 class GalaxyObj(BaseCatalogObj):
     """
@@ -218,8 +219,17 @@ def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, const
 
         #We know that galtileid comes back with the query, but we don't want
         #to add it to the query since it's generated on the fly.
-        while 'galtileid' in colnames:
-            colnames.remove('galtileid')
+        #
+        # 25 August 2015
+        # The code below has been modified to remove all column names
+        # that contain 'galtileid.'  This is to accommodate the
+        # CompoundInstanceCatalog and CompoundDBObject classes, which
+        # mangle column names such that they include the objid of the
+        # specific CatalogDBObject that is asking for them.
+        for name in colnames:
+            if 'galtileid' in name:
+                colnames.remove(name)
+
         mappedcolnames = ["%s as %s"%(self.columnMap[x], x) for x in colnames]
         mappedcolnames = ",".join(mappedcolnames)
 
@@ -253,6 +263,27 @@ def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, const
 
         return ChunkIterator(self, query, chunk_size)
 
+
+class GalaxyTileCompoundObj(CompoundCatalogDBObject, GalaxyTileObj):
+    """
+    This is a daughter class of CompoundCatalogDBObject that specifically
+    inherits from GalaxyTileObj.  This is necessary because GalaxyTileObj
+    implements a custom query_columns that executes a stored procedure
+    on fatboy (as opposed to the generic query_columns implemented in
+    CatalogDBObject, which converts self.columns into a SQL
+    query in the most naive way possible).
+    """
+
+    _table_restriction = ['galaxy']
+
+    def _final_pass(self, results):
+        for name in results.dtype.fields:
+            if 'raJ2000' in name or 'decJ2000' in name:
+                results[name] = numpy.radians(results[name])
+
+        return results
+
+
 class GalaxyBulgeObj(GalaxyTileObj):
     objid = 'galaxyBulge'
     componentSubset = 'BULGE'

python/lsst/sims/catUtils/baseCatalogModels/StarModels.py

@@ -1,5 +1,6 @@
 from .BaseCatalogModels import BaseCatalogObj
 from lsst.sims.utils import ObservationMetaData
+from lsst.sims.catalogs.generation.db import CompoundCatalogDBObject
 
 __all__ = ["StarBase", "StarObj", "MsStarObj", "WdStarObj", "RRLyStarObj",
            "BhbStarObj", "EbStarObj", "CepheidStarObj", "EasterEggStarObj",
@@ -239,4 +240,3 @@ class DwarfGalStarObj(StarBase):
                ('radialVelocity', 'vrad'),
                ('variabilityParameters', 'varParamStr', str, 256),
                ('sedFilename', 'sedfilename', unicode, 40)]
-

tests/scratchSpace/.gitignore

@@ -0,0 +1,5 @@
+*.py
+*.pyc
+*.txt
+*.dat
+*.sav

tests/testCompoundCatalogs.py

@@ -0,0 +1,192 @@
+import unittest
+import os
+import lsst.utils.tests as utilsTests
+from lsst.utils import getPackageDir
+
+from lsst.sims.utils import ObservationMetaData
+from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj, GalaxyDiskObj, \
+                                                 GalaxyAgnObj, GalaxyTileCompoundObj, \
+                                                 StarObj
+
+from lsst.sims.catalogs.measures.instance import InstanceCatalog, CompoundInstanceCatalog
+
+class BulgeDiskCatalog(InstanceCatalog):
+    cannot_be_null = ['sedFilename']
+    column_outputs = ['galtileid', 'raJ2000', 'decJ2000',
+                     'componentra', 'componentdec',
+                     'magNorm', 'sedFilename',
+                     'majorAxis', 'minorAxis',
+                     'positionAngle',
+                     'halfLightRadius',
+                     'internalExtinctionModel',
+                     'internalAv', 'internalRv',
+                     ]
+
+
+class AgnCatalog(InstanceCatalog):
+    cannot_be_null = ['sedFilename']
+    column_outputs = ['galtileid', 'raJ2000', 'decJ2000',
+                      'componentra','componentdec',
+                      'magNorm', 'sedFilename',
+                      'variabilityParameters',
+                      ]
+
+
+class StarCatalog(InstanceCatalog):
+    cannot_be_null = ['sedFilename']
+    column_outputs = ['id', 'raJ2000', 'decJ2000',
+                      'glon', 'glat', 'magNorm',
+                      'properMotionRa', 'properMotionDec',
+                      'parallax', 'galacticAv', 'radialVelocity',
+                      'variabilityParameters', 'sedFilename']
+
+class CompoundCatalogTest(unittest.TestCase):
+
+    def setUp(self):
+        self.baseDir = os.path.join(getPackageDir('sims_catUtils'), \
+                                    'tests', 'scratchSpace')
+
+    @unittest.expectedFailure
+    def testGalaxyCatalog(self):
+        """
+        Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks,
+        and agns using both the 'old fashioned way' (one catalog at a time), and
+        using CompoundInstanceCatalog
+        """
+        controlFileName = os.path.join(self.baseDir, 'gal_compound_control.txt')
+        testFileName = os.path.join(self.baseDir, 'gal_compound_test.txt')
+
+        if os.path.exists(controlFileName):
+            os.unlink(controlFileName)
+        if os.path.exists(testFileName):
+            os.unlink(testFileName)
+
+        obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0,
+                                  boundType='circle', boundLength=0.05)
+
+        dbBulge = GalaxyBulgeObj()
+        dbDisk = GalaxyDiskObj()
+        dbAgn = GalaxyAgnObj()
+
+        catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
+        catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
+        catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
+
+        catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000)
+        catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
+        catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
+
+
+        # You need to reinstantiate the catalogs because the process of writing them
+        # above stripped galtileid from their _active_columns, which is the only way
+        # CompoundInstanceCatalog can know that it needs to worry about galtileid
+        catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
+        catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
+        catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
+
+        totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn],
+                                           obs_metadata=obs,
+                                           compoundDBclass=GalaxyTileCompoundObj)
+
+        totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000)
+
+        controlFile = open(controlFileName, 'r')
+        control = controlFile.readlines()
+        controlFile.close()
+
+        testFile = open(testFileName, 'r')
+        test = testFile.readlines()
+        testFile.close()
+
+        for line in control:
+            self.assertTrue(line in test)
+
+        for line in test:
+            self.assertTrue(line in control)
+
+        if os.path.exists(controlFileName):
+            os.unlink(controlFileName)
+
+        if os.path.exists(testFileName):
+            os.unlink(testFileName)
+
+    @unittest.expectedFailure
+    def testGalaxyAndStarCatalog(self):
+        """
+        Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks,
+        agns, and stars using both the 'old fashioned way' (one catalog at a time), and
+        using CompoundInstanceCatalog
+        """
+        controlFileName = os.path.join(self.baseDir, 'galStar_compound_control.txt')
+        testFileName = os.path.join(self.baseDir, 'galStar_compound_test.txt')
+
+        if os.path.exists(controlFileName):
+            os.unlink(controlFileName)
+        if os.path.exists(testFileName):
+            os.unlink(testFileName)
+
+        obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0,
+                                  boundType='circle', boundLength=0.05)
+
+        dbBulge = GalaxyBulgeObj()
+        dbDisk = GalaxyDiskObj()
+        dbAgn = GalaxyAgnObj()
+        dbStar = StarObj()
+
+        catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
+        catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
+        catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
+        catStar = StarCatalog(dbStar, obs_metadata=obs)
+
+        catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000)
+        catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
+        catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
+        catStar.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
+
+
+        # You need to reinstantiate the catalogs because the process of writing them
+        # above stripped galtileid from their _active_columns, which is the only way
+        # CompoundInstanceCatalog can know that it needs to worry about galtileid
+        catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
+        catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
+        catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
+        catStar = StarCatalog(dbStar, obs_metadata=obs)
+
+        totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn, catStar],
+                                           obs_metadata=obs,
+                                           compoundDBclass=GalaxyTileCompoundObj)
+
+        totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000)
+
+        controlFile = open(controlFileName, 'r')
+        control = controlFile.readlines()
+        controlFile.close()
+
+        testFile = open(testFileName, 'r')
+        test = testFile.readlines()
+        testFile.close()
+
+        for line in control:
+            self.assertTrue(line in test)
+
+        for line in test:
+            self.assertTrue(line in control)
+
+        if os.path.exists(controlFileName):
+            os.unlink(controlFileName)
+
+        if os.path.exists(testFileName):
+            os.unlink(testFileName)
+
+
+def suite():
+    utilsTests.init()
+    suites = []
+    suites += unittest.makeSuite(CompoundCatalogTest)
+
+    return unittest.TestSuite(suites)
+
+def run(shouldExit = False):
+    utilsTests.run(suite(), shouldExit)
+if __name__ == "__main__":
+    run(True)

tests/testPhoSimCatalogs.py

@@ -13,7 +13,7 @@
 import json
 import lsst.utils
 from collections import OrderedDict
-from lsst.sims.catalogs.measures.instance import compound
+from lsst.sims.catalogs.measures.instance import compound, CompoundInstanceCatalog
 from lsst.sims.catUtils.utils import testStarsDBObj, testGalaxyDiskDBObj, \
                                      testGalaxyBulgeDBObj, testGalaxyAgnDBObj
 from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogSersic2D, PhoSimCatalogPoint, \
@@ -48,7 +48,9 @@ def testCatalog(self):
         testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata)
         testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata)
 
-        catName = 'phoSimTestCatalog.txt'
+        catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'),
+                               'tests','scratchSpace','phoSimTestCatalog.txt')
+
         testBulge.write_catalog(catName)
         testDisk.write_catalog(catName, write_header=False, write_mode='a')
         testAgn.write_catalog(catName, write_header=False, write_mode='a')
@@ -69,6 +71,46 @@ def testCatalog(self):
         if os.path.exists(catName):
             os.unlink(catName)
 
+
+    def testCompoundCatalog(self):
+        """
+        This test writes a PhoSim input catalog and compares it, one line at a time
+        to a previously written catalog that should be identical.
+
+        This test uses CompoundInstanceCatalog
+        """
+        testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata)
+        testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata)
+        testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata)
+        testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata)
+
+        compoundCatalog = CompoundInstanceCatalog([testBulge, testDisk, testAgn, testStar],
+                                                   obs_metadata=self.obs_metadata)
+
+        self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3)
+
+        catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace',
+                               'phoSimTestCatalog_compound.txt')
+
+        compoundCatalog.write_catalog(catName)
+
+        testFile = open(catName,'r')
+        testLines = testFile.readlines()
+        testFile.close()
+        controlLines = self.baseLineFile.readlines()
+        for line in testLines:
+            msg = '%s not in controlLines' % line
+            self.assertTrue(line in controlLines, msg=msg)
+
+        for line in controlLines:
+            msg = '%s not in testLines'
+            self.assertTrue(line in testLines, msg=msg)
+
+        if os.path.exists(catName):
+            os.unlink(catName)
+
+
+
 def suite():
     utilsTests.init()
     suites = []