Add DR16 versions of APOGEE tables

CHANGELOG.rst

@@ -12,6 +12,7 @@ This document records the main changes to the ``sdssdb`` code.
 * Add ``lite`` materialized views to ``catalogdb``.
 * Add ``catalogdb.gaia_assas_sn_cepheids`` table.
 * Move deprecated tables to ``deprecated`` schema in ``sdss5db``.
+* Add DR16 versions of APOGEE tables.
 
 * :release:`0.4.3 <2020-06-05>`
 * Add schema for ``gaia_dr2_ruwe``.

python/sdssdb/peewee/sdss5db/catalogdb.py

@@ -138,65 +138,59 @@ class Meta:
         table_name = 'sdss_dr13_photoobj'
 
 
-class SDSS_DR14_APOGEE_Visit(CatalogdbModel):
+class SDSS_DR16_APOGEE_Visit(CatalogdbModel):
 
     visit_id = TextField(primary_key=True)
 
     class Meta:
-        table_name = 'sdss_dr14_apogeevisit'
+        table_name = 'sdss_dr16_apogeevisit'
 
 
-class SDSS_DR14_APOGEE_Star(CatalogdbModel):
+class SDSS_DR16_APOGEE_Star(CatalogdbModel):
 
     apstar_id = TextField(primary_key=True)
     apogee_id = TextField()
 
-    visits = ManyToManyField(SDSS_DR14_APOGEE_Visit,
+    gaia = ForeignKeyField(Gaia_DR2,
+                           field='source_id',
+                           column_name='gaia_source_id',
+                           object_id_name='gaia_source_id',
+                           backref='apogee_star')
+
+    visits = ManyToManyField(SDSS_DR16_APOGEE_Visit,
                              through_model=_APOGEE_Star_Visit_Deferred,
                              backref='stars')
 
     class Meta:
-        table_name = 'sdss_dr14_apogeestar'
+        table_name = 'sdss_dr16_apogeestar'
 
 
-class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel):
+class SDSS_DR16_APOGEE_Star_Visit(CatalogdbModel):
 
-    apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star,
+    apstar = ForeignKeyField(SDSS_DR16_APOGEE_Star,
                              backref='+',
                              lazy_load=False)
 
-    visit = ForeignKeyField(SDSS_DR14_APOGEE_Visit,
+    visit = ForeignKeyField(SDSS_DR16_APOGEE_Visit,
                             backref='+',
                             lazy_load=False)
 
     class Meta:
-        table_name = 'sdss_dr14_apogeestarvisit'
-
-
-class SDSS_DR14_ASCAP_Star(CatalogdbModel):
-
-    apstar_id = TextField(primary_key=True)
-    apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star,
-                             backref='ascap_stars')
-
-    class Meta:
-        table_name = 'sdss_dr14_ascapstar'
+        table_name = 'sdss_dr16_apogeestarvisit'
 
 
-class SDSS_DR14_Cannon_Star(CatalogdbModel):
+class SDSS_DR16_APOGEE_Star_AllVisit(CatalogdbModel):
 
-    cannon_id = TextField(primary_key=True)
-
-    @property
-    def apstars(self):
-        """Returns the associated stars in ``SDSSDR14APOGEEStar``."""
+    apstar = ForeignKeyField(SDSS_DR16_APOGEE_Star,
+                             backref='+',
+                             lazy_load=False)
 
-        return (SDSS_DR14_APOGEE_Star
-                .select()
-                .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id))
+    visit = ForeignKeyField(SDSS_DR16_APOGEE_Visit,
+                            backref='+',
+                            lazy_load=False)
 
     class Meta:
-        table_name = 'sdss_dr14_cannonstar'
+        table_name = 'sdss_dr16_apogeestarallvisit'
 
 
 class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel):
@@ -207,9 +201,9 @@ class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel):
     def apstars(self):
         """Returns the stars on `.SDSS_DR14_APOGEE_Star` with matching ``apogee_id``."""
 
-        return (SDSS_DR14_APOGEE_Star
+        return (SDSS_DR16_APOGEE_Star
                 .select()
-                .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id))
+                .where(SDSS_DR16_APOGEE_Star.apogee_id == self.apogee_id))
 
     class Meta:
         table_name = 'sdss_apogeeallstarmerge_r13'
@@ -1035,6 +1029,10 @@ class GAIA_ASSAS_SN_Cepheids(CatalogdbModel):
                            backref='assas')
 
 
+_Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour)
+_APOGEE_Star_Visit_Deferred.set_model(SDSS_DR16_APOGEE_Star_AllVisit)
+
+
 # Add relational tables to namespace.
 if database.connected and database.is_connection_usable():
     all_tables = database.get_tables('catalogdb')

schema/sdss5db/catalogdb/foreignKeys.sql

@@ -139,6 +139,45 @@ ALTER TABLE catalogdb.sdss_dr14_ascapStar
     DEFERRABLE INITIALLY DEFERRED;
 
 
+-- sdss_dr16_apogeeStar
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeStar
+    ADD CONSTRAINT gaia_source_id_fk
+    FOREIGN KEY (gaia_source_id)
+    REFERENCES catalogdb.gaia_dr2_source (source_id)
+    DEFERRABLE INITIALLY DEFERRED;
+
+
+-- sdss_dr16_apogeeStarVisit
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeStarVisit
+    ADD CONSTRAINT visit_id_fk
+    FOREIGN KEY (visit_id)
+    REFERENCES catalogdb.sdss_dr16_apogeeVisit (visit_id)
+    DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeStarVisit
+    ADD CONSTRAINT apstar_id_fk
+    FOREIGN KEY (apstar_id)
+    REFERENCES catalogdb.sdss_dr16_apogeeStar (apstar_id)
+    DEFERRABLE INITIALLY DEFERRED;
+
+
+-- sdss_dr16_apogeeAllStarVisit
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeAllStarVisit
+    ADD CONSTRAINT visit_id_fk
+    FOREIGN KEY (visit_id)
+    REFERENCES catalogdb.sdss_dr16_apogeeVisit (visit_id)
+    DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeAllStarVisit
+    ADD CONSTRAINT apstar_id_fk
+    FOREIGN KEY (apstar_id)
+    REFERENCES catalogdb.sdss_dr16_apogeeStar (apstar_id)
+    DEFERRABLE INITIALLY DEFERRED;
+
+
 -- tic_v8
 
 CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (sdss);

schema/sdss5db/catalogdb/sdssApogeeStar/dr16/sdss_dr16_apogeeStar.sql

@@ -0,0 +1,154 @@
+/*
+
+schema for dr 14 apogeeStar table.
+
+Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx
+
+file is /uufs/chpc.utah.edu/common/home/sdss/dr16/casload/apCSV/spectro/sqlApogeeStar.csv.bz2
+
+apstar_id	varchar	64	 	 	Unique ID for combined star spectrum of form apogee.[telescope].[cs].apstar_version.location_id.apogee_id (Primary key)
+target_id	varchar	64	 	 	target ID (Foreign key, of form [location_id].[apogee_id])
+alt_id	varchar	64	 	 	ID alternate star identification, for apo1m used within reductions
+file	varchar	128	 	 	File base name with combined star spectrum
+apogee_id	varchar	32	 	 	2MASS-style star identification
+telescope	varchar	32	 	 	Telescope where data was taken
+location_id	bigint 8	 	 	Location ID for the field this visit is in (Foreign key)
+field	varchar	100	 	 	Name of field
+ra	float	8	 	deg	Right ascension, J2000
+dec	float	8	 	deg	Declination, J2000
+glon	float	8	 	deg	Galactic longitude
+glat	float	8	 	deg	Galactic latitude
+apogee_target1	bigint	8	 	 	APOGEE target flag (first 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET1)
+apogee_target2	bigint	8	 	 	APOGEE target flag (second 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET2)
+apogee_target3	bigint	8	 	 	APOGEE target flag (third 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET2)
+apogee2_target1	bigint	8	 	 	APOGEE target flag (first 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET1)
+apogee2_target2	bigint	8	 	 	APOGEE target flag (second 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET2)
+apogee2_target3	bigint	8	 	 	APOGEE target flag (third 64 bits) (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_TARGET2)
+min_h	real	4	 	 	minimum H mag for cohort for main survey target
+max_h	real	4	 	 	maximum H mag for cohort for main survey target
+min_jk	real	4	 	 	minimum J-K mag for cohort for main survey target
+max_jk	real	4	 	 	maximum J-K mag for cohort for main survey target
+survey	varchar	100	 	 	Name of survey (apogee/apogee2/apo1m)
+extratarg	bigint	8	 	 	Shorthand flag to denote not a main survey object (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_EXTRATARG)
+nvisits	bigint	8	 	 	Number of visits contributing to the combined spectrum
+commiss	bigint	8	 	 	Set to 1 if this is commissioning data
+snr	real	4	 	 	Median signal-to-noise ratio per pixel
+starflag	bigint	8	 	 	Bit mask with APOGEE star flags; each bit is set here if it is set in any visit (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_STARFLAG)
+andflag	bigint	8	 	 	AND of visit bit mask with APOGEE star flags; each bit is set if set in all visits (see http://www.sdss.org/dr12/algorithms/bitmasks/#APOGEE_STARFLAG)
+vhelio_avg	real	4	 	km/s	Signal-to-noise weighted average of heliocentric radial velocity, as determined relative to combined spectrum, with zeropoint from xcorr of combined spectrum with best-fitting template
+vscatter	real	4	 	km/s	Standard deviation of scatter of individual visit RVs around average
+verr	real	4	 	km/s	Weighted error of heliocentric RV
+verr_med	real	4	 	km/s	Median of individual visit RV errors
+synthvhelio_avg	real	4	 	km/s	Signal-to-noise weighted average of heliocentric radial velocity relative to single best-fit synthetic template
+synthvscatter	real	4	 	km/s	Standard deviation of scatter of visit radial velocities determined from combined spectrum and best-fit synthetic template
+synthverr	real	4	 	km/s	Error in signal-to-noise weighted average of heliocentric radial velocity relative to single best-fit synthetic template
+synthverr_med	real	4	 	km/s	Median of individual visit synthetic RV errors
+rv_teff	real	4	 	deg K	effective temperature from RV template match
+rv_logg	real	4	 	dex	log g from RV template match
+rv_feh	real	4	 	dex	[Fe/H] from RV template match
+rv_ccfwhm	real	4	 	km/s	FWHM of cross-correlation peak from combined vs best-match synthetic spectrum
+rv_autofwhm	real	4	 	km/s	FWHM of auto-correlation of best-match synthetic spectrum
+synthscatter	real	4	 	km/s	scatter between RV using combined spectrum and RV using synthetic spectrum
+stablerv_chi2	real	4	 	 	Chi-squared of RV distribution under assumption of a stable single-valued RV; perhaps not currently useful because of issues with understanding RV errors
+stablerv_rchi2	real	4	 	 	Reduced chi^2 of RV distribution
+chi2_threshold	real	4	 	 	Threshold chi^2 for possible binary determination (not currently valid)
+stablerv_chi2_prob	real	4	 	 	Probability of obtaining observed chi^2 under assumption of stable RV
+apstar_version	varchar	32	 	 	Reduction version of spectrum combination
+gaia_source_id	bigint	8	 	 	GAIA DR2 source id
+gaia_parallax	real	4	 	mas	GAIA DR2 parallax
+gaia_parallax_error	real	4	 	mas	GAIA DR2 parallax uncertainty
+gaia_pmra	real	4	 	mas/yr	GAIA DR2 proper motion in RA
+gaia_pmra_error	real	4	 	mas/yr	GAIA DR2 proper motion in RA uncertainty
+gaia_pmdec	real	4	 	mas/yr	GAIA DR2 proper motion in DEC
+gaia_pmdec_error	real	4	 	mas/yr	GAIA DR2 proper motion in DEC uncertainty
+gaia_phot_g_mean_mag	real	4	 	 	GAIA DR2 g mean magnitude
+gaia_phot_bp_mean_mag	real	4	 	 	GAIA DR2 Bp mean magnitude
+gaia_phot_rp_mean_mag	real	4	 	 	GAIA DR2 Rp mean magnitude
+gaia_radial_velocity	real	4	 	km/s	GAIA DR2 radial velocity
+gaia_radial_velocity_error	real	4	 	km/s	GAIA DR2 radial velocity
+gaia_r_est	real	4	 	pc	GAIA DR2 Bailer Jones r_est
+gaia_r_lo	real	4	 	pc	GAIA DR2 Bailer Jones r_lo
+gaia_r_hi	real	4	 	pc	GAIA DR2 Bailer Jones r_hi
+htmID	bigint	8	 	 	HTM ID
+*/
+
+CREATE TABLE catalogdb.sdss_dr16_apogeeStar(
+    apstar_id TEXT PRIMARY KEY,
+    target_id TEXT,
+    alt_id TEXT,
+    file TEXT,
+    apogee_id TEXT,
+    telescope TEXT,
+    location_id	BIGINT,
+    field TEXT,
+    ra	DOUBLE PRECISION,
+    dec	DOUBLE PRECISION,
+    glon	DOUBLE PRECISION,
+    glat	DOUBLE PRECISION,
+    apogee_target1	BIGINT,
+    apogee_target2	BIGINT,
+    apogee_target3	BIGINT,
+    apogee2_target1	BIGINT,
+    apogee2_target2	BIGINT,
+    apogee2_target3	BIGINT,
+    min_h	REAL,
+    max_h	REAL,
+    min_jk	REAL,
+    max_jk	REAL,
+    survey	TEXT,
+    extratarg	BIGINT,
+    nvisits	BIGINT,
+    commiss	BIGINT,
+    snr	REAL,
+    starflag	BIGINT,
+    andflag	BIGINT,
+    vhelio_avg	REAL,
+    vscatter	REAL,
+    verr	REAL,
+    verr_med	REAL,
+    synthvhelio_avg	REAL,
+    synthvscatter	REAL,
+    synthverr	REAL,
+    synthverr_med	REAL,
+    rv_teff	REAL,
+    rv_logg	REAL,
+    rv_feh	REAL,
+    rv_ccfwhm	REAL,
+    rv_autofwhm	REAL,
+    synthscatter	REAL,
+    stablerv_chi2	REAL,
+    stablerv_rchi2	REAL,
+    chi2_threshold	REAL,
+    stablerv_chi2_prob	REAL,
+    apstar_version	TEXT,
+    gaia_source_id	BIGINT,
+    gaia_parallax	REAL,
+    gaia_parallax_error	REAL,
+    gaia_pmra	REAL,
+    gaia_pmra_error	REAL,
+    gaia_pmdec	REAL,
+    gaia_pmdec_error	REAL,
+    gaia_phot_g_mean_mag	REAL,
+    gaia_phot_bp_mean_mag	REAL,
+    gaia_phot_rp_mean_mag	REAL,
+    gaia_radial_velocity	REAL,
+    gaia_radial_velocity_error	REAL,
+    gaia_r_est	REAL,
+    gaia_r_lo	REAL,
+    gaia_r_hi	REAL,
+    htmID	BIGINT
+);
+
+
+\copy catalogdb.sdss_dr16_apogeeStar FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/sdssApogeeStar/dr16/sqlApogeeStar.csv.bz2' WITH CSV HEADER;
+
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar (q3c_ang2ipix(ra, dec));
+CLUSTER sdss_dr16_apogeeStar_q3c_ang2ipix_idx ON catalogdb.sdss_dr16_apogeeStar;
+ANALYZE catalogdb.sdss_dr16_apogeeStar;
+
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar USING BTREE (ra);
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar USING BTREE (dec);
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar USING BTREE (glon);
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar USING BTREE (glat);
+
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStar USING BTREE (apogee_id);

schema/sdss5db/catalogdb/sdssApogeeStarAllVisit/dr16/sdss_dr16_apogeeAllStarVisit.sql

@@ -0,0 +1,26 @@
+/*
+
+schema for dr 16 apogeeAllStarVisit table.
+
+Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx
+
+file is /uufs/chpc.utah.edu/common/home/sdss/dr16/casload/apCSV/spectro/sqlApogeeStarVisit.csv.bz2
+
+visit_id    varchar 64          Unique ID for visit spectrum, of form apogee.[telescope].[cs].[apred_version].plate.mjd.fiberid
+apstar_id   varchar 64          Unique ID for combined star spectrum of form apogee.[telescope].[cs].apstar_version.location_id.apogee_id
+
+
+*/
+
+CREATE TABLE catalogdb.sdss_dr16_apogeeStarAllVisit (
+    visit_id varchar(64),
+    apstar_id varchar(64)
+);
+
+
+\copy catalogdb.sdss_dr16_apogeeStarAllVisit FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/sdssApogeeStarAllVisit/dr16/sqlApogeeStarAllVisit.csv.bz2' WITH CSV HEADER;
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeStarAllVisit ADD COLUMN pk BIGSERIAL PRIMARY KEY;
+
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStarAllVisit using BTREE (apstar_id);
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStarAllVisit using BTREE (visit_id);

schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr16/sdss_dr16_apogeeStarVisit.sql

@@ -0,0 +1,26 @@
+/*
+
+schema for dr 16 apogeeStarVisit table.
+
+Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx
+
+file is /uufs/chpc.utah.edu/common/home/sdss/dr16/casload/apCSV/spectro/sqlApogeeStarVisit.csv.bz2
+
+visit_id    varchar 64          Unique ID for visit spectrum, of form apogee.[telescope].[cs].[apred_version].plate.mjd.fiberid
+apstar_id   varchar 64          Unique ID for combined star spectrum of form apogee.[telescope].[cs].apstar_version.location_id.apogee_id
+
+
+*/
+
+CREATE TABLE catalogdb.sdss_dr16_apogeeStarVisit(
+    visit_id varchar(64),
+    apstar_id varchar(64)
+);
+
+
+\copy catalogdb.sdss_dr16_apogeeStarVisit FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/sdssApogeeStarVisit/dr16/sqlApogeeStarVisit.csv.bz2' WITH CSV HEADER;
+
+ALTER TABLE catalogdb.sdss_dr16_apogeeStarVisit ADD COLUMN pk BIGSERIAL PRIMARY KEY;
+
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStarVisit using BTREE (apstar_id);
+CREATE INDEX ON catalogdb.sdss_dr16_apogeeStarVisit using BTREE (visit_id);