From 8de0688487972aa8dd0019d66a885be9ebd51585 Mon Sep 17 00:00:00 2001 From: Tom Dwelly Date: Wed, 11 Mar 2020 12:19:36 +0000 Subject: [PATCH 001/197] added peewee models for BhmSpidersAgnSuperset and BhmSpidersClustersSuperset --- python/sdssdb/peewee/sdss5db/catalogdb.py | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index ca654fc2..67b55526 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1947,4 +1947,95 @@ class Meta: schema = 'catalogdb' + +# The following model (BhmSpidersGenericSuperset) does not need to be represented as a table in the database +# it is used only as the parent class of BhmSpidersAGNSuperset, BhmSpidersClustersSuperset +# which are real database tables. Am assuming that PeeWee can handle such a scheme without issues. + +class BhmSpidersGenericSuperset(SDSS5dbModel): + + pk = BigAutoField() + + # Parameters derived from eROSITA eSASS catalogue + # Chosen to match X-ray columns defined in eROSITA/SDSS-V MoU (v2.0, April 2019) + ero_version = TextField(index=True, null==True) # string identifying this eROSITA data reduction version + ero_souuid = TextField(index=True, null=True) # string identifying this X-ray source + ero_xray_flux = FloatField(null=True) # X-ray flux, 0.5-8keV band, erg/cm2/s + ero_xray_flux_err = FloatField(null=True) # X-ray flux uncertainty, 0.5-8keV band, erg/cm2/s + ero_ext = FloatField(null=True) # X-ray extent parameter - arcsec + ero_ext_err = FloatField(null=True) # X-ray extent parameter uncertainty - arcsec + ero_ext_like = FloatField(null=True) # X-ray extent likelihood + ero_det_like = FloatField(null=True) # X-ray detection likelihood + ero_ra = DoubleField(index=True, null=True) # X-ray position, RA, ICRS, degrees + ero_dec = DoubleField(index=True, null=True) # X-ray position, Dec, ICRS, degrees + ero_radec_err = FloatField(null=True) # X-ray position uncertainty, arcsec + + # Parameters describing the cross-matching of X-ray to optical/IR catalogue(s) + xmatch_method = TextField(null=True) # 'ML+NWAY', 'LR' , 'SDSS_REDMAPPER', 'LS_REDMAPPER', 'HSC_REDMAPPER', 'MCMF' etc + xmatch_version = TextField(null=True) # version identifier for cross-matching algorithm + xmatch_dist_arcsec = FloatField(null=True) # separation between X-ray position and opt positions - arcsec + xmatch_qual_metric = FloatField(null=True) # measure of quality of xmatch (e.g. p_any for Nway, LR) + xmatch_qual_flags = IntegerField(null=True) # e.g. NWAY match_flag - treat as bitmask + + # Parameters that describe the sub-class + priority rank of the object + target_class = IntegerField(null=True) # TBD, but e.g. 1=AGN candidate, 2=Stellar candidate, + # 3=TDE candidate, 4=Compact object, Cluster BCG=11, Cluster member-galaxy=12, low-z galaxy=13 etc + target_priority = IntegerField(null=True) # allows priority ranking based on info not available in catalogdb + target_has_spec = IntegerField(null=True) # (bitmask) allows flagging of targets that have a redshift from a catalogue that is not listed in catalogdb + + # Parameters derived from the cross-matched opt/IR catalogue + best_opt = TextField(index=True, null=True) # which optical catalogue(and version) provided this counterpart, e.g. 'ls_dr8', 'ps1_dr2' ... + # will also be the origin of the photometry columns below + + ls_id = BigIntegerField(index=True, null=True) # arithmetically derived from ls_release, ls_brickid and ls_objid + # ls_id = ls_objid + ls_brickid * 2**16 + ls_release * 2**40 + # - make sure that we have a common definition within CatalogDB + + ps1_dr2_objid = BigIntegerField(index=True, # Pan-STARRS1-DR2 object id (= ObjectThin.ObjID = StackObjectThin.ObjID) + null=True) # + + gaia_dr2_source_id = BigIntegerField(index=True, # derived from legacysurvey sweeps OPT_REF_ID when OPT_REF_CAT='G2' + null=True) # - SPIDERS team can also pre-match this for photometry from non-LS catalogues + + + opt_ra = DoubleField(index=True, null=True) # included for convenience, but are copied from columns in other tables + opt_dec = DoubleField(index=True, null=True) + opt_pmra = FloatField(null=True) + opt_pmdec = FloatField(null=True) + opt_epoch = FloatField(null=True) + + # For convenience we send a subset of magnitude columns over to the database + # - the full set of columns is available via a database JOIN to e.g. main ls_dr8 catalogue + # Note to self: Be careful with use of modelflux, fiberflux, fiber2flux etc! + opt_modelflux_g = FloatField(null=True) + opt_modelflux_ivar_g = FloatField(null=True) + opt_modelflux_r = FloatField(null=True) + opt_modelflux_ivar_r = FloatField(null=True) + opt_modelflux_r = FloatField(null=True) + opt_modelflux_ivar_r = FloatField(null=True) + opt_modelflux_i = FloatField(null=True) + opt_modelflux_ivar_i = FloatField(null=True) + opt_modelflux_z = FloatField(null=True) + opt_modelflux_ivar_z = FloatField(null=True) + + + class Meta: + table_name = 'bhm_spiders_generic_superset' + schema = 'catalogdb' + + +# Note that following models are currently identical in form, but may well diverge in the future + +class BhmSpidersAgnSuperset(BhmSpidersGenericSuperset): + class Meta: + table_name = 'bhm_spiders_agn_superset' + +class BhmSpidersClustersSuperset(BhmSpidersGenericSuperset): + class Meta: + table_name = 'bhm_spiders_clusters_superset' + + + + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) From 73f76c617d21990bcf12b95576e5c3553376d7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 11 Mar 2020 20:45:15 -0700 Subject: [PATCH 002/197] More clear structure for foreign keys --- schema/sdss5db/catalogdb/foreignKeys.sql | 37 +++++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index d3f4072b..f4ea8756 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -9,11 +9,32 @@ https://www.postgresql.org/docs/8.2/static/sql-altertable.html */ -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT original_ext_source_id_fk FOREIGN KEY (original_ext_source_id) REFERENCES catalogdb.twomass_psc (designation) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT tmass_pts_key_fk FOREIGN KEY (tmass_pts_key) REFERENCES catalogdb.twomass_psc (pts_key) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE catalogdb.sdss_dr14_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT original_ext_source_id_fk + FOREIGN KEY (original_ext_source_id) + REFERENCES catalogdb.twomass_psc (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT tmass_pts_key_fk + FOREIGN KEY (tmass_pts_key) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.sdss_dr14_specobj + ADD CONSTRAINT bestobjid_fk + FOREIGN KEY (bestobjid) + REFERENCES catalogdb.sdss_dr13_photoobj (objid) + ON UPDATE CASCADE ON DELETE CASCADE; From 6c487fabce7a000136963bcec0f699ece7b1b2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 14:09:06 -0700 Subject: [PATCH 003/197] Files for ingesting unWISE --- schema/sdss5db/catalogdb/unWISE/unWISE.sql | 60 +++++++++++++++++++ .../catalogdb/unWISE/unWISE_generate_csv.py | 40 +++++++++++++ schema/sdss5db/catalogdb/unWISE/unWISE_load | 4 ++ setup.cfg | 2 + 4 files changed, 106 insertions(+) create mode 100644 schema/sdss5db/catalogdb/unWISE/unWISE.sql create mode 100644 schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py create mode 100644 schema/sdss5db/catalogdb/unWISE/unWISE_load diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE.sql b/schema/sdss5db/catalogdb/unWISE/unWISE.sql new file mode 100644 index 00000000..f9d06192 --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE.sql @@ -0,0 +1,60 @@ +/* + +Schema for unWISE + +Docs: http://catalog.unwise.me/catalogs.html +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/unwise/release/band-merged + +*/ + +CREATE TABLE catalogdb.unwise ( + x_w1 double precision, + x_w2 double precision, + y_w1 double precision, + y_w2 double precision, + flux_w1 real, + flux_w2 real, + dx_w1 real, + dx_w2 real, + dy_w1 real, + dy_w2 real, + dflux_w1 real, + dflux_w2 real, + qf_w1 real, + qf_w2 real, + rchi2_w1 real, + rchi2_w2 real, + fracflux_w1 real, + fracflux_w2 real, + fluxlbs_w1 real, + fluxlbs_w2 real, + dfluxlbs_w1 real, + dfluxlbs_w2 real, + fwhm_w1 real, + fwhm_w2 real, + spread_model_w1 real, + spread_model_w2 real, + dspread_model_w1 real, + dspread_model_w2 real, + sky_w1 real, + sky_w2 real, + ra12_w1 double precision, + ra12_w2 double precision, + dec12_w1 double precision, + dec12_w2 double precision, + coadd_id bytea, + unwise_detid_w1 bytea, + unwise_detid_w2 bytea, + nm_w1 integer, + nm_w2 integer, + primary12_w1 integer, + primary12_w2 integer, + flags_unwise_w1 integer, + flags_unwise_w2 integer, + flags_info_w1 integer, + flags_info_w2 integer, + ra double precision, + dec double precision, + primary_status integer, + unwise_objid bytea +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py new file mode 100644 index 00000000..a0f2975c --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-12 +# @Filename: unWISE_generate_csv.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + + +import glob + +from astropy import table + + +def unWISE_to_CSV(filename): + """Converts FITS to CSV for ingestion.""" + + new_table = table.Table() + + file_table = table.Table.read(filename) + + for colname in file_table.colnames: + column = file_table[colname] + if column.ndim == 1: + new_table.add_column(column, copy=False) + else: + w1 = table.Column(column[:, 0], colname + '_w1') + w2 = table.Column(column[:, 1], colname + '_w2') + new_table.add_columns([w1, w2]) + + pandas = new_table.to_pandas() + pandas.to_csv(filename + '.csv', header=False, index=False) + + +if __name__ == '__main__': + + files = glob.glob('./*.fits') + for nn, file_ in enumerate(files): + print(f'Converting {file_} ({nn+1}/{len(files)})') + unWISE_to_CSV(file_) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_load b/schema/sdss5db/catalogdb/unWISE/unWISE_load new file mode 100644 index 00000000..df6aebbf --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_load @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +ls ./*.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.unwise FROM '{}' WITH DELIMITER ',' NULL '\N' CSV;;\"" diff --git a/setup.cfg b/setup.cfg index cd23693e..f726b249 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,8 @@ dev = pytest-cov>=2.4.0 pytest-sugar>=0.8.0 pydot>=1.4.1 + astropy>=4.0.0 + pandas>=1.0.0 docs = Sphinx>=1.8.0 sphinx_bootstrap_theme>=0.4.12 From 7d7003446e6ae767a755466fa577384ceaf99ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 14:13:03 -0700 Subject: [PATCH 004/197] Use multiprocessing --- schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py index a0f2975c..81a1bd8a 100644 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py @@ -8,6 +8,7 @@ import glob +import multiprocessing from astropy import table @@ -35,6 +36,6 @@ def unWISE_to_CSV(filename): if __name__ == '__main__': files = glob.glob('./*.fits') - for nn, file_ in enumerate(files): - print(f'Converting {file_} ({nn+1}/{len(files)})') - unWISE_to_CSV(file_) + + pool = multiprocessing.Pool(25) + pool.map(unWISE_to_CSV, files) From 7f3d90e0ec7dc45663c2189b9076e5d1bbffa3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 01:05:40 +0000 Subject: [PATCH 005/197] Change permissionins --- schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py | 0 schema/sdss5db/catalogdb/unWISE/unWISE_load | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py mode change 100644 => 100755 schema/sdss5db/catalogdb/unWISE/unWISE_load diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py old mode 100644 new mode 100755 diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_load b/schema/sdss5db/catalogdb/unWISE/unWISE_load old mode 100644 new mode 100755 From 5703f79924ff2831fcb82995545048647840ccf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 18:52:38 -0700 Subject: [PATCH 006/197] Replace bytea with text --- schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py new file mode 100644 index 00000000..e69de29b From 27ae7fbb7bcccda390f3269f81a23e27cf1bfdff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 18:55:08 -0700 Subject: [PATCH 007/197] Replace bytea with text --- schema/sdss5db/catalogdb/unWISE/unWISE.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE.sql b/schema/sdss5db/catalogdb/unWISE/unWISE.sql index f9d06192..4662be23 100644 --- a/schema/sdss5db/catalogdb/unWISE/unWISE.sql +++ b/schema/sdss5db/catalogdb/unWISE/unWISE.sql @@ -42,9 +42,9 @@ CREATE TABLE catalogdb.unwise ( ra12_w2 double precision, dec12_w1 double precision, dec12_w2 double precision, - coadd_id bytea, - unwise_detid_w1 bytea, - unwise_detid_w2 bytea, + coadd_id TEXT, + unwise_detid_w1 TEXT, + unwise_detid_w2 TEXT, nm_w1 integer, nm_w2 integer, primary12_w1 integer, @@ -56,5 +56,5 @@ CREATE TABLE catalogdb.unwise ( ra double precision, dec double precision, primary_status integer, - unwise_objid bytea + unwise_objid TEXT ) WITHOUT OIDS; From 4b3fc90249c6a87188d6d6aa1653a84b24d359d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 18:56:03 -0700 Subject: [PATCH 008/197] Remove legacy survey file that doesnt go in this branch --- schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py deleted file mode 100644 index e69de29b..00000000 From 8ed6bfd4cd57e25a918014967dc8b9bac7bbfdb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 19:41:53 -0700 Subject: [PATCH 009/197] Changes to unWISE scripts --- schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py | 9 ++++++--- schema/sdss5db/catalogdb/unWISE/unWISE_load | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py index 81a1bd8a..267bdef1 100755 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py @@ -29,13 +29,16 @@ def unWISE_to_CSV(filename): w2 = table.Column(column[:, 1], colname + '_w2') new_table.add_columns([w1, w2]) - pandas = new_table.to_pandas() - pandas.to_csv(filename + '.csv', header=False, index=False) + new_table.write(filename + '.csv', format='ascii.fast_no_header', + delimiter=',', overwrite=True) + # pandas = new_table.to_pandas() + # print(pandas['unwise_objid']) + # pandas.to_csv(filename + '.csv', header=False, index=False) if __name__ == '__main__': - files = glob.glob('./*.fits') + files = glob.glob('./*.cat.fits') pool = multiprocessing.Pool(25) pool.map(unWISE_to_CSV, files) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_load b/schema/sdss5db/catalogdb/unWISE/unWISE_load index df6aebbf..326d8524 100755 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_load +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_load @@ -1,4 +1,4 @@ -#! /usr/bin/bash +#!/usr/bin/env bash # encoding: utf-8 -ls ./*.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.unwise FROM '{}' WITH DELIMITER ',' NULL '\N' CSV;;\"" +ls ./*cat.fits.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.unwise FROM '{}' WITH DELIMITER ',' NULL '\N' CSV;;\"" From be2f87fe2897067ffb20eb7ce5392152d23ae7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 19:46:48 -0700 Subject: [PATCH 010/197] Remove comments --- schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py index 267bdef1..44e3aee3 100755 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py @@ -31,9 +31,6 @@ def unWISE_to_CSV(filename): new_table.write(filename + '.csv', format='ascii.fast_no_header', delimiter=',', overwrite=True) - # pandas = new_table.to_pandas() - # print(pandas['unwise_objid']) - # pandas.to_csv(filename + '.csv', header=False, index=False) if __name__ == '__main__': From ec4c2d8fce4702ab515a5d9dea76bb4d501894d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 21:11:06 -0700 Subject: [PATCH 011/197] Initial legacy survey ingestion files --- .../sdssdb/utils/{database.py => ingest.py} | 3 +- .../catalogdb/legacy_survey/legacy_survey.sql | 130 ++++++++++++++++++ .../legacy_survey/legacy_survey_load.py | 31 +++++ 3 files changed, 162 insertions(+), 2 deletions(-) rename python/sdssdb/utils/{database.py => ingest.py} (99%) create mode 100644 schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql create mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py diff --git a/python/sdssdb/utils/database.py b/python/sdssdb/utils/ingest.py similarity index 99% rename from python/sdssdb/utils/database.py rename to python/sdssdb/utils/ingest.py index c0385c65..a3c35118 100644 --- a/python/sdssdb/utils/database.py +++ b/python/sdssdb/utils/ingest.py @@ -3,7 +3,7 @@ # # @Author: José Sánchez-Gallego (gallegoj@uw.edu) # @Date: 2019-09-21 -# @Filename: load.py +# @Filename: ingest.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) # # @Last modified by: José Sánchez-Gallego (gallegoj@uw.edu) @@ -214,7 +214,6 @@ def copy_data(data, connection, table_name, schema=None, chunk_size=10000, row = data[ii] - # Adds the pk row_data = [] for col_value in row: diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql new file mode 100644 index 00000000..520f3c24 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql @@ -0,0 +1,130 @@ +/* + +Schema for Legacy Survey. + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/legacysurvey/dr8 + +*/ + +CREATE TABLE catalogdb.legacy_survey ( + release integer, + brickid bigint, + brickname text, + objid bigint, + type text, + ra double precision, + dec double precision, + ra_ivar real, + dec_ivar real, + dchisq real[5], + ebv real, + flux_g real, + flux_r real, + flux_z real, + flux_w1 real, + flux_w2 real, + flux_w3 real, + flux_w4 real, + flux_ivar_g real, + flux_ivar_r real, + flux_ivar_z real, + flux_ivar_w1 real, + flux_ivar_w2 real, + flux_ivar_w3 real, + flux_ivar_w4 real, + mw_transmission_g real, + mw_transmission_r real, + mw_transmission_z real, + mw_transmission_w1 real, + mw_transmission_w2 real, + mw_transmission_w3 real, + mw_transmission_w4 real, + nobs_g integer, + nobs_r integer, + nobs_z integer, + nobs_w1 integer, + nobs_w2 integer, + nobs_w3 integer, + nobs_w4 integer, + rchisq_g real, + rchisq_r real, + rchisq_z real, + rchisq_w1 real, + rchisq_w2 real, + rchisq_w3 real, + rchisq_w4 real, + fracflux_g real, + fracflux_r real, + fracflux_z real, + fracflux_w1 real, + fracflux_w2 real, + fracflux_w3 real, + fracflux_w4 real, + fracmasked_g real, + fracmasked_r real, + fracmasked_z real, + fracin_g real, + fracin_r real, + fracin_z real, + anymask_g integer, + anymask_r integer, + anymask_z integer, + allmask_g integer, + allmask_r integer, + allmask_z integer, + wisemask_w1 smallint, + wisemask_w2 smallint, + psfsize_g real, + psfsize_r real, + psfsize_z real, + psfdepth_g real, + psfdepth_r real, + psfdepth_z real, + galdepth_g real, + galdepth_r real, + galdepth_z real, + psfdepth_w1 real, + psfdepth_w2 real, + wise_coadd_id text, + fracdev real, + fracdev_ivar real, + shapedev_r real, + shapedev_r_ivar real, + shapedev_e1 real, + shapedev_e1_ivar real, + shapedev_e2 real, + shapedev_e2_ivar real, + shapeexp_r real, + shapeexp_r_ivar real, + shapeexp_e1 real, + shapeexp_e1_ivar real, + shapeexp_e2 real, + shapeexp_e2_ivar real, + fiberflux_g real, + fiberflux_r real, + fiberflux_z real, + fibertotflux_g real, + fibertotflux_r real, + fibertotflux_z real, + ref_cat text, + ref_id bigint, + ref_epoch real, + gaia_phot_g_mean_mag real, + gaia_phot_g_mean_flux_over_error real, + gaia_phot_bp_mean_mag real, + gaia_phot_bp_mean_flux_over_error real, + gaia_phot_rp_mean_mag real, + gaia_phot_rp_mean_flux_over_error real, + gaia_astrometric_excess_noise real, + gaia_duplicated_source bool, + gaia_phot_bp_rp_excess_factor real, + gaia_astrometric_sigma5d_max real, + gaia_astrometric_params_solved smallint, + parallax real, + parallax_ivar real, + pmra real, + pmra_ivar real, + pmdec real, + pmdec_ivar real, + maskbits integer +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py new file mode 100755 index 00000000..aac60069 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-12 +# @Filename: legacy_survey_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import multiprocessing + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +assert database.connected, 'database is not connected' + + +def ingest(file_): + data = astropy.table.Table.read(file_) + copy_data(data, database, 'legacy_survey', schema='catalogdb') + + +if __name__ == '__main__': + + files = glob.glob('sweep*.fits') + + pool = multiprocessing.Pool(25) + pool.map(ingest, files) From c0f62b11534e974192bb30cc7c83bac69485c202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 22:57:11 -0700 Subject: [PATCH 012/197] Tweak packaging to use sdsstools --- docs/sphinx/conf.py | 5 ++-- docs/sphinx/installation.rst | 6 ++-- python/sdssdb/__init__.py | 56 +++++++----------------------------- setup.cfg | 28 ++---------------- 4 files changed, 20 insertions(+), 75 deletions(-) diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index b8ecf614..41f49fb9 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -49,7 +49,7 @@ # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.autosummary', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.mathjax', - 'sphinx.ext.intersphinx', 'releases'] + 'sphinx.ext.intersphinx', 'sdsstools.releases'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -133,7 +133,8 @@ napoleon_use_rtype = False napoleon_use_ivar = True -rst_epilog = """ +rst_epilog = f""" +.. |sdssdb_version| replace:: {__version__} .. |numpy_array| replace:: Numpy array """ diff --git a/docs/sphinx/installation.rst b/docs/sphinx/installation.rst index 9613097e..026369e5 100644 --- a/docs/sphinx/installation.rst +++ b/docs/sphinx/installation.rst @@ -10,11 +10,13 @@ Installation pip install sdssdb -then you can check that it works by running a python terminal and doing :: +then you can check that it works by running a python terminal and doing + +.. parsed-literal:: >>> import sdssdb >>> sdssdb.__version__ - 0.3.3dev + |sdssdb_version| If you are working from Utah, ``sdssdb`` is installed as a module and you should be able to do :: diff --git a/python/sdssdb/__init__.py b/python/sdssdb/__init__.py index 2c4d267f..a180f81f 100644 --- a/python/sdssdb/__init__.py +++ b/python/sdssdb/__init__.py @@ -1,68 +1,34 @@ # encoding: utf-8 -# flake8: noqa from __future__ import absolute_import, division, print_function, unicode_literals -import os import warnings -from pkg_resources import parse_version -import yaml +from sdsstools import get_config, get_logger, get_package_version -# Inits the logging system. Only shell logging, and exception and warning catching. -# File logging can be started by calling log.start_file_logger(name). -from .misc import log # noqa - - -def merge(user, default): - """Merges a user configuration with the default one.""" - - if isinstance(user, dict) and isinstance(default, dict): - for kk, vv in default.items(): - if kk not in user: - user[kk] = vv - else: - user[kk] = merge(user[kk], vv) - - return user +warnings.filterwarnings( + 'ignore', '.*Skipped unsupported reflection of expression-based index .*q3c.*') NAME = 'sdssdb' +__version__ = get_package_version(path='./', package_name=NAME) -yaml_kwds = dict() -if parse_version(yaml.__version__) >= parse_version('5.1'): - yaml_kwds.update(Loader=yaml.FullLoader) - +log = get_logger(NAME) # Loads config -config_file = os.path.dirname(__file__) + '/etc/{0}.yml'.format(NAME) -config = yaml.load(open(config_file), **yaml_kwds) - -# If there is a custom configuration file, updates the defaults using it. -custom_config_fn = os.path.expanduser('~/.{0}/{0}.yml'.format(NAME)) -if os.path.exists(custom_config_fn): - config = merge(yaml.load(open(custom_config_fn), **yaml_kwds), config) - -# from sdssdb.sqlalchemy import db -# config['db'] = db - -warnings.filterwarnings( - 'ignore', '.*Skipped unsupported reflection of expression-based index .*q3c.*') - - -__version__ = '0.3.3dev' +config = get_config(NAME, user_path='~/.sdssdb/sdssdb.yml') try: - import peewee + import peewee # noqa _peewee = True except ImportError: _peewee = False try: - import sqlalchemy + import sqlalchemy # noqa _sqla = True except ImportError: _sqla = False @@ -71,9 +37,7 @@ def merge(user, default): 'Install at least one of them to use sdssdb.') -from .connection import DatabaseConnection - if _peewee: - from .connection import PeeweeDatabaseConnection + from .connection import PeeweeDatabaseConnection # noqa if _sqla: - from .connection import SQLADatabaseConnection + from .connection import SQLADatabaseConnection # noqa diff --git a/setup.cfg b/setup.cfg index f726b249..d34b5c5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ install_requires = psycopg2-binary>=2.7.7 six>=1.12.0 peewee>=3.9.6 + sdsstools>=0.1.5 [options.packages.find] where = @@ -50,9 +51,9 @@ sdssdb = [options.extras_require] all = - astropy>=3.2.1 sqlalchemy>=1.3.6 progressbar2>=3.46.1 + sdssdb[dev,docs] dev = pytest>=3.0.7 pytest-cov>=2.4.0 @@ -60,11 +61,10 @@ dev = pydot>=1.4.1 astropy>=4.0.0 pandas>=1.0.0 + sdsstools[dev]>=0.1.5 docs = Sphinx>=1.8.0 sphinx_bootstrap_theme>=0.4.12 - releases>=1.6.1 - semantic-version==2.6.0 # Un-pin when releases gets updated. [isort] line_length = 99 @@ -80,27 +80,5 @@ ignore = W505 max-line-length = 99 -[bumpversion] -current_version = 0.3.3dev -commit = True -tag = False -tag_name = {new_version} -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? -serialize = - {major}.{minor}.{patch}{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = alpha -values = - dev - alpha - -[bumpversion:file:python/sdssdb/__init__.py] - -[bumpversion:file:setup.cfg] - -[bumpversion:file:docs/sphinx/installation.rst] - [coverage:run] branch = true From 949ddfeafe1cbcc59ea64818ffb87766e13aac42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 22:57:42 -0700 Subject: [PATCH 013/197] Add deprecation warnings in catalogdb for SQLA --- python/sdssdb/sqlalchemy/sdss5db/catalogdb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py index 72a145af..bc5c9c9d 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py @@ -10,13 +10,20 @@ from __future__ import absolute_import, division, print_function +import warnings + from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr from sqlalchemy.orm import relationship +from sdssdb.core.exceptions import SdssdbWarning from sdssdb.sqlalchemy.sdss5db import SDSS5Base, database +warnings.warn('support for catalogdb in SQLalchemy is incomplete. Use Peewee instead.', + SdssdbWarning) + + class Base(AbstractConcreteBase, SDSS5Base): __abstract__ = True _schema = 'catalogdb' From 712c624251407763b17952890ec83ea3826de7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 23:21:18 -0700 Subject: [PATCH 014/197] More unWISE implementation --- python/sdssdb/peewee/sdss5db/catalogdb.py | 56 +++++++++++ schema/sdss5db/catalogdb/unWISE/unWISE.sql | 92 +++++++++---------- .../sdss5db/catalogdb/unWISE/unWISE_index.sql | 4 + 3 files changed, 106 insertions(+), 46 deletions(-) create mode 100644 schema/sdss5db/catalogdb/unWISE/unWISE_index.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index ca654fc2..81672479 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1947,4 +1947,60 @@ class Meta: schema = 'catalogdb' +class unWISE(SDSS5dbModel): + + unwise_objid = TextField(primary_key=True) + x_w1 = DoubleField(null=True) + x_w2 = DoubleField(null=True) + y_w1 = DoubleField(null=True) + y_w2 = DoubleField(null=True) + flux_w1 = FloatField(null=True) + flux_w2 = FloatField(null=True) + dx_w1 = FloatField(null=True) + dx_w2 = FloatField(null=True) + dy_w1 = FloatField(null=True) + dy_w2 = FloatField(null=True) + dflux_w1 = FloatField(null=True) + dflux_w2 = FloatField(null=True) + qf_w1 = FloatField(null=True) + qf_w2 = FloatField(null=True) + rchi2_w1 = FloatField(null=True) + rchi2_w2 = FloatField(null=True) + fracflux_w1 = FloatField(null=True) + fracflux_w2 = FloatField(null=True) + fluxlbs_w1 = FloatField(null=True) + fluxlbs_w2 = FloatField(null=True) + dfluxlbs_w1 = FloatField(null=True) + dfluxlbs_w2 = FloatField(null=True) + fwhm_w1 = FloatField(null=True) + fwhm_w2 = FloatField(null=True) + spread_model_w1 = FloatField(null=True) + spread_model_w2 = FloatField(null=True) + dspread_model_w1 = FloatField(null=True) + dspread_model_w2 = FloatField(null=True) + sky_w1 = FloatField(null=True) + sky_w2 = FloatField(null=True) + ra12_w1 = DoubleField(null=True) + ra12_w2 = DoubleField(null=True) + dec12_w1 = DoubleField(null=True) + dec12_w2 = DoubleField(null=True) + coadd_id = TextField(null=True) + unwise_detid_w1 = TextField(null=True) + unwise_detid_w2 = TextField(null=True) + nm_w1 = IntegerField(null=True) + nm_w2 = IntegerField(null=True) + primary12_w1 = IntegerField(null=True) + primary12_w2 = IntegerField(null=True) + flags_unwise_w1 = IntegerField(null=True) + flags_unwise_w2 = IntegerField(null=True) + flags_info_w1 = IntegerField(null=True) + flags_info_w2 = IntegerField(null=True) + ra = DoubleField(null=True) + dec = DoubleField(null=True) + primary_status = IntegerField(null=True) + + class Meta: + table_name = 'unwise' + schema = 'catalogdb' + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE.sql b/schema/sdss5db/catalogdb/unWISE/unWISE.sql index 4662be23..5543ae22 100644 --- a/schema/sdss5db/catalogdb/unWISE/unWISE.sql +++ b/schema/sdss5db/catalogdb/unWISE/unWISE.sql @@ -8,53 +8,53 @@ Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/unwise/relea */ CREATE TABLE catalogdb.unwise ( - x_w1 double precision, - x_w2 double precision, - y_w1 double precision, - y_w2 double precision, - flux_w1 real, - flux_w2 real, - dx_w1 real, - dx_w2 real, - dy_w1 real, - dy_w2 real, - dflux_w1 real, - dflux_w2 real, - qf_w1 real, - qf_w2 real, - rchi2_w1 real, - rchi2_w2 real, - fracflux_w1 real, - fracflux_w2 real, - fluxlbs_w1 real, - fluxlbs_w2 real, - dfluxlbs_w1 real, - dfluxlbs_w2 real, - fwhm_w1 real, - fwhm_w2 real, - spread_model_w1 real, - spread_model_w2 real, - dspread_model_w1 real, - dspread_model_w2 real, - sky_w1 real, - sky_w2 real, - ra12_w1 double precision, - ra12_w2 double precision, - dec12_w1 double precision, - dec12_w2 double precision, + x_w1 DOUBLE PRECISION, + x_w2 DOUBLE PRECISION, + y_w1 DOUBLE PRECISION, + y_w2 DOUBLE PRECISION, + flux_w1 REAL, + flux_w2 REAL, + dx_w1 REAL, + dx_w2 REAL, + dy_w1 REAL, + dy_w2 REAL, + dflux_w1 REAL, + dflux_w2 REAL, + qf_w1 REAL, + qf_w2 REAL, + rchi2_w1 REAL, + rchi2_w2 REAL, + fracflux_w1 REAL, + fracflux_w2 REAL, + fluxlbs_w1 REAL, + fluxlbs_w2 REAL, + dfluxlbs_w1 REAL, + dfluxlbs_w2 REAL, + fwhm_w1 REAL, + fwhm_w2 REAL, + spread_model_w1 REAL, + spread_model_w2 REAL, + dspread_model_w1 REAL, + dspread_model_w2 REAL, + sky_w1 REAL, + sky_w2 REAL, + ra12_w1 DOUBLE PRECISION, + ra12_w2 DOUBLE PRECISION, + dec12_w1 DOUBLE PRECISION, + dec12_w2 DOUBLE PRECISION, coadd_id TEXT, unwise_detid_w1 TEXT, unwise_detid_w2 TEXT, - nm_w1 integer, - nm_w2 integer, - primary12_w1 integer, - primary12_w2 integer, - flags_unwise_w1 integer, - flags_unwise_w2 integer, - flags_info_w1 integer, - flags_info_w2 integer, - ra double precision, - dec double precision, - primary_status integer, - unwise_objid TEXT + nm_w1 INTEGER, + nm_w2 INTEGER, + primary12_w1 INTEGER, + primary12_w2 INTEGER, + flags_unwise_w1 INTEGER, + flags_unwise_w2 INTEGER, + flags_info_w1 INTEGER, + flags_info_w2 INTEGER, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + primary_status INTEGER, + unwise_objid TEXT PRIMARY KEY ) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql new file mode 100644 index 00000000..465cba3c --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql @@ -0,0 +1,4 @@ + +CREATE INDEX on catalogdb.unwise (q3c_ang2ipix(ra, dec)); +CLUSTER unwise_q3c_ang2ipix_idx on catalogdb.unwise; +ANALYZE catalogdb.unwise; From 707534d2e8f1325bb01bd942719ec8094dae45de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 23:21:30 -0700 Subject: [PATCH 015/197] Fix reference to utils.ingest in docs --- docs/sphinx/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index dd781224..111d9a58 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -32,7 +32,7 @@ SQLAlchemy Utils ----- -.. automodule:: sdssdb.utils.database +.. automodule:: sdssdb.utils.ingest :members: :show-inheritance: From 3e93f62ec429f68170fbd4f96e884ae0a2510463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 23:24:51 -0700 Subject: [PATCH 016/197] Legacy Survey updates --- python/sdssdb/peewee/sdss5db/catalogdb.py | 136 +++++++++- .../catalogdb/legacy_survey/legacy_index.sql | 4 + .../catalogdb/legacy_survey/legacy_survey.sql | 240 +++++++++--------- 3 files changed, 257 insertions(+), 123 deletions(-) create mode 100644 schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 81672479..273560e9 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -10,9 +10,9 @@ # @Last modified time: 2019-09-23 00:54:42 -from peewee import (AutoField, BigAutoField, BigIntegerField, BooleanField, CharField, - DateField, DecimalField, DeferredThroughModel, DoubleField, - FloatField, ForeignKeyField, IntegerField, ManyToManyField, TextField) +from peewee import (AutoField, BigAutoField, BigIntegerField, BooleanField, CharField, DateField, + DecimalField, DeferredThroughModel, DoubleField, FloatField, ForeignKeyField, + IntegerField, ManyToManyField, SmallIntegerField, TextField) from playhouse.postgres_ext import ArrayField from . import SDSS5dbModel, database # noqa @@ -2003,4 +2003,134 @@ class Meta: table_name = 'unwise' schema = 'catalogdb' + +class LegacySurvey(SDSS5dbModel): + + objid = BigIntegerField(primary_key=True) + release = IntegerField(null=True) + brickid = BigIntegerField(null=True) + brickname = TextField(null=True) + type = TextField(null=True) + ra = DoubleField(null=True) + dec = DoubleField(null=True) + ra_ivar = FloatField(null=True) + dec_ivar = FloatField(null=True) + dchisq = ArrayField(field_class=FloatField, null=True) + ebv = FloatField(null=True) + flux_g = FloatField(null=True) + flux_r = FloatField(null=True) + flux_z = FloatField(null=True) + flux_w1 = FloatField(null=True) + flux_w2 = FloatField(null=True) + flux_w3 = FloatField(null=True) + flux_w4 = FloatField(null=True) + flux_ivar_g = FloatField(null=True) + flux_ivar_r = FloatField(null=True) + flux_ivar_z = FloatField(null=True) + flux_ivar_w1 = FloatField(null=True) + flux_ivar_w2 = FloatField(null=True) + flux_ivar_w3 = FloatField(null=True) + flux_ivar_w4 = FloatField(null=True) + mw_transmission_g = FloatField(null=True) + mw_transmission_r = FloatField(null=True) + mw_transmission_z = FloatField(null=True) + mw_transmission_w1 = FloatField(null=True) + mw_transmission_w2 = FloatField(null=True) + mw_transmission_w3 = FloatField(null=True) + mw_transmission_w4 = FloatField(null=True) + nobs_g = IntegerField(null=True) + nobs_r = IntegerField(null=True) + nobs_z = IntegerField(null=True) + nobs_w1 = IntegerField(null=True) + nobs_w2 = IntegerField(null=True) + nobs_w3 = IntegerField(null=True) + nobs_w4 = IntegerField(null=True) + rchisq_g = FloatField(null=True) + rchisq_r = FloatField(null=True) + rchisq_z = FloatField(null=True) + rchisq_w1 = FloatField(null=True) + rchisq_w2 = FloatField(null=True) + rchisq_w3 = FloatField(null=True) + rchisq_w4 = FloatField(null=True) + fracflux_g = FloatField(null=True) + fracflux_r = FloatField(null=True) + fracflux_z = FloatField(null=True) + fracflux_w1 = FloatField(null=True) + fracflux_w2 = FloatField(null=True) + fracflux_w3 = FloatField(null=True) + fracflux_w4 = FloatField(null=True) + fracmasked_g = FloatField(null=True) + fracmasked_r = FloatField(null=True) + fracmasked_z = FloatField(null=True) + fracin_g = FloatField(null=True) + fracin_r = FloatField(null=True) + fracin_z = FloatField(null=True) + anymask_g = IntegerField(null=True) + anymask_r = IntegerField(null=True) + anymask_z = IntegerField(null=True) + allmask_g = IntegerField(null=True) + allmask_r = IntegerField(null=True) + allmask_z = IntegerField(null=True) + wisemask_w1 = SmallIntegerField(null=True) + wisemask_w2 = SmallIntegerField(null=True) + psfsize_g = FloatField(null=True) + psfsize_r = FloatField(null=True) + psfsize_z = FloatField(null=True) + psfdepth_g = FloatField(null=True) + psfdepth_r = FloatField(null=True) + psfdepth_z = FloatField(null=True) + galdepth_g = FloatField(null=True) + galdepth_r = FloatField(null=True) + galdepth_z = FloatField(null=True) + psfdepth_w1 = FloatField(null=True) + psfdepth_w2 = FloatField(null=True) + wise_coadd_id = TextField(null=True) + fracdev = FloatField(null=True) + fracdev_ivar = FloatField(null=True) + shapedev_r = FloatField(null=True) + shapedev_r_ivar = FloatField(null=True) + shapedev_e1 = FloatField(null=True) + shapedev_e1_ivar = FloatField(null=True) + shapedev_e2 = FloatField(null=True) + shapedev_e2_ivar = FloatField(null=True) + shapeexp_r = FloatField(null=True) + shapeexp_r_ivar = FloatField(null=True) + shapeexp_e1 = FloatField(null=True) + shapeexp_e1_ivar = FloatField(null=True) + shapeexp_e2 = FloatField(null=True) + shapeexp_e2_ivar = FloatField(null=True) + fiberflux_g = FloatField(null=True) + fiberflux_r = FloatField(null=True) + fiberflux_z = FloatField(null=True) + fibertotflux_g = FloatField(null=True) + fibertotflux_r = FloatField(null=True) + fibertotflux_z = FloatField(null=True) + ref_cat = TextField(null=True) + ref_id = BigIntegerField(null=True) + ref_epoch = FloatField(null=True) + gaia_phot_g_mean_mag = FloatField(null=True) + gaia_phot_g_mean_flux_over_error = FloatField(null=True) + gaia_phot_bp_mean_mag = FloatField(null=True) + gaia_phot_bp_mean_flux_over_error = FloatField(null=True) + gaia_phot_rp_mean_mag = FloatField(null=True) + gaia_phot_rp_mean_flux_over_error = FloatField(null=True) + gaia_astrometric_excess_noise = FloatField(null=True) + gaia_duplicated_source = BooleanField(null=True) + gaia_phot_bp_rp_excess_factor = FloatField(null=True) + gaia_astrometric_sigma5d_max = FloatField(null=True) + gaia_astrometric_params_solved = SmallIntegerField(null=True) + parallax = FloatField(null=True) + parallax_ivar = FloatField(null=True) + pmra = FloatField(null=True) + pmra_ivar = FloatField(null=True) + pmdec = FloatField(null=True) + pmdec_ivar = FloatField(null=True) + maskbits = IntegerField(null=True) + + class Meta: + table_name = 'legacy_survey' + schema = 'catalogdb' + primary_key = False + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql new file mode 100644 index 00000000..b1dc8ef8 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql @@ -0,0 +1,4 @@ + +CREATE INDEX on catalogdb.legacy_survey (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey; +ANALYZE catalogdb.legacy_survey; diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql index 520f3c24..36ed2226 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql @@ -7,124 +7,124 @@ Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/legacysurvey */ CREATE TABLE catalogdb.legacy_survey ( - release integer, - brickid bigint, - brickname text, - objid bigint, - type text, - ra double precision, - dec double precision, - ra_ivar real, - dec_ivar real, - dchisq real[5], - ebv real, - flux_g real, - flux_r real, - flux_z real, - flux_w1 real, - flux_w2 real, - flux_w3 real, - flux_w4 real, - flux_ivar_g real, - flux_ivar_r real, - flux_ivar_z real, - flux_ivar_w1 real, - flux_ivar_w2 real, - flux_ivar_w3 real, - flux_ivar_w4 real, - mw_transmission_g real, - mw_transmission_r real, - mw_transmission_z real, - mw_transmission_w1 real, - mw_transmission_w2 real, - mw_transmission_w3 real, - mw_transmission_w4 real, - nobs_g integer, - nobs_r integer, - nobs_z integer, - nobs_w1 integer, - nobs_w2 integer, - nobs_w3 integer, - nobs_w4 integer, - rchisq_g real, - rchisq_r real, - rchisq_z real, - rchisq_w1 real, - rchisq_w2 real, - rchisq_w3 real, - rchisq_w4 real, - fracflux_g real, - fracflux_r real, - fracflux_z real, - fracflux_w1 real, - fracflux_w2 real, - fracflux_w3 real, - fracflux_w4 real, - fracmasked_g real, - fracmasked_r real, - fracmasked_z real, - fracin_g real, - fracin_r real, - fracin_z real, - anymask_g integer, - anymask_r integer, - anymask_z integer, - allmask_g integer, - allmask_r integer, - allmask_z integer, - wisemask_w1 smallint, - wisemask_w2 smallint, - psfsize_g real, - psfsize_r real, - psfsize_z real, - psfdepth_g real, - psfdepth_r real, - psfdepth_z real, - galdepth_g real, - galdepth_r real, - galdepth_z real, - psfdepth_w1 real, - psfdepth_w2 real, - wise_coadd_id text, - fracdev real, - fracdev_ivar real, - shapedev_r real, - shapedev_r_ivar real, - shapedev_e1 real, - shapedev_e1_ivar real, - shapedev_e2 real, - shapedev_e2_ivar real, - shapeexp_r real, - shapeexp_r_ivar real, - shapeexp_e1 real, - shapeexp_e1_ivar real, - shapeexp_e2 real, - shapeexp_e2_ivar real, - fiberflux_g real, - fiberflux_r real, - fiberflux_z real, - fibertotflux_g real, - fibertotflux_r real, - fibertotflux_z real, - ref_cat text, - ref_id bigint, - ref_epoch real, - gaia_phot_g_mean_mag real, - gaia_phot_g_mean_flux_over_error real, - gaia_phot_bp_mean_mag real, - gaia_phot_bp_mean_flux_over_error real, - gaia_phot_rp_mean_mag real, - gaia_phot_rp_mean_flux_over_error real, - gaia_astrometric_excess_noise real, - gaia_duplicated_source bool, - gaia_phot_bp_rp_excess_factor real, - gaia_astrometric_sigma5d_max real, - gaia_astrometric_params_solved smallint, - parallax real, - parallax_ivar real, - pmra real, - pmra_ivar real, - pmdec real, - pmdec_ivar real, - maskbits integer + release INTEGER, + brickid BIGINT, + brickname TEXT, + objid BIGINT PRIMARY KEY, + type TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + ra_ivar REAL, + dec_ivar REAL, + dchisq REAL[5], + ebv REAL, + flux_g REAL, + flux_r REAL, + flux_z REAL, + flux_w1 REAL, + flux_w2 REAL, + flux_w3 REAL, + flux_w4 REAL, + flux_ivar_g REAL, + flux_ivar_r REAL, + flux_ivar_z REAL, + flux_ivar_w1 REAL, + flux_ivar_w2 REAL, + flux_ivar_w3 REAL, + flux_ivar_w4 REAL, + mw_transmission_g REAL, + mw_transmission_r REAL, + mw_transmission_z REAL, + mw_transmission_w1 REAL, + mw_transmission_w2 REAL, + mw_transmission_w3 REAL, + mw_transmission_w4 REAL, + nobs_g INTEGER, + nobs_r INTEGER, + nobs_z INTEGER, + nobs_w1 INTEGER, + nobs_w2 INTEGER, + nobs_w3 INTEGER, + nobs_w4 INTEGER, + rchisq_g REAL, + rchisq_r REAL, + rchisq_z REAL, + rchisq_w1 REAL, + rchisq_w2 REAL, + rchisq_w3 REAL, + rchisq_w4 REAL, + fracflux_g REAL, + fracflux_r REAL, + fracflux_z REAL, + fracflux_w1 REAL, + fracflux_w2 REAL, + fracflux_w3 REAL, + fracflux_w4 REAL, + fracmasked_g REAL, + fracmasked_r REAL, + fracmasked_z REAL, + fracin_g REAL, + fracin_r REAL, + fracin_z REAL, + anymask_g INTEGER, + anymask_r INTEGER, + anymask_z INTEGER, + allmask_g INTEGER, + allmask_r INTEGER, + allmask_z INTEGER, + wisemask_w1 SMALLINT, + wisemask_w2 SMALLINT, + psfsize_g REAL, + psfsize_r REAL, + psfsize_z REAL, + psfdepth_g REAL, + psfdepth_r REAL, + psfdepth_z REAL, + galdepth_g REAL, + galdepth_r REAL, + galdepth_z REAL, + psfdepth_w1 REAL, + psfdepth_w2 REAL, + wise_coadd_id TEXT, + fracdev REAL, + fracdev_ivar REAL, + shapedev_r REAL, + shapedev_r_ivar REAL, + shapedev_e1 REAL, + shapedev_e1_ivar REAL, + shapedev_e2 REAL, + shapedev_e2_ivar REAL, + shapeexp_r REAL, + shapeexp_r_ivar REAL, + shapeexp_e1 REAL, + shapeexp_e1_ivar REAL, + shapeexp_e2 REAL, + shapeexp_e2_ivar REAL, + fiberflux_g REAL, + fiberflux_r REAL, + fiberflux_z REAL, + fibertotflux_g REAL, + fibertotflux_r REAL, + fibertotflux_z REAL, + ref_cat TEXT, + ref_id BIGINT, + ref_epoch REAL, + gaia_phot_g_mean_mag REAL, + gaia_phot_g_mean_flux_over_error REAL, + gaia_phot_bp_mean_mag REAL, + gaia_phot_bp_mean_flux_over_error REAL, + gaia_phot_rp_mean_mag REAL, + gaia_phot_rp_mean_flux_over_error REAL, + gaia_astrometric_excess_noise REAL, + gaia_duplicated_source BOOL, + gaia_phot_bp_rp_excess_factor REAL, + gaia_astrometric_sigma5d_max REAL, + gaia_astrometric_params_solved SMALLINT, + parallax REAL, + parallax_ivar REAL, + pmra REAL, + pmra_ivar REAL, + pmdec REAL, + pmdec_ivar REAL, + maskbits INTEGER ) WITHOUT OIDS; From 1d728280dff5404793bf0e45f22b9876d43277a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 12 Mar 2020 23:30:10 -0700 Subject: [PATCH 017/197] Change dependencies in all --- setup.cfg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index d34b5c5b..a4d2b697 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,13 +53,12 @@ sdssdb = all = sqlalchemy>=1.3.6 progressbar2>=3.46.1 - sdssdb[dev,docs] + pydot>=1.4.1 + astropy>=4.0.0 dev = pytest>=3.0.7 pytest-cov>=2.4.0 pytest-sugar>=0.8.0 - pydot>=1.4.1 - astropy>=4.0.0 pandas>=1.0.0 sdsstools[dev]>=0.1.5 docs = From 6a3aca88d5ceeb689eabeab1b1bf5763d3e69f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 00:32:10 -0700 Subject: [PATCH 018/197] Do not use multiprocessing in legacy loading --- .../sdss5db/catalogdb/legacy_survey/legacy_survey_load.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py index aac60069..68907dd3 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py @@ -7,9 +7,9 @@ # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) import glob -import multiprocessing import astropy.table +from progressbar import progressbar from sdssdb.peewee.sdss5db import database from sdssdb.utils.ingest import copy_data @@ -27,5 +27,5 @@ def ingest(file_): files = glob.glob('sweep*.fits') - pool = multiprocessing.Pool(25) - pool.map(ingest, files) + for file_ in progressbar(files): + ingest(file_) From 041de929471b447a17f297c885e00255ac9d13f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 00:55:08 -0700 Subject: [PATCH 019/197] Rename --- .../legacy_survey/{legacy_index.sql => legacy_survey_index.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename schema/sdss5db/catalogdb/legacy_survey/{legacy_index.sql => legacy_survey_index.sql} (100%) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql similarity index 100% rename from schema/sdss5db/catalogdb/legacy_survey/legacy_index.sql rename to schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql From fbc4a9807886f9f25a7acc2bc93c1f245c598e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 13:37:09 -0700 Subject: [PATCH 020/197] Add to_csv function to convert tables to CSV --- python/sdssdb/utils/ingest.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index a3c35118..1c9e80f8 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -37,6 +37,45 @@ } +def to_csv(table, path, header=True, convert_arrays=True, **kwargs): + """Creates a PostgreSQL-valid CSV file from a table, handling arrays. + + Parameters + ---------- + table : astropy.table.Table + The table to convert. + path : str + The path to which to write the CSV file. + header : bool + Whether to add a header with the column names. + convert_arrays : bool + If `True`, the arrays in the table are converted into a + PostgreSQL-valid string. + kwargs : dict + Other arguments to pass to `~astropy.table.Table.write`. + + """ + + if convert_arrays: + columns = [col for col in table.colnames if table[col].ndim > 1] + for col in columns: + index = table.index_column(col) + col_str = ['{' + ','.join(map(str, xx)) + '}' for xx in table[col]] + table.remove_column(col) + table.add_column(astropy.table.Column(col_str, col), index) + + if header: + write_kwargs = {'format': 'csv', 'fast_writer': True} + else: + write_kwargs = {'format': 'ascii.no_header', + 'delimiter': ',', + 'fast_writer': True} + + write_kwargs.update(kwargs) + + table.write(path, **write_kwargs) + + def table_exists(table_name, connection, schema=None): """Returns `True` if a table exists in a database. From f3ea634567b8459dee6de78b9efc9347c6077dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 13:37:33 -0700 Subject: [PATCH 021/197] Updated scripts for legacy survey --- .../legacy_survey_generate_csv.py | 41 +++++++++++++++++++ .../legacy_survey/legacy_survey_load | 4 ++ .../legacy_survey/legacy_survey_load.py | 31 -------------- 3 files changed, 45 insertions(+), 31 deletions(-) create mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py create mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load delete mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py new file mode 100755 index 00000000..60946027 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-12 +# @Filename: legacy_survey_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import multiprocessing + +import astropy.table +import progressbar + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected, 'database is not connected' + + +def convert_to_csv(file_): + + data = astropy.table.Table.read(file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, convert_arrays=True, overwrite=True) + + +if __name__ == '__main__': + + files = glob.glob('sweep*.fits') + + pool = multiprocessing.Pool(1) + + for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), + max_value=len(files)): + pass + + pool.close() + pool.join() diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load new file mode 100755 index 00000000..12060c86 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# encoding: utf-8 + +ls ./*.fits.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py deleted file mode 100755 index 68907dd3..00000000 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# @Author: José Sánchez-Gallego (gallegoj@uw.edu) -# @Date: 2020-03-12 -# @Filename: legacy_survey_load.py -# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) - -import glob - -import astropy.table -from progressbar import progressbar - -from sdssdb.peewee.sdss5db import database -from sdssdb.utils.ingest import copy_data - - -assert database.connected, 'database is not connected' - - -def ingest(file_): - data = astropy.table.Table.read(file_) - copy_data(data, database, 'legacy_survey', schema='catalogdb') - - -if __name__ == '__main__': - - files = glob.glob('sweep*.fits') - - for file_ in progressbar(files): - ingest(file_) From fc2a4e878f750eca53b6e3c51eeab70631bb27cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 13:39:02 -0700 Subject: [PATCH 022/197] Remove use of database connection --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 60946027..06be0dfc 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -12,13 +12,9 @@ import astropy.table import progressbar -from sdssdb.peewee.sdss5db import database from sdssdb.utils.ingest import to_csv -assert database.connected, 'database is not connected' - - def convert_to_csv(file_): data = astropy.table.Table.read(file_) From 832c77b28a635f4bec68de5520412476f3319419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 13:40:23 -0700 Subject: [PATCH 023/197] Use 25 cores --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 06be0dfc..71d05bf6 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -27,7 +27,7 @@ def convert_to_csv(file_): files = glob.glob('sweep*.fits') - pool = multiprocessing.Pool(1) + pool = multiprocessing.Pool(25) for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), max_value=len(files)): From 2110b190ba34b995c8c0bcdb0bf1815ae5521fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 13:52:42 -0700 Subject: [PATCH 024/197] Try with fewer cores --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 71d05bf6..7a270a77 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -21,13 +21,14 @@ def convert_to_csv(file_): data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) to_csv(data, file_ + '.csv', header=True, convert_arrays=True, overwrite=True) + del data if __name__ == '__main__': files = glob.glob('sweep*.fits') - pool = multiprocessing.Pool(25) + pool = multiprocessing.Pool(10) for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), max_value=len(files)): From b13466aba69cd17da0945533db11725a9eba7c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 14:31:08 -0700 Subject: [PATCH 025/197] Increase polling --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 7a270a77..916927bd 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -31,7 +31,7 @@ def convert_to_csv(file_): pool = multiprocessing.Pool(10) for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), - max_value=len(files)): + max_value=len(files), poll_interval=1): pass pool.close() From dfd88b1732bff8a5c3fc44a082b3c74d6b8828b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 15:51:00 -0700 Subject: [PATCH 026/197] Set legacy survey PK after ingestion --- schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql index 36ed2226..445edf1d 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql @@ -10,7 +10,7 @@ CREATE TABLE catalogdb.legacy_survey ( release INTEGER, brickid BIGINT, brickname TEXT, - objid BIGINT PRIMARY KEY, + objid BIGINT, type TEXT, ra DOUBLE PRECISION, dec DOUBLE PRECISION, @@ -128,3 +128,7 @@ CREATE TABLE catalogdb.legacy_survey ( pmdec_ivar REAL, maskbits INTEGER ) WITHOUT OIDS; + + +-- Add the serial PK after copying the data. +ALTER TABLE catalogdb.legacy_survey ADD COLUMN pk BIGSERIAL PRIMARY KEY; From c1b7265b6317ad2968fd388c5b364cc991299e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 16:08:13 -0700 Subject: [PATCH 027/197] Files for Gaia_unWISE_AGN --- python/sdssdb/peewee/sdss5db/catalogdb.py | 47 ++++++++++++++++++ .../Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql | 48 +++++++++++++++++++ .../Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql | 4 ++ .../Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py | 33 +++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql create mode 100644 schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql create mode 100755 schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 273560e9..ebe75f1a 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -2133,4 +2133,51 @@ class Meta: primary_key = False +class GaiaUnwiseAgn(SDSS5dbModel): + + gaia_sourceid = BigAutoField(primary_key=True) + ra = DoubleField(null=True) + dec = DoubleField(null=True) + plx = DoubleField(null=True) + plx_err = DoubleField(null=True) + pmra = DoubleField(null=True) + pmra_err = DoubleField(null=True) + pmdec = DoubleField(null=True) + pmdec_err = DoubleField(null=True) + plxsig = DoubleField(null=True) + pmsig = DoubleField(null=True) + ebv = DoubleField(null=True) + n_obs = IntegerField(null=True) + g = DoubleField(null=True) + bp = DoubleField(null=True) + rp = DoubleField(null=True) + w1 = DoubleField(null=True) + w2 = DoubleField(null=True) + bp_g = DoubleField(null=True) + bp_rp = DoubleField(null=True) + g_rp = DoubleField(null=True) + g_w1 = DoubleField(null=True) + gw_sep = DoubleField(null=True) + w1_w2 = DoubleField(null=True) + g_var = DoubleField(null=True) + bprp_ef = DoubleField(null=True) + aen = DoubleField(null=True) + gof = DoubleField(null=True) + cnt1 = IntegerField(null=True) + cnt2 = IntegerField(null=True) + cnt4 = IntegerField(null=True) + cnt8 = IntegerField(null=True) + cnt16 = IntegerField(null=True) + cnt32 = IntegerField(null=True) + phot_z = DoubleField(null=True) + prob_rf = DoubleField(null=True) + + unwise = ForeignKeyField(unWISE, column_name='unwise_objid') + gaia = ForeignKeyField(GaiaDR2Source) + + class Meta: + table_name = 'gaia_unwise_agn' + schema = 'catalogdb' + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql new file mode 100644 index 00000000..f5b53cee --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql @@ -0,0 +1,48 @@ +/* + +Schema for Gaia_unWISE_AGN + +Docs: https://people.ast.cam.ac.uk/~ypshu/AGN_Catalogues.html +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/gaia_unwise_agn/v1 + +*/ + +CREATE TABLE catalogdb.gaia_unwise_agn ( + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + gaia_sourceid BIGINT PRIMARY KEY, + unwise_objid TEXT, + plx DOUBLE PRECISION, + plx_err DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_err DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_err DOUBLE PRECISION, + plxsig DOUBLE PRECISION, + pmsig DOUBLE PRECISION, + ebv DOUBLE PRECISION, + n_obs INTEGER, + g DOUBLE PRECISION, + bp DOUBLE PRECISION, + rp DOUBLE PRECISION, + w1 DOUBLE PRECISION, + w2 DOUBLE PRECISION, + bp_g DOUBLE PRECISION, + bp_rp DOUBLE PRECISION, + g_rp DOUBLE PRECISION, + g_w1 DOUBLE PRECISION, + gw_sep DOUBLE PRECISION, + w1_w2 DOUBLE PRECISION, + g_var DOUBLE PRECISION, + bprp_ef DOUBLE PRECISION, + aen DOUBLE PRECISION, + gof DOUBLE PRECISION, + cnt1 INTEGER, + cnt2 INTEGER, + cnt4 INTEGER, + cnt8 INTEGER, + cnt16 INTEGER, + cnt32 INTEGER, + phot_z DOUBLE PRECISION, + prob_rf DOUBLE PRECISION +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql new file mode 100644 index 00000000..5c63eccc --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql @@ -0,0 +1,4 @@ + +CREATE INDEX on catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_unwise_agn_q3c_ang2ipix_idx on catalogdb.gaia_unwise_agn; +ANALYZE catalogdb.gaia_unwise_agn; diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py new file mode 100755 index 00000000..eccf0d51 --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-13 +# @Filename: Gaia_unWISE_AGN_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + file_ = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/gaia_unwise_agn/v1/Gaia_unWISE_AGNs.fits' # noqa + + data = astropy.table.Table.read(file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, overwrite=True) + del data + + + + +if __name__ == '__main__': + + main() From cb898a5e7ee1c5a42344fdf6de3cbac0a39284a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 18:49:05 -0700 Subject: [PATCH 028/197] Actually copy the data --- .../sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py index eccf0d51..7875bd33 100755 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -25,7 +25,8 @@ def main(): to_csv(data, file_ + '.csv', header=True, overwrite=True) del data - + cursor = database.cursor() + cursor.copy_from(open(file_ + '.csv'), 'catalogdb.gaia_unwise_agn', sep=',') if __name__ == '__main__': From 4803b578a7487f09b0a5cb1e5a31075228bc2ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 18:52:45 -0700 Subject: [PATCH 029/197] Skip the header --- .../sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py index 7875bd33..12e5b810 100755 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -26,7 +26,9 @@ def main(): del data cursor = database.cursor() - cursor.copy_from(open(file_ + '.csv'), 'catalogdb.gaia_unwise_agn', sep=',') + fileobj = open(file_ + '.csv') + fileobj.readline() # Read header + cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') if __name__ == '__main__': From 1c07da8e1f695e5b318e25169ea1cc040075c402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 20:03:17 -0600 Subject: [PATCH 030/197] Commit the copy --- .../sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py index 12e5b810..96105c1a 100755 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -22,13 +22,14 @@ def main(): data = astropy.table.Table.read(file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) - to_csv(data, file_ + '.csv', header=True, overwrite=True) + # to_csv(data, file_ + '.csv', header=True, overwrite=True) del data cursor = database.cursor() fileobj = open(file_ + '.csv') fileobj.readline() # Read header cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') + database.commit() if __name__ == '__main__': From 6734c80ae2f01c604edf2a000b6bd0c24e15a007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 13 Mar 2020 19:06:24 -0700 Subject: [PATCH 031/197] Use single process to load legacy data --- schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load index 12060c86..00b77e55 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load @@ -1,4 +1,4 @@ #!/usr/bin/env bash # encoding: utf-8 -ls ./*.fits.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" +ls ./*.fits.csv | parallel -j1 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" From 26dd21ddbbddc0ab1a56ea2dd3406980b66106fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 19:19:10 -0700 Subject: [PATCH 032/197] Try to avoid memory problems by closing pool --- .../legacy_survey/legacy_survey_generate_csv.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 916927bd..e904676f 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -8,6 +8,7 @@ import glob import multiprocessing +from contextlib import closing import astropy.table import progressbar @@ -28,11 +29,8 @@ def convert_to_csv(file_): files = glob.glob('sweep*.fits') - pool = multiprocessing.Pool(10) + with closing(multiprocessing.Pool(15)) as pool: - for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), - max_value=len(files), poll_interval=1): - pass - - pool.close() - pool.join() + for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), + max_value=len(files), poll_interval=1): + pass From cc4c803ddf4c343de0ceb9813795c2b3d898de44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 19:28:41 -0700 Subject: [PATCH 033/197] Use single process --- .../legacy_survey/legacy_survey_generate_csv.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index e904676f..77c409a7 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -7,8 +7,6 @@ # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) import glob -import multiprocessing -from contextlib import closing import astropy.table import progressbar @@ -29,8 +27,5 @@ def convert_to_csv(file_): files = glob.glob('sweep*.fits') - with closing(multiprocessing.Pool(15)) as pool: - - for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), - max_value=len(files), poll_interval=1): - pass + for file_ in progressbar.progressbar(files, poll_interval=1): + convert_to_csv(file_) From 19bdc994512efa0918107c70e9e8653b90b47fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 21:01:12 -0700 Subject: [PATCH 034/197] Try using argv for parallel --- .../catalogdb/legacy_survey/legacy_survey_generate_csv | 4 ++++ .../legacy_survey/legacy_survey_generate_csv.py | 9 +++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv new file mode 100755 index 00000000..5ed780b0 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# encoding: utf-8 + +ls ./*.fits | parallel -j25 "python legacy_survey_generate_csv.py {}" diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 77c409a7..97489546 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -6,10 +6,9 @@ # @Filename: legacy_survey_load.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) -import glob +import sys import astropy.table -import progressbar from sdssdb.utils.ingest import to_csv @@ -25,7 +24,5 @@ def convert_to_csv(file_): if __name__ == '__main__': - files = glob.glob('sweep*.fits') - - for file_ in progressbar.progressbar(files, poll_interval=1): - convert_to_csv(file_) + file_ = sys.argv[1] + convert_to_csv(file_) From 79fba53c44d3fa6268831b7256c7c19e55a90d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 21:03:23 -0700 Subject: [PATCH 035/197] Use full path --- .../sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv index 5ed780b0..78a356b4 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv @@ -1,4 +1,4 @@ #!/usr/bin/env bash # encoding: utf-8 -ls ./*.fits | parallel -j25 "python legacy_survey_generate_csv.py {}" +ls ./*.fits | parallel -j25 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py {}" From 3dd4ba9fe1635c38e842d4150fc301fb7acf168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 21:32:47 -0700 Subject: [PATCH 036/197] Revert to using pool --- .../legacy_survey/legacy_survey_generate_csv | 4 ---- .../legacy_survey/legacy_survey_generate_csv.py | 14 +++++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) delete mode 100755 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv deleted file mode 100755 index 78a356b4..00000000 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# encoding: utf-8 - -ls ./*.fits | parallel -j25 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py {}" diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 97489546..c4cdcb05 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -6,9 +6,12 @@ # @Filename: legacy_survey_load.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) -import sys +import glob +import multiprocessing +from contextlib import closing import astropy.table +import progressbar from sdssdb.utils.ingest import to_csv @@ -24,5 +27,10 @@ def convert_to_csv(file_): if __name__ == '__main__': - file_ = sys.argv[1] - convert_to_csv(file_) + files = glob.glob('sweep*.fits') + + with multiprocessing.Pool(10) as pool: + + for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), + max_value=len(files), poll_interval=1): + pass From 6c48f09bcca0aa61f99a15df9a99d300e7869fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 22:22:00 -0700 Subject: [PATCH 037/197] Unused import --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index c4cdcb05..cb6dffa9 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -8,7 +8,6 @@ import glob import multiprocessing -from contextlib import closing import astropy.table import progressbar From ddc5d6ecf3da0a02bb489eaf989c6f612d2cebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 14 Mar 2020 23:02:12 -0700 Subject: [PATCH 038/197] Use 10 cores to load legacy data --- schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load index 00b77e55..03ecbdcc 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load @@ -1,4 +1,4 @@ #!/usr/bin/env bash # encoding: utf-8 -ls ./*.fits.csv | parallel -j1 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" +ls ./*.fits.csv | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" From 2a5b713589d48575c7cc87e880e7945f5f5bf011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:15:30 -0700 Subject: [PATCH 039/197] Allow to run generate_csv for a file --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index cb6dffa9..9b4192d4 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -8,6 +8,7 @@ import glob import multiprocessing +import sys import astropy.table import progressbar @@ -26,6 +27,10 @@ def convert_to_csv(file_): if __name__ == '__main__': + if len(sys.argv) > 1: + convert_to_csv(sys.argv[1]) + sys.exit(0) + files = glob.glob('sweep*.fits') with multiprocessing.Pool(10) as pool: From 488d18505ade2348ef069ed98e0c3e9673752304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:16:33 -0700 Subject: [PATCH 040/197] Use memmap=True --- .../catalogdb/legacy_survey/legacy_survey_generate_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py index 9b4192d4..d02afcb9 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py @@ -18,7 +18,7 @@ def convert_to_csv(file_): - data = astropy.table.Table.read(file_) + data = astropy.table.Table.read(file_, memmap=True) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) to_csv(data, file_ + '.csv', header=True, convert_arrays=True, overwrite=True) From 5a777a3d99af8956b000a7adcb250793b6abb17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:40:16 -0700 Subject: [PATCH 041/197] Add specObj DR16 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 7 + .../sdssSpecObj/dr16/sdss_dr16_specObj.sql | 411 ++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index ebe75f1a..da1e98b0 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1649,6 +1649,13 @@ class Meta: schema = 'catalogdb' +class SDSSDR16SpecObj(SDSSDR14SpecObj): + + class Meta: + table_name = 'sdss_dr16_specobj' + schema = 'catalogdb' + + class TIC_v8(SDSS5dbModel): allwise = CharField(index=True, null=True) diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql new file mode 100644 index 00000000..7f8ce875 --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -0,0 +1,411 @@ +/* + +Schema for sdssSpecObj DR16 table. + +Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx + +File is /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj + +specObjID decimal 16 ID_CATALOG Unique database ID based on PLATE, MJD, FIBERID, RUN2D +bestObjID bigint 8 ID_MAIN Object ID of photoObj match (position-based) +fluxObjID bigint 8 ID_MAIN Object ID of photoObj match (flux-based) +targetObjID bigint 8 ID_CATALOG Object ID of original target +plateID decimal 16 Database ID of Plate +sciencePrimary smallint 2 Best version of spectrum at this location (defines default view SpecObj) +sdssPrimary smallint 2 Best version of spectrum at this location among SDSS plates (defines default view SpecObj) +legacyPrimary smallint 2 Best version of spectrum at this location, among Legacy plates +seguePrimary smallint 2 Best version of spectrum at this location, among SEGUE plates +segue1Primary smallint 2 Best version of spectrum at this location, among SEGUE-1 plates +segue2Primary smallint 2 Best version of spectrum at this location, among SEGUE-2 plates +bossPrimary smallint 2 Best version of spectrum at this location, among BOSS plates +bossSpecObjID int 4 Index of BOSS observation in spAll flat file +firstRelease varchar 32 Name of first release this object was associated with +survey varchar 32 Survey name +instrument varchar 32 Instrument used (SDSS or BOSS spectrograph) +programname varchar 32 Program name +chunk varchar 32 Chunk name +platerun varchar 32 Plate drill run name +mjd int 4 days MJD of observation +plate smallint 2 Plate number +fiberID smallint 2 Fiber ID +run1d varchar 32 1D Reduction version of spectrum +run2d varchar 32 2D Reduction version of spectrum +tile int 4 Tile number +designID int 4 Design ID number +legacy_target1 bigint 8 for Legacy program, target selection information at plate design +legacy_target2 bigint 8 for Legacy program target selection information at plate design, secondary/qa/calibration +special_target1 bigint 8 for Special program target selection information at plate design +special_target2 bigint 8 for Special program target selection information at plate design, secondary/qa/calibration +segue1_target1 bigint 8 SEGUE-1 target selection information at plate design, primary science selection +segue1_target2 bigint 8 SEGUE-1 target selection information at plate design, secondary/qa/calib selection +segue2_target1 bigint 8 SEGUE-2 target selection information at plate design, primary science selection +segue2_target2 bigint 8 SEGUE-2 target selection information at plate design, secondary/qa/calib selection +boss_target1 bigint 8 BOSS target selection information at plate +eboss_target0 bigint 8 EBOSS target selection information, for SEQUELS plates +eboss_target1 bigint 8 EBOSS target selection information, for eBOSS plates +eboss_target2 bigint 8 EBOSS target selection information, for TDSS, SPIDERS, ELG, etc. plates +eboss_target_id bigint 8 EBOSS unique target identifier for every spectroscopic target, +ancillary_target1 bigint 8 BOSS ancillary science target selection information at plate design +ancillary_target2 bigint 8 BOSS ancillary target selection information at plate design +thing_id_targeting bigint 8 thing_id value from the version of resolve from which the targeting was created +thing_id int 4 Unique identifier from global resolve +primTarget bigint 8 target selection information at plate design, primary science selection (for backwards compatibility) +secTarget bigint 8 target selection information at plate design, secondary/qa/calib selection (for backwards compatibility) +spectrographID smallint 2 which spectrograph (1,2) +sourceType varchar 128 For Legacy, SEGUE-2 and BOSS science targets, type of object targeted as (target bits contain full information and are recommended) +targetType varchar 128 Nature of target: SCIENCE, STANDARD, or SKY +ra float 8 deg DR8 Right ascension of fiber, J2000 +dec float 8 deg DR8 Declination of fiber, J2000 +cx float 8 POS_EQ_CART_X x of Normal unit vector in J2000 +cy float 8 POS_EQ_CART_Y y of Normal unit vector in J2000 +cz float 8 POS_EQ_CART_Z z of Normal unit vector in J2000 +xFocal real 4 mm X focal plane position (+RA direction) +yFocal real 4 mm Y focal plane position (+Dec direction) +lambdaEff real 4 Angstroms Effective wavelength that hole was drilled for (accounting for atmopheric refraction) +blueFiber int 4 Set to 1 if this hole was designated a "blue fiber", 0 if designated a "red fiber" (high redshift LRGs are preferentially in "red fibers") +zOffset real 4 microns Washer thickness used (for backstopping BOSS quasar targets, so they are closer to 4000 Angstrom focal plan) +z real 4 Final Redshift +zErr real 4 Redshift error +zWarning int 4 Bitmask of warning values; 0 means all is well +class varchar 32 Spectroscopic class (GALAXY, QSO, or STAR) +subClass varchar 32 Spectroscopic subclass +rChi2 real 4 Reduced chi-squared of best fit +DOF real 4 Degrees of freedom in best fit +rChi2Diff real 4 Difference in reduced chi-squared between best and second best fit +z_noqso real 4 Best redshift when excluding QSO fit in BOSS spectra (right redshift to use for galaxy targets) +zErr_noqso real 4 Error in "z_noqso" redshift (BOSS spectra only) +zWarning_noqso int 4 Warnings in "z_noqso" redshift (BOSS spectra only) +class_noqso varchar 32 Classification in "z_noqso" redshift +subClass_noqso varchar 32 Sub-classification in "z_noqso" redshift +rChi2Diff_noqso real 4 Reduced chi-squared difference from next best redshift, for "z_noqso" redshift +z_person real 4 Person-assigned redshift, if this object has been inspected +class_person varchar 32 Person-assigned classification, if this object has been inspected +comments_person varchar 200 Comments from person for inspected objects +tFile varchar 32 File name of best fit template source +tColumn_0 smallint 2 Which column of the template file corresponds to template #0 +tColumn_1 smallint 2 Which column of the template file corresponds to template #1 +tColumn_2 smallint 2 Which column of the template file corresponds to template #2 +tColumn_3 smallint 2 Which column of the template file corresponds to template #3 +tColumn_4 smallint 2 Which column of the template file corresponds to template #4 +tColumn_5 smallint 2 Which column of the template file corresponds to template #5 +tColumn_6 smallint 2 Which column of the template file corresponds to template #6 +tColumn_7 smallint 2 Which column of the template file corresponds to template #7 +tColumn_8 smallint 2 Which column of the template file corresponds to template #8 +tColumn_9 smallint 2 Which column of the template file corresponds to template #9 +nPoly real 4 Number of polynomial terms used in the fit +theta_0 real 4 Coefficient for template #0 of fit +theta_1 real 4 Coefficient for template #1 of fit +theta_2 real 4 Coefficient for template #2 of fit +theta_3 real 4 Coefficient for template #3 of fit +theta_4 real 4 Coefficient for template #4 of fit +theta_5 real 4 Coefficient for template #5 of fit +theta_6 real 4 Coefficient for template #6 of fit +theta_7 real 4 Coefficient for template #7 of fit +theta_8 real 4 Coefficient for template #8 of fit +theta_9 real 4 Coefficient for template #9 of fit +velDisp real 4 km/s Velocity dispersion +velDispErr real 4 km/s Error in velocity dispersion +velDispZ real 4 Redshift associated with best fit velocity dispersion +velDispZErr real 4 Error in redshift associated with best fit velocity dispersion +velDispChi2 real 4 Chi-squared associated with velocity dispersion fit +velDispNPix int 4 Number of pixels overlapping best template in velocity dispersion fit +velDispDOF int 4 Number of degrees of freedom in velocity dispersion fit +waveMin real 4 Angstroms Minimum observed (vacuum) wavelength +waveMax real 4 Angstroms Maximum observed (vacuum) wavelength +wCoverage real 4 Coverage in wavelength, in units of log10 wavelength +snMedian_u real 4 Median signal-to-noise over all good pixels in u-band +snMedian_g real 4 Median signal-to-noise over all good pixels in g-band +snMedian_r real 4 Median signal-to-noise over all good pixels in r-band +snMedian_i real 4 Median signal-to-noise over all good pixels in i-band +snMedian_z real 4 Median signal-to-noise over all good pixels in z-band +snMedian real 4 Median signal-to-noise over all good pixels +chi68p real 4 68-th percentile value of abs(chi) of the best-fit synthetic spectrum to the actual spectrum (around 1.0 for a good fit) +fracNSigma_1 real 4 Fraction of pixels deviant by more than 1 sigma relative to best-fit +fracNSigma_2 real 4 Fraction of pixels deviant by more than 2 sigma relative to best-fit +fracNSigma_3 real 4 Fraction of pixels deviant by more than 3 sigma relative to best-fit +fracNSigma_4 real 4 Fraction of pixels deviant by more than 4 sigma relative to best-fit +fracNSigma_5 real 4 Fraction of pixels deviant by more than 5 sigma relative to best-fit +fracNSigma_6 real 4 Fraction of pixels deviant by more than 6 sigma relative to best-fit +fracNSigma_7 real 4 Fraction of pixels deviant by more than 7 sigma relative to best-fit +fracNSigma_8 real 4 Fraction of pixels deviant by more than 8 sigma relative to best-fit +fracNSigma_9 real 4 Fraction of pixels deviant by more than 9 sigma relative to best-fit +fracNSigma_10 real 4 Fraction of pixels deviant by more than 10 sigma relative to best-fit +fracNSigHi_1 real 4 Fraction of pixels high by more than 1 sigma relative to best-fit +fracNSigHi_2 real 4 Fraction of pixels high by more than 2 sigma relative to best-fit +fracNSigHi_3 real 4 Fraction of pixels high by more than 3 sigma relative to best-fit +fracNSigHi_4 real 4 Fraction of pixels high by more than 4 sigma relative to best-fit +fracNSigHi_5 real 4 Fraction of pixels high by more than 5 sigma relative to best-fit +fracNSigHi_6 real 4 Fraction of pixels high by more than 6 sigma relative to best-fit +fracNSigHi_7 real 4 Fraction of pixels high by more than 7 sigma relative to best-fit +fracNSigHi_8 real 4 Fraction of pixels high by more than 8 sigma relative to best-fit +fracNSigHi_9 real 4 Fraction of pixels high by more than 9 sigma relative to best-fit +fracNSigHi_10 real 4 Fraction of pixels high by more than 10 sigma relative to best-fit +fracNSigLo_1 real 4 Fraction of pixels low by more than 1 sigma relative to best-fit +fracNSigLo_2 real 4 Fraction of pixels low by more than 2 sigma relative to best-fit +fracNSigLo_3 real 4 Fraction of pixels low by more than 3 sigma relative to best-fit +fracNSigLo_4 real 4 Fraction of pixels low by more than 4 sigma relative to best-fit +fracNSigLo_5 real 4 Fraction of pixels low by more than 5 sigma relative to best-fit +fracNSigLo_6 real 4 Fraction of pixels low by more than 6 sigma relative to best-fit +fracNSigLo_7 real 4 Fraction of pixels low by more than 7 sigma relative to best-fit +fracNSigLo_8 real 4 Fraction of pixels low by more than 8 sigma relative to best-fit +fracNSigLo_9 real 4 Fraction of pixels low by more than 9 sigma relative to best-fit +fracNSigLo_10 real 4 Fraction of pixels low by more than 10 sigma relative to best-fit +spectroFlux_u real 4 nanomaggies Spectrum projected onto u filter +spectroFlux_g real 4 nanomaggies Spectrum projected onto g filter +spectroFlux_r real 4 nanomaggies Spectrum projected onto r filter +spectroFlux_i real 4 nanomaggies Spectrum projected onto i filter +spectroFlux_z real 4 nanomaggies Spectrum projected onto z filter +spectroSynFlux_u real 4 nanomaggies Best-fit template spectrum projected onto u filter +spectroSynFlux_g real 4 nanomaggies Best-fit template spectrum projected onto g filter +spectroSynFlux_r real 4 nanomaggies Best-fit template spectrum projected onto r filter +spectroSynFlux_i real 4 nanomaggies Best-fit template spectrum projected onto i filter +spectroSynFlux_z real 4 nanomaggies Best-fit template spectrum projected onto z filter +spectroFluxIvar_u real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto u filter +spectroFluxIvar_g real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto g filter +spectroFluxIvar_r real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto r filter +spectroFluxIvar_i real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto i filter +spectroFluxIvar_z real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto z filter +spectroSynFluxIvar_u real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto u filter +spectroSynFluxIvar_g real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto g filter +spectroSynFluxIvar_r real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto r filter +spectroSynFluxIvar_i real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto i filter +spectroSynFluxIvar_z real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto z filter +spectroSkyFlux_u real 4 nanomaggies Sky spectrum projected onto u filter +spectroSkyFlux_g real 4 nanomaggies Sky spectrum projected onto g filter +spectroSkyFlux_r real 4 nanomaggies Sky spectrum projected onto r filter +spectroSkyFlux_i real 4 nanomaggies Sky spectrum projected onto i filter +spectroSkyFlux_z real 4 nanomaggies Sky spectrum projected onto z filter +anyAndMask int 4 For each bit, records whether any pixel in the spectrum has that bit set in its ANDMASK +anyOrMask int 4 For each bit, records whether any pixel in the spectrum has that bit set in its ORMASK +plateSN2 real 4 Overall signal-to-noise-squared measure for plate (only set for SDSS spectrograph) +deredSN2 real 4 Dereddened signal-to-noise-squared measure for plate (only set for BOSS spectrograph) +snTurnoff real 4 Signal to noise measure for MS turnoff stars on plate (-9999 if not appropriate) +sn1_g real 4 (S/N)^2 at g=20.20 for spectrograph #1 +sn1_r real 4 (S/N)^2 at r=20.25 for spectrograph #1 +sn1_i real 4 (S/N)^2 at i=19.90 for spectrograph #1 +sn2_g real 4 (S/N)^2 at g=20.20 for spectrograph #2 +sn2_r real 4 (S/N)^2 at r=20.25 for spectrograph #2 +sn2_i real 4 (S/N)^2 at i=19.90 for spectrograph #2 +elodieFileName varchar 32 File name for best-fit Elodie star +elodieObject varchar 32 Star name (mostly Henry Draper names) +elodieSpType varchar 32 Spectral type +elodieBV real 4 mag (B-V) color +elodieTEff real 4 Kelvin Effective temperature +elodieLogG real 4 log10(gravity) +elodieFeH real 4 Metallicity ([Fe/H]) +elodieZ real 4 Redshift +elodieZErr real 4 Redshift error (negative for invalid fit) +elodieZModelErr real 4 Standard deviation in redshift among the 12 best-fit stars +elodieRChi2 real 4 Reduced chi^2 +elodieDOF real 4 Degrees of freedom for fit +htmID bigint 8 CODE_HTM 20 deep Hierarchical Triangular Mesh ID +loadVersion int 4 ID_TRACER Load Version +img varbinary -1 IMAGE? Spectrum Image + +*/ + +CREATE TABLE catalogdb.sdss_dr16_specobj( + specObjID numeric(20), + bestObjID bigint, + fluxObjID bigint, + targetObjID bigint, + plateID numeric(20), + sciencePrimary smallint, + sdssPrimary smallint, + legacyPrimary smallint, + seguePrimary smallint, + segue1Primary smallint, + segue2Primary smallint, + bossPrimary smallint, + bossSpecObjID integer, + firstRelease varchar(32), + survey varchar(32), + instrument varchar(32), + programname varchar(32), + chunk varchar(32), + platerun varchar(32), + mjd integer, + plate smallint, + fiberID smallint, + run1d varchar(32), + run2d varchar(32), + tile integer, + designID integer, + legacy_target1 bigint, + legacy_target2 bigint, + special_target1 bigint, + special_target2 bigint, + segue1_target1 bigint, + segue1_target2 bigint, + segue2_target1 bigint, + segue2_target2 bigint, + boss_target1 bigint, + eboss_target0 bigint, + eboss_target1 bigint, + eboss_target2 bigint, + eboss_target_id bigint, + ancillary_target1 bigint, + ancillary_target2 bigint, + thing_id_targeting bigint, + thing_id integer, + primTarget bigint, + secTarget bigint, + spectrographID smallint, + sourceType varchar(128), + targetType varchar(128), + ra double precision, + dec double precision, + cx double precision, + cy double precision, + cz double precision, + xFocal real, + yFocal real, + lambdaEff real, + blueFiber integer, + zOffset real, + z real, + zErr real, + zWarning integer, + class varchar(32), + subClass varchar(32), + rChi2 real, + DOF real, + rChi2Diff real, + z_noqso real, + zErr_noqso real, + zWarning_noqso integer, + class_noqso varchar(32), + subClass_noqso varchar(32), + rChi2Diff_noqso real, + z_person real, + class_person varchar(32), + comments_person varchar(200), + tFile varchar(32), + tColumn_0 smallint, + tColumn_1 smallint, + tColumn_2 smallint, + tColumn_3 smallint, + tColumn_4 smallint, + tColumn_5 smallint, + tColumn_6 smallint, + tColumn_7 smallint, + tColumn_8 smallint, + tColumn_9 smallint, + nPoly real, + theta_0 real, + theta_1 real, + theta_2 real, + theta_3 real, + theta_4 real, + theta_5 real, + theta_6 real, + theta_7 real, + theta_8 real, + theta_9 real, + velDisp real, + velDispErr real, + velDispZ real, + velDispZErr real, + velDispChi2 real, + velDispNPix integer, + velDispDOF integer, + waveMin real, + waveMax real, + wCoverage real, + snMedian_u real, + snMedian_g real, + snMedian_r real, + snMedian_i real, + snMedian_z real, + snMedian real, + chi68p real, + fracNSigma_1 real, + fracNSigma_2 real, + fracNSigma_3 real, + fracNSigma_4 real, + fracNSigma_5 real, + fracNSigma_6 real, + fracNSigma_7 real, + fracNSigma_8 real, + fracNSigma_9 real, + fracNSigma_10 real, + fracNSigHi_1 real, + fracNSigHi_2 real, + fracNSigHi_3 real, + fracNSigHi_4 real, + fracNSigHi_5 real, + fracNSigHi_6 real, + fracNSigHi_7 real, + fracNSigHi_8 real, + fracNSigHi_9 real, + fracNSigHi_10 real, + fracNSigLo_1 real, + fracNSigLo_2 real, + fracNSigLo_3 real, + fracNSigLo_4 real, + fracNSigLo_5 real, + fracNSigLo_6 real, + fracNSigLo_7 real, + fracNSigLo_8 real, + fracNSigLo_9 real, + fracNSigLo_10 real, + spectroFlux_u real, + spectroFlux_g real, + spectroFlux_r real, + spectroFlux_i real, + spectroFlux_z real, + spectroSynFlux_u real, + spectroSynFlux_g real, + spectroSynFlux_r real, + spectroSynFlux_i real, + spectroSynFlux_z real, + spectroFluxIvar_u real, + spectroFluxIvar_g real, + spectroFluxIvar_r real, + spectroFluxIvar_i real, + spectroFluxIvar_z real, + spectroSynFluxIvar_u real, + spectroSynFluxIvar_g real, + spectroSynFluxIvar_r real, + spectroSynFluxIvar_i real, + spectroSynFluxIvar_z real, + spectroSkyFlux_u real, + spectroSkyFlux_g real, + spectroSkyFlux_r real, + spectroSkyFlux_i real, + spectroSkyFlux_z real, + anyAndMask integer, + anyOrMask integer, + plateSN2 real, + deredSN2 real, + snTurnoff real, + sn1_g real, + sn1_r real, + sn1_i real, + sn2_g real, + sn2_r real, + sn2_i real, + elodieFileName varchar(32), + elodieObject varchar(32), + elodieSpType varchar(32), + elodieBV real, + elodieTEff real, + elodieLogG real, + elodieFeH real, + elodieZ real, + elodieZErr real, + elodieZModelErr real, + elodieRChi2 real, + elodieDOF real, + htmID bigint, + loadVersion integer +); + + +\copy catalogdb.sdss_dr16_specobj FROM program 'bzcat ./sqlSpecObj.csv.bz2' WITH CSV HEADER; + +alter table catalogdb.sdss_dr16_specobj add primary key(specObjID); + +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_bestObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (bestObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fluxObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_targetObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (targetObjID); From 8c67fa9a35c60de47671c71b5a4b3e6d651c56a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:51:17 -0700 Subject: [PATCH 042/197] Use full path --- schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index 7f8ce875..3fbceb3c 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -402,7 +402,7 @@ CREATE TABLE catalogdb.sdss_dr16_specobj( ); -\copy catalogdb.sdss_dr16_specobj FROM program 'bzcat ./sqlSpecObj.csv.bz2' WITH CSV HEADER; +\copy catalogdb.sdss_dr16_specobj FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj/dr16/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; alter table catalogdb.sdss_dr16_specobj add primary key(specObjID); From 3f07d01a8eda6e31752aa8e516196a2f3e216039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:59:03 -0700 Subject: [PATCH 043/197] Add 13c index --- .../sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index 3fbceb3c..c29330de 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -409,3 +409,8 @@ alter table catalogdb.sdss_dr16_specobj add primary key(specObjID); CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_bestObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (bestObjID); CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fluxObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (fluxObjID); CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_targetObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (targetObjID); + + +CREATE INDEX on catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx on catalogdb.sdss_dr16_specobj; +ANALYZE catalogdb.sdss_dr16_specobj; From c3f2373c6f47793ecb4395bf453295829c31e938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:59:35 -0700 Subject: [PATCH 044/197] Make source_id pk --- schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql index b09cb428..f19f8d2c 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql @@ -19,7 +19,7 @@ drop schema catalogdb cascade; CREATE TABLE catalogdb.gaia_dr2_source ( solution_id bigint, designation text, - source_id bigint, + source_id bigint primary key, random_index bigint, ref_epoch double precision, ra double precision, @@ -111,7 +111,3 @@ CREATE TABLE catalogdb.gaia_dr2_source ( lum_val real, lum_percentile_lower real, lum_percentile_upper real); - - - - From 9c4539ddb5421060233a329812af51254d7b4cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:59:42 -0700 Subject: [PATCH 045/197] Fix fk --- schema/sdss5db/targetdb/targetdb.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/targetdb/targetdb.sql b/schema/sdss5db/targetdb/targetdb.sql index 0935954c..ace525e3 100644 --- a/schema/sdss5db/targetdb/targetdb.sql +++ b/schema/sdss5db/targetdb/targetdb.sql @@ -252,5 +252,5 @@ CREATE INDEX CONCURRENTLY field_cadence_pk_idx ON targetdb.field using BTREE(cad CREATE INDEX CONCURRENTLY observatory_pk_idx ON targetdb.field using BTREE(observatory_pk); CREATE INDEX CONCURRENTLY positioner_status_pk_idx ON targetdb.positioner using BTREE(positioner_status_pk); -CREATE INDEX CONCURRENTLY positioner_type_pk_idx ON targetdb.positioner using BTREE(positioner_type_pk); +CREATE INDEX CONCURRENTLY positioner_info_pk_idx ON targetdb.positioner using BTREE(positioner_info_pk); CREATE INDEX CONCURRENTLY positioner_observatory_pk_idx ON targetdb.positioner using BTREE(observatory_pk); From 6f2cf1e857b2423559560c42e43e7d4416d221cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 12:59:50 -0700 Subject: [PATCH 046/197] Add ipython to dev deps --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a4d2b697..a0548b36 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,6 +61,7 @@ dev = pytest-sugar>=0.8.0 pandas>=1.0.0 sdsstools[dev]>=0.1.5 + ipython>=7.13.0 docs = Sphinx>=1.8.0 sphinx_bootstrap_theme>=0.4.12 From 39a1362ba153d5d47d7c6649ce8bd672c34ad5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 22:56:36 -0700 Subject: [PATCH 047/197] Uncomment to_csv --- .../sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py index 96105c1a..e10d1e6a 100755 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -22,7 +22,7 @@ def main(): data = astropy.table.Table.read(file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) - # to_csv(data, file_ + '.csv', header=True, overwrite=True) + to_csv(data, file_ + '.csv', header=True, overwrite=True) del data cursor = database.cursor() From fe75170665fd5cbc95169374c956dc67e9eb44b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 23:13:57 -0700 Subject: [PATCH 048/197] Files for ebosstarget v5 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 61 ++++++++++++++++-- .../ebosstarget/v5/ebosstarget_v5.sql | 63 +++++++++++++++++++ .../ebosstarget/v5/ebosstarget_v5_load.py | 40 ++++++++++++ 3 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql create mode 100755 schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index da1e98b0..24cc47dc 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1857,7 +1857,7 @@ class DR14QV44(SDSS5dbModel): err_hmag = FloatField(null=True) err_jmag = FloatField(null=True) err_kmag = FloatField(null=True) - err_psfmag = ArrayField(null=True) # ARRAY + err_psfmag = ArrayField(null=True) err_w1mag = FloatField(null=True) err_w2mag = FloatField(null=True) err_w3mag = FloatField(null=True) @@ -1876,14 +1876,14 @@ class DR14QV44(SDSS5dbModel): flux_2_0_12_0kev_err = FloatField(column_name='flux_2.0_12.0kev_err', null=True) fuv = FloatField(null=True) fuv_ivar = FloatField(null=True) - gal_ext = ArrayField(null=True) # ARRAY + gal_ext = ArrayField(null=True) galex_matched = FloatField(null=True) hflux = FloatField(null=True) hflux_err = FloatField(null=True) hmag = FloatField(null=True) hrdflag = IntegerField(null=True) hsnr = FloatField(null=True) - ivar_psfflux = ArrayField(null=True) # ARRAY + ivar_psfflux = ArrayField(null=True) jflux = FloatField(null=True) jflux_err = FloatField(null=True) jmag = FloatField(null=True) @@ -1907,8 +1907,8 @@ class DR14QV44(SDSS5dbModel): ph_flags = CharField(null=True) plate = IntegerField(null=True) plate_duplicate = ArrayField(field_class=IntegerField, null=True) - psfflux = ArrayField(null=True) # ARRAY - psfmag = ArrayField(null=True) # ARRAY + psfflux = ArrayField(null=True) + psfmag = ArrayField(null=True) ra = FloatField(null=True) rass_counts = FloatField(null=True) rass_counts_snr = FloatField(null=True) @@ -2187,4 +2187,55 @@ class Meta: schema = 'catalogdb' +class EbosstargetV5(SDSS5dbModel): + + run = IntegerField(null=True) + camcol = IntegerField(null=True) + field = IntegerField(null=True) + id = IntegerField(null=True) + rerun = TextField(null=True) + fibermag = ArrayField(field_class=FloatField, null=True) + fiber2mag = ArrayField(field_class=FloatField, null=True) + calib_status = ArrayField(field_class=IntegerField, null=True) + ra = DoubleField(null=True) + dec = DoubleField(null=True) + epoch = FloatField(null=True) + pmra = FloatField(null=True) + pmdec = FloatField(null=True) + eboss_target1 = BigIntegerField(null=True) + eboss_target2 = BigIntegerField(null=True) + eboss_target_id = BigIntegerField(null=True) + thing_id_targeting = IntegerField(null=True) + objc_type = IntegerField(null=True) + objc_flags = IntegerField(null=True) + objc_flags2 = IntegerField(null=True) + flags = IntegerField(null=True) + flags2 = IntegerField(null=True) + psf_fwhm = ArrayField(field_class=FloatField, null=True) + psfflux = ArrayField(field_class=FloatField, null=True) + psfflux_ivar = ArrayField(field_class=FloatField, null=True) + extinction = ArrayField(field_class=FloatField, null=True) + fiberflux = ArrayField(field_class=FloatField, null=True) + fiberflux_ivar = ArrayField(field_class=FloatField, null=True) + fiber2flux = ArrayField(field_class=FloatField, null=True) + fiber2flux_ivar = ArrayField(field_class=FloatField, null=True) + modelflux = ArrayField(field_class=FloatField, null=True) + modelflux_ivar = ArrayField(field_class=FloatField, null=True) + modelmag = ArrayField(field_class=FloatField, null=True) + modelmag_ivar = ArrayField(field_class=FloatField, null=True) + resolve_status = IntegerField(null=True) + w1_mag = FloatField(null=True) + w1_mag_err = FloatField(null=True) + w1_nanomaggies = FloatField(null=True) + w1_nanomaggies_ivar = FloatField(null=True) + w2_nanomaggies = FloatField(null=True) + w2_nanomaggies_ivar = FloatField(null=True) + has_wise_phot = BooleanField(null=True) + objid_targeting = BigAutoField(primary_key=True) + + class Meta: + table_name = 'ebosstarget_v5' + schema = 'catalogdb' + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql new file mode 100644 index 00000000..b09d715e --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql @@ -0,0 +1,63 @@ +/* + +Schema for eBOSS Target. + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005 + +*/ + +CREATE TABLE catalogdb.ebosstarget_v5 ( + run INTEGER, + camcol INTEGER, + field INTEGER, + id INTEGER, + rerun TEXT, + fibermag REAL[2], + fiber2mag REAL[2], + calib_status INTEGER[2], + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + epoch REAL, + pmra REAL, + pmdec REAL, + eboss_target1 BIGINT, + eboss_target2 BIGINT, + eboss_target_id BIGINT, + thing_id_targeting INTEGER, + objc_type INTEGER, + objc_flags INTEGER, + objc_flags2 INTEGER, + flags INTEGER, + flags2 INTEGER, + psf_fwhm REAL[2], + psfflux REAL[2], + psfflux_ivar REAL[2], + extinction REAL[2], + fiberflux REAL[2], + fiberflux_ivar REAL[2], + fiber2flux REAL[2], + fiber2flux_ivar REAL[2], + modelflux REAL[2], + modelflux_ivar REAL[2], + modelmag REAL[2], + modelmag_ivar REAL[2], + resolve_status INTEGER, + w1_mag REAL, + w1_mag_err REAL, + w1_nanomaggies REAL, + w1_nanomaggies_ivar REAL, + w2_nanomaggies REAL, + w2_nanomaggies_ivar REAL, + has_wise_phot BOOLEAN, + objid_targeting BIGINT PRIMARY KEY +) WITHOUT OIDS; + + +-- Create indices here directly since it's a small table + +CREATE INDEX on catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); +CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; +ANALYZE catalogdb.ebosstarget_v5; + +CREATE INDEX CONCURRENTLY ebosstarget_v5_objc_type_idx + ON catalogdb.ebosstarget_v5 using BTREE (objc_type); diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py new file mode 100755 index 00000000..0af93a65 --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-15 +# @Filename: ebosstarget_v5_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + path = '/uufs/chpc.utah.edu/common/home/sdss/ebosswork/eboss/target/ebosstarget/v0005/' + files = ['ebosstarget-v0005-qso.fits', 'ebosstarget-v0005-std.fits'] + + for file_ in files: + + data = astropy.table.Table.read(path + file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, overwrite=True, convert_arrays=True) + del data + + cursor = database.cursor() + fileobj = open(file_ + path + '.csv') + fileobj.readline() # Read header + cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') + database.commit() + + +if __name__ == '__main__': + + main() From 0e30abda5a75e74e697f9152b398ded7f8da3f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 23:27:21 -0700 Subject: [PATCH 049/197] Fix path --- .../sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 0af93a65..27a365e3 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -22,14 +22,16 @@ def main(): for file_ in files: + print(f'Converting {file_}') data = astropy.table.Table.read(path + file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) - to_csv(data, file_ + '.csv', header=True, overwrite=True, convert_arrays=True) + to_csv(data, path + file_ + '.csv', header=True, overwrite=True, convert_arrays=True) del data + print(f'Copying {file_}') cursor = database.cursor() - fileobj = open(file_ + path + '.csv') + fileobj = open(path + file_ + '.csv') fileobj.readline() # Read header cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') database.commit() From 2094fa0f63b7161f4f3a76f9484fb4abdc973c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 23:41:13 -0700 Subject: [PATCH 050/197] Change path so that it's writable --- schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 27a365e3..61a75ef7 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -17,7 +17,7 @@ def main(): - path = '/uufs/chpc.utah.edu/common/home/sdss/ebosswork/eboss/target/ebosstarget/v0005/' + path = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005' files = ['ebosstarget-v0005-qso.fits', 'ebosstarget-v0005-std.fits'] for file_ in files: From cc8375ec306fc9d4ffe47c09799e4ac9d803b92c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 15 Mar 2020 23:47:30 -0700 Subject: [PATCH 051/197] Trailing slash --- schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 61a75ef7..1d6d5fc7 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -17,7 +17,7 @@ def main(): - path = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005' + path = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005/' files = ['ebosstarget-v0005-qso.fits', 'ebosstarget-v0005-std.fits'] for file_ in files: From 99747492baf9a68bbdceb1a34be37f72b8d94502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 16 Mar 2020 04:07:55 -0700 Subject: [PATCH 052/197] Use different output directory due to permissions --- .../sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 1d6d5fc7..f313cbb9 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -20,18 +20,20 @@ def main(): path = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005/' files = ['ebosstarget-v0005-qso.fits', 'ebosstarget-v0005-std.fits'] + out = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/' + for file_ in files: print(f'Converting {file_}') data = astropy.table.Table.read(path + file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) - to_csv(data, path + file_ + '.csv', header=True, overwrite=True, convert_arrays=True) + to_csv(data, out + file_ + '.csv', header=True, overwrite=True, convert_arrays=True) del data print(f'Copying {file_}') cursor = database.cursor() - fileobj = open(path + file_ + '.csv') + fileobj = open(out + file_ + '.csv') fileobj.readline() # Read header cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') database.commit() From 85b2a14068d0388393b5933f5d8648c05f3bb135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 16 Mar 2020 04:38:29 -0700 Subject: [PATCH 053/197] Fix incorrect table --- schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index f313cbb9..62c3fcd1 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -35,7 +35,7 @@ def main(): cursor = database.cursor() fileobj = open(out + file_ + '.csv') fileobj.readline() # Read header - cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') + cursor.copy_from(fileobj, 'catalogdb.ebosstarget_v5', sep=',') database.commit() From 4709c3fb45bcde90a63c3cef84a5879d5bc0ce23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 16 Mar 2020 09:49:32 -0700 Subject: [PATCH 054/197] Fixes to ebosstarget --- .../ebosstarget/v5/ebosstarget_v5.sql | 44 +++++++------------ .../ebosstarget/v5/ebosstarget_v5_load.py | 2 + .../v5/ebosstarget_v5_post_load.sql | 15 +++++++ 3 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql index b09d715e..1d1f1281 100644 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql @@ -12,9 +12,9 @@ CREATE TABLE catalogdb.ebosstarget_v5 ( field INTEGER, id INTEGER, rerun TEXT, - fibermag REAL[2], - fiber2mag REAL[2], - calib_status INTEGER[2], + fibermag REAL[], + fiber2mag REAL[], + calib_status INTEGER[], ra DOUBLE PRECISION, dec DOUBLE PRECISION, epoch REAL, @@ -29,18 +29,18 @@ CREATE TABLE catalogdb.ebosstarget_v5 ( objc_flags2 INTEGER, flags INTEGER, flags2 INTEGER, - psf_fwhm REAL[2], - psfflux REAL[2], - psfflux_ivar REAL[2], - extinction REAL[2], - fiberflux REAL[2], - fiberflux_ivar REAL[2], - fiber2flux REAL[2], - fiber2flux_ivar REAL[2], - modelflux REAL[2], - modelflux_ivar REAL[2], - modelmag REAL[2], - modelmag_ivar REAL[2], + psf_fwhm REAL[], + psfflux REAL[], + psfflux_ivar REAL[], + extinction REAL[], + fiberflux REAL[], + fiberflux_ivar REAL[], + fiber2flux REAL[], + fiber2flux_ivar REAL[], + modelflux REAL[], + modelflux_ivar REAL[], + modelmag REAL[], + modelmag_ivar REAL[], resolve_status INTEGER, w1_mag REAL, w1_mag_err REAL, @@ -48,16 +48,6 @@ CREATE TABLE catalogdb.ebosstarget_v5 ( w1_nanomaggies_ivar REAL, w2_nanomaggies REAL, w2_nanomaggies_ivar REAL, - has_wise_phot BOOLEAN, - objid_targeting BIGINT PRIMARY KEY + has_wise_phot BOOLEAN NULL, + objid_targeting BIGINT ) WITHOUT OIDS; - - --- Create indices here directly since it's a small table - -CREATE INDEX on catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); -CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; -ANALYZE catalogdb.ebosstarget_v5; - -CREATE INDEX CONCURRENTLY ebosstarget_v5_objc_type_idx - ON catalogdb.ebosstarget_v5 using BTREE (objc_type); diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 62c3fcd1..9bd5cde0 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -28,6 +28,8 @@ def main(): data = astropy.table.Table.read(path + file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + if 'std' in file_: + data['has_wise_phot'] = '0' to_csv(data, out + file_ + '.csv', header=True, overwrite=True, convert_arrays=True) del data diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql new file mode 100644 index 00000000..327dc0db --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql @@ -0,0 +1,15 @@ +/* + +To be run after the data has been loaded. + +*/ + + +ALTER TABLE catalogdb.ebosstarget_v5 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX on catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); +CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; +ANALYZE catalogdb.ebosstarget_v5; + +CREATE INDEX CONCURRENTLY ebosstarget_v5_objc_type_idx + ON catalogdb.ebosstarget_v5 using BTREE (objc_type); From 0b1e32049c431db5a02709b22d4c629ff572c60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 16 Mar 2020 10:21:47 -0700 Subject: [PATCH 055/197] Try copy_expert --- .../sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py index 9bd5cde0..1dd49bd9 100755 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -35,9 +35,9 @@ def main(): print(f'Copying {file_}') cursor = database.cursor() - fileobj = open(out + file_ + '.csv') - fileobj.readline() # Read header - cursor.copy_from(fileobj, 'catalogdb.ebosstarget_v5', sep=',') + cursor.copy_expert('COPY catalogdb.ebosstarget_v5 FROM STDIN ' + 'WITH DELIMITER \',\' NULL \'\\N\' CSV HEADER;', + open(out + file_ + '.csv')) database.commit() From 4a75eb27252579bc3adadbf4f780a08b99471c7f Mon Sep 17 00:00:00 2001 From: Tom Dwelly Date: Mon, 16 Mar 2020 23:43:57 +0000 Subject: [PATCH 056/197] tweaks to the SPIDERS/eFEDS AGN/Clusters proto-data model --- python/sdssdb/peewee/sdss5db/catalogdb.py | 55 ++++++++++++++++------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 67b55526..752a80bb 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1960,8 +1960,8 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): # Chosen to match X-ray columns defined in eROSITA/SDSS-V MoU (v2.0, April 2019) ero_version = TextField(index=True, null==True) # string identifying this eROSITA data reduction version ero_souuid = TextField(index=True, null=True) # string identifying this X-ray source - ero_xray_flux = FloatField(null=True) # X-ray flux, 0.5-8keV band, erg/cm2/s - ero_xray_flux_err = FloatField(null=True) # X-ray flux uncertainty, 0.5-8keV band, erg/cm2/s + ero_flux = FloatField(null=True) # X-ray flux, 0.5-8keV band, erg/cm2/s + ero_flux_err = FloatField(null=True) # X-ray flux uncertainty, 0.5-8keV band, erg/cm2/s ero_ext = FloatField(null=True) # X-ray extent parameter - arcsec ero_ext_err = FloatField(null=True) # X-ray extent parameter uncertainty - arcsec ero_ext_like = FloatField(null=True) # X-ray extent likelihood @@ -1973,29 +1973,52 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): # Parameters describing the cross-matching of X-ray to optical/IR catalogue(s) xmatch_method = TextField(null=True) # 'ML+NWAY', 'LR' , 'SDSS_REDMAPPER', 'LS_REDMAPPER', 'HSC_REDMAPPER', 'MCMF' etc xmatch_version = TextField(null=True) # version identifier for cross-matching algorithm - xmatch_dist_arcsec = FloatField(null=True) # separation between X-ray position and opt positions - arcsec + xmatch_dist = FloatField(null=True) # separation between X-ray position and opt positions - arcsec xmatch_qual_metric = FloatField(null=True) # measure of quality of xmatch (e.g. p_any for Nway, LR) xmatch_qual_flags = IntegerField(null=True) # e.g. NWAY match_flag - treat as bitmask # Parameters that describe the sub-class + priority rank of the object - target_class = IntegerField(null=True) # TBD, but e.g. 1=AGN candidate, 2=Stellar candidate, + target_class = TextField(null=True) # TBD, but e.g. 'unknown', 'AGN', 'Star', 'Galaxy' # 3=TDE candidate, 4=Compact object, Cluster BCG=11, Cluster member-galaxy=12, low-z galaxy=13 etc target_priority = IntegerField(null=True) # allows priority ranking based on info not available in catalogdb - target_has_spec = IntegerField(null=True) # (bitmask) allows flagging of targets that have a redshift from a catalogue that is not listed in catalogdb + target_has_spec = IntegerField(null=True) # (bitmask) allows flagging of targets that have a redshift + # from a catalogue that might not be listed in catalogdb + # follow bit pattern in spec compilation + # values < 0 means 'unknown' # Parameters derived from the cross-matched opt/IR catalogue - best_opt = TextField(index=True, null=True) # which optical catalogue(and version) provided this counterpart, e.g. 'ls_dr8', 'ps1_dr2' ... - # will also be the origin of the photometry columns below - ls_id = BigIntegerField(index=True, null=True) # arithmetically derived from ls_release, ls_brickid and ls_objid - # ls_id = ls_objid + ls_brickid * 2**16 + ls_release * 2**40 - # - make sure that we have a common definition within CatalogDB - - ps1_dr2_objid = BigIntegerField(index=True, # Pan-STARRS1-DR2 object id (= ObjectThin.ObjID = StackObjectThin.ObjID) - null=True) # - - gaia_dr2_source_id = BigIntegerField(index=True, # derived from legacysurvey sweeps OPT_REF_ID when OPT_REF_CAT='G2' - null=True) # - SPIDERS team can also pre-match this for photometry from non-LS catalogues + # which optical catalogue(and version) provided this counterpart, e.g. 'ls_dr8', 'ps1_dr2' ... + # will also be the origin of the photometry columns below + best_opt = TextField(index=True, null=True) + + # arithmetically derived from ls_release, ls_brickid and ls_objid + # ls_id = ls_objid + ls_brickid * 2**16 + ls_release * 2**40 + # - make sure that we have a common definition within CatalogDB + # must be used when ls was the main source of counterparts to erosita source, otherwise is optional + ls_id = BigIntegerField(index=True, null=True) + + # Pan-STARRS1-DR2 object id (= ObjectThin.ObjID = StackObjectThin.ObjID) + # Must be used when ps1-dr2(+unWISE) was the main source of counterparts to an erosita source, otherwise is optional + ps1_dr2_objid = BigIntegerField(index=True, null=True) + + # derived from legacysurvey sweeps OPT_REF_ID when OPT_REF_CAT='G2' + # must be used when ls was the main source of counterparts to erosita source, otherwise is optional + # - SPIDERS team should also pre-match to gaia dr2 when using counterparts from non-LS catalogues + gaia_dr2_source_id = BigIntegerField(index=True, null=True) + + # Corresponds to the unWISE catalog band-merged 'unwise_objid' + # - should be used when ps1-dr2+unWISE was the main source of counterparts to erosita sources, otherwise is optional + unwise_dr1_objid = CharField(index=True, null=True, max_length=16) + + # provisional: + # Corresponds to the DES dr1 coadd_object_id + # - must be used when DES-dr1 was the primary source of counterpart to an erosita source, otherwise is optional + des_dr1_coadd_object_id = BigIntegerField(index=True, null=True) + + # Corresponds to the SDSS DR16 photoObj [[https://www.sdss.org/dr13/help/glossary/#ObjID]] + # - must be used when SDSS photoObj was the primary source of counterpart to an erosita source, otherwise is optional + sdss_dr16_objid = BigIntegerField(index=True, null=True) opt_ra = DoubleField(index=True, null=True) # included for convenience, but are copied from columns in other tables From 280986079cf0e2af23992b044ebd157e015f206c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 16 Mar 2020 17:13:16 -0700 Subject: [PATCH 057/197] Rename to LegacySurveyDR8 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql | 6 +----- .../catalogdb/legacy_survey/legacy_survey_index.sql | 8 +++++--- schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 24cc47dc..9434706a 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -2011,7 +2011,7 @@ class Meta: schema = 'catalogdb' -class LegacySurvey(SDSS5dbModel): +class LegacySurveyDR8(SDSS5dbModel): objid = BigIntegerField(primary_key=True) release = IntegerField(null=True) diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql index 445edf1d..2d80d925 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql @@ -6,7 +6,7 @@ Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/legacysurvey */ -CREATE TABLE catalogdb.legacy_survey ( +CREATE TABLE catalogdb.legacy_survey_dr8 ( release INTEGER, brickid BIGINT, brickname TEXT, @@ -128,7 +128,3 @@ CREATE TABLE catalogdb.legacy_survey ( pmdec_ivar REAL, maskbits INTEGER ) WITHOUT OIDS; - - --- Add the serial PK after copying the data. -ALTER TABLE catalogdb.legacy_survey ADD COLUMN pk BIGSERIAL PRIMARY KEY; diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql index b1dc8ef8..7180b319 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql @@ -1,4 +1,6 @@ +-- Add the serial PK after copying the data. +ALTER TABLE catalogdb.legacy_survey ADD COLUMN pk BIGSERIAL PRIMARY KEY; -CREATE INDEX on catalogdb.legacy_survey (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey; -ANALYZE catalogdb.legacy_survey; +CREATE INDEX on catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey_dr8; +ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load index 03ecbdcc..edfdf279 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load +++ b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load @@ -1,4 +1,4 @@ #!/usr/bin/env bash # encoding: utf-8 -ls ./*.fits.csv | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" +ls ./*.fits.csv | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" From e534ff6a41b63d5f0fa98d1eab9a2d54a8a6f218 Mon Sep 17 00:00:00 2001 From: Tom Dwelly Date: Tue, 17 Mar 2020 14:41:34 +0000 Subject: [PATCH 058/197] added CSC data model, small tweaks to BhmSpidersGenericSuperset --- python/sdssdb/peewee/sdss5db/catalogdb.py | 38 ++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 752a80bb..ab1e061e 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1950,7 +1950,8 @@ class Meta: # The following model (BhmSpidersGenericSuperset) does not need to be represented as a table in the database # it is used only as the parent class of BhmSpidersAGNSuperset, BhmSpidersClustersSuperset -# which are real database tables. Am assuming that PeeWee can handle such a scheme without issues. +# which are real database tables. I am assuming that PeeWee can handle such a scheme without problems +# If not, then we will have to duplicate the class BhmSpidersGenericSuperset(SDSS5dbModel): @@ -1974,12 +1975,12 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): xmatch_method = TextField(null=True) # 'ML+NWAY', 'LR' , 'SDSS_REDMAPPER', 'LS_REDMAPPER', 'HSC_REDMAPPER', 'MCMF' etc xmatch_version = TextField(null=True) # version identifier for cross-matching algorithm xmatch_dist = FloatField(null=True) # separation between X-ray position and opt positions - arcsec - xmatch_qual_metric = FloatField(null=True) # measure of quality of xmatch (e.g. p_any for Nway, LR) - xmatch_qual_flags = IntegerField(null=True) # e.g. NWAY match_flag - treat as bitmask + xmatch_metric = FloatField(null=True) # measure of quality of xmatch (e.g. p_any for Nway, LR) + xmatch_flags = BigIntegerField(null=True) # flavour of match, quality flags, e.g. NWAY match_flag - treat as bitmask - # Parameters that describe the sub-class + priority rank of the object + # Parameters that describe the major class of the object target_class = TextField(null=True) # TBD, but e.g. 'unknown', 'AGN', 'Star', 'Galaxy' - # 3=TDE candidate, 4=Compact object, Cluster BCG=11, Cluster member-galaxy=12, low-z galaxy=13 etc + target_priority = IntegerField(null=True) # allows priority ranking based on info not available in catalogdb target_has_spec = IntegerField(null=True) # (bitmask) allows flagging of targets that have a redshift # from a catalogue that might not be listed in catalogdb @@ -1990,7 +1991,7 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): # which optical catalogue(and version) provided this counterpart, e.g. 'ls_dr8', 'ps1_dr2' ... # will also be the origin of the photometry columns below - best_opt = TextField(index=True, null=True) + best_opt = TextField(null=True) # arithmetically derived from ls_release, ls_brickid and ls_objid # ls_id = ls_objid + ls_brickid * 2**16 + ls_release * 2**40 @@ -2000,7 +2001,7 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): # Pan-STARRS1-DR2 object id (= ObjectThin.ObjID = StackObjectThin.ObjID) # Must be used when ps1-dr2(+unWISE) was the main source of counterparts to an erosita source, otherwise is optional - ps1_dr2_objid = BigIntegerField(index=True, null=True) + ps1_dr2_objid = BigIntegerField(index=True, null=True) # derived from legacysurvey sweeps OPT_REF_ID when OPT_REF_CAT='G2' # must be used when ls was the main source of counterparts to erosita source, otherwise is optional @@ -2059,6 +2060,29 @@ class Meta: +class BhmCsc(SDSS5dbModel): + pk = BigAutoField() + + csc_version = TextField(null=True) + cxo_name = TextField(null=True) + + oir_ra = DoubleField(null=True, index=true) + oir_dec = DoubleField(null=True, index=true) + + mag_g = FloatField(null=True) + mag_r = FloatField(null=True) + mag_i = FloatField(null=True) + mag_z = FloatField(null=True) + mag_H = FloatField(null=True) + + spectrograph = CharField(index=True, null=True, max_length=6) + + class Meta: + table_name = 'bhm_csc' + schema = 'catalogdb' + + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) From daee3e9bff37caca33f7b22d6f2c106ec2473f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 09:37:32 -0700 Subject: [PATCH 059/197] Fix name of table in ORM --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 9434706a..a6ab5ee1 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -2135,7 +2135,7 @@ class LegacySurveyDR8(SDSS5dbModel): maskbits = IntegerField(null=True) class Meta: - table_name = 'legacy_survey' + table_name = 'legacy_survey_dr8' schema = 'catalogdb' primary_key = False From 36a483a2a3e4892b7ca0f8372a7f0912749c65ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 19:43:14 -0600 Subject: [PATCH 060/197] Change permissions --- schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py diff --git a/schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py b/schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py old mode 100644 new mode 100755 From ca3cc049a68143dce4c34f47daf69e8122416299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 18:44:38 -0700 Subject: [PATCH 061/197] Rename to bhm_csc --- .../{csc/v0.1.0/csc.sql => bhm_csc/v0.1.0/bhm_csc.sql} | 4 ++-- .../sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql | 10 ++++++++++ .../csc_load.py => bhm_csc/v0.1.0/bhm_csc_load.py} | 4 ++-- schema/sdss5db/catalogdb/csc/v0.1.0/csc_index.sql | 10 ---------- 4 files changed, 14 insertions(+), 14 deletions(-) rename schema/sdss5db/catalogdb/{csc/v0.1.0/csc.sql => bhm_csc/v0.1.0/bhm_csc.sql} (85%) create mode 100644 schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql rename schema/sdss5db/catalogdb/{csc/v0.1.0/csc_load.py => bhm_csc/v0.1.0/bhm_csc_load.py} (88%) delete mode 100644 schema/sdss5db/catalogdb/csc/v0.1.0/csc_index.sql diff --git a/schema/sdss5db/catalogdb/csc/v0.1.0/csc.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql similarity index 85% rename from schema/sdss5db/catalogdb/csc/v0.1.0/csc.sql rename to schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql index 2cd02696..147fc3fb 100644 --- a/schema/sdss5db/catalogdb/csc/v0.1.0/csc.sql +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql @@ -4,14 +4,14 @@ Catalogue of optical/NIR counterparts to Chandra Source Catalogue v2 sources Note that 'fake' targets have been removed -##Filename rows +# Filename rows # CSC2_stub1_realonly_v0.1.0.fits 86082 Files: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/csc/csc_for_catalogdb_v0 */ -CREATE TABLE catalogdb.csc ( +CREATE TABLE catalogdb.bhm_csc ( id BIGSERIAL PRIMARY KEY, csc_version TEXT, cxo_name TEXT, diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql new file mode 100644 index 00000000..2212c251 --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql @@ -0,0 +1,10 @@ + +CREATE INDEX on catalogdb.bhm_csc (q3c_ang2ipix(oir_ra, oir_dec)); +CLUSTER bhm_csc_q3c_ang2ipix_idx on catalogdb.bhm_csc; +ANALYZE catalogdb.bhm_csc; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_g); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_r); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_i); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_z); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_h); diff --git a/schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py similarity index 88% rename from schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py rename to schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py index 891482b3..27cb8072 100755 --- a/schema/sdss5db/catalogdb/csc/v0.1.0/csc_load.py +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py @@ -3,7 +3,7 @@ # # @Author: José Sánchez-Gallego (gallegoj@uw.edu) # @Date: 2020-03-17 -# @Filename: csc_load.py +# @Filename: bhm_csc_load.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) import astropy.table @@ -23,7 +23,7 @@ def main(): data = astropy.table.Table.read(file_) data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'id'), 0) # Add id pk. - copy_data(data, database, 'csc', schema='catalogdb') + copy_data(data, database, 'bhm_csc', schema='catalogdb') if __name__ == '__main__': diff --git a/schema/sdss5db/catalogdb/csc/v0.1.0/csc_index.sql b/schema/sdss5db/catalogdb/csc/v0.1.0/csc_index.sql deleted file mode 100644 index 4b9d2069..00000000 --- a/schema/sdss5db/catalogdb/csc/v0.1.0/csc_index.sql +++ /dev/null @@ -1,10 +0,0 @@ - -CREATE INDEX on catalogdb.csc (q3c_ang2ipix(oir_ra, oir_dec)); -CLUSTER csc_q3c_ang2ipix_idx on catalogdb.csc; -ANALYZE catalogdb.csc; - -CREATE INDEX CONCURRENTLY ON catalogdb.csc using BTREE (mag_g); -CREATE INDEX CONCURRENTLY ON catalogdb.csc using BTREE (mag_r); -CREATE INDEX CONCURRENTLY ON catalogdb.csc using BTREE (mag_i); -CREATE INDEX CONCURRENTLY ON catalogdb.csc using BTREE (mag_z); -CREATE INDEX CONCURRENTLY ON catalogdb.csc using BTREE (mag_h); From e0481d5cf82deff3c28934ddae65f8ece3a19253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 19:16:26 -0700 Subject: [PATCH 062/197] Change id->pk --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql | 2 +- schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 2dc73a75..e1d36fbd 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -2370,7 +2370,7 @@ class Meta: class BhmCsc(SDSS5dbModel): - id = BigAutoField() + pk = BigAutoField() csc_version = TextField(null=True) cxo_name = TextField(null=True) diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql index 147fc3fb..abeb2fab 100644 --- a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql @@ -12,7 +12,7 @@ Files: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/csc/csc_f */ CREATE TABLE catalogdb.bhm_csc ( - id BIGSERIAL PRIMARY KEY, + pk BIGSERIAL PRIMARY KEY, csc_version TEXT, cxo_name TEXT, oir_ra DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py index 27cb8072..bdce528e 100755 --- a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py @@ -21,7 +21,7 @@ def main(): 'target/catalogs/csc/csc_for_catalogdb_v0/CSC2_stub1_realonly_v0.1.0.fits') data = astropy.table.Table.read(file_) - data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'id'), 0) # Add id pk. + data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. copy_data(data, database, 'bhm_csc', schema='catalogdb') From 9d00fb8d41f0723c7c7da443419aa2ccc6de2329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 19:19:57 -0700 Subject: [PATCH 063/197] Files for eRosita/v0.1.0 --- .../catalogdb/eRosita/v0.1.0/eRosita.sql | 104 ++++++++++++++++++ .../eRosita/v0.1.0/eRosita_index.sql | 13 +++ .../catalogdb/eRosita/v0.1.0/eRosita_load.py | 36 ++++++ 3 files changed, 153 insertions(+) create mode 100644 schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql create mode 100644 schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql create mode 100755 schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql new file mode 100644 index 00000000..a9674a7e --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql @@ -0,0 +1,104 @@ +/* + +Catalogues of optical counterparts to eROSITA sources in the 'eFEDS' performace verification field (aka GAMA09) + +There are two catalogues here, one for 'AGN candidates' and one for 'Galaxy cluster candidates' + +PeeWee proto-data models are available for these two catalogues here: +https://github.com/sdss/sdssdb/blob/add_bhm_peewee_classes1/python/sdssdb/peewee/sdss5db/catalogdb.py +see classes 'BhmSpidersAgnSuperset' and 'BhmSpidersClustersSuperset' + +##Filename rows +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.0.fits 23994 +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.0.fits 19636 + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/eRosita/eFEDS_for_catalogdb_v0 + +*/ + +CREATE TABLE catalogdb.bhm_spiders_clusters_superset ( + pk BIGSERIAL PRIMARY KEY, + ero_version TEXT, + ero_souuid TEXT, + ero_flux REAL, + ero_flux_err REAL, + ero_ext REAL, + ero_ext_err REAL, + ero_ext_like REAL, + ero_det_like REAL, + ero_ra DOUBLE PRECISION, + ero_dec DOUBLE PRECISION, + ero_radec_err REAL, + xmatch_method TEXT, + xmatch_version TEXT, + xmatch_dist REAL, + xmatch_metric REAL, + xmatch_flags BIGINT, + target_class TEXT, + target_priority INTEGER, + target_has_spec INTEGER, + best_opt TEXT, + ls_id BIGINT, + ps1_dr2_objid BIGINT, + gaia_dr2_source_id BIGINT, + unwise_dr1_objid TEXT, + des_dr1_coadd_object_id BIGINT, + sdss_dr16_objid BIGINT, + opt_ra DOUBLE PRECISION, + opt_dec DOUBLE PRECISION, + opt_pmra REAL, + opt_pmdec REAL, + opt_epoch REAL, + opt_modelflux_g REAL, + opt_modelflux_ivar_g REAL, + opt_modelflux_r REAL, + opt_modelflux_ivar_r REAL, + opt_modelflux_i REAL, + opt_modelflux_ivar_i REAL, + opt_modelflux_z REAL, + opt_modelflux_ivar_z REAL +) WITHOUT OIDS; + + +CREATE TABLE catalogdb.bhm_spiders_agn_superset ( + pk BIGSERIAL PRIMARY KEY, + ero_version TEXT, + ero_souuid TEXT, + ero_flux REAL, + ero_flux_err REAL, + ero_ext REAL, + ero_ext_err REAL, + ero_ext_like REAL, + ero_det_like REAL, + ero_ra DOUBLE PRECISION, + ero_dec DOUBLE PRECISION, + ero_radec_err REAL, + xmatch_method TEXT, + xmatch_version TEXT, + xmatch_dist REAL, + xmatch_metric REAL, + xmatch_flags BIGINT, + target_class TEXT, + target_priority INTEGER, + target_has_spec INTEGER, + best_opt TEXT, + ls_id BIGINT, + ps1_dr2_objid BIGINT, + gaia_dr2_source_id BIGINT, + unwise_dr1_objid TEXT, + des_dr1_coadd_object_id BIGINT, + sdss_dr16_objid BIGINT, + opt_ra DOUBLE PRECISION, + opt_dec DOUBLE PRECISION, + opt_pmra REAL, + opt_pmdec REAL, + opt_epoch REAL, + opt_modelflux_g REAL, + opt_modelflux_ivar_g REAL, + opt_modelflux_r REAL, + opt_modelflux_ivar_r REAL, + opt_modelflux_i REAL, + opt_modelflux_ivar_i REAL, + opt_modelflux_z REAL, + opt_modelflux_ivar_z REAL +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql new file mode 100644 index 00000000..87676fb9 --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql @@ -0,0 +1,13 @@ + +CREATE INDEX on catalogdb.bhm_spiders_agn_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CLUSTER bhm_spiders_agn_superset_q3c_ang2ipix_idx on catalogdb.bhm_spiders_agn_superset; +ANALYZE catalogdb.bhm_spiders_agn_superset; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset using BTREE (ero_flux); + + +CREATE INDEX on catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CLUSTER bhm_spiders_clusters_superset_q3c_ang2ipix_idx on catalogdb.bhm_spiders_clusters_superset; +ANALYZE catalogdb.bhm_spiders_clusters_superset; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset using BTREE (ero_flux); diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py new file mode 100755 index 00000000..fd62c5b6 --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-17 +# @Filename: eRosita_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table +import numpy + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' + 'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/') + + for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.0.fits', + 'bhm_spiders_clusters_superset'), + ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.0.fits', + 'bhm_spiders_agn_superset')): + + data = astropy.table.Table.read(path_ + file_) + data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. + + copy_data(data, database, table, schema='catalogdb') + + +if __name__ == '__main__': + + main() From dd9046c6c576cc44e6a40a420979550bc52398a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 20:40:50 -0700 Subject: [PATCH 064/197] Deprecate WD v1 --- .../catalogdb/wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1.sql | 0 .../wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1_index.sql | 0 .../wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1_load | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename schema/sdss5db/catalogdb/wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1.sql (100%) rename schema/sdss5db/catalogdb/wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1_index.sql (100%) rename schema/sdss5db/catalogdb/wd/{v1 => deprecated}/gaia_dr2_wd_candidates_v1_load (100%) diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1.sql b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1.sql similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1.sql rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1.sql diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_index.sql b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_index.sql similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_index.sql rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_index.sql diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_load b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_load similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_load rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_load From 8eb31287c2e39bcdee4b117094211ba1fd88feb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 21:05:47 -0700 Subject: [PATCH 065/197] Loading scripts for WD DR2 --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 0 .../sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 345 ++++++++++++++++++ .../sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 47 +++ 3 files changed, 392 insertions(+) create mode 100644 schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql create mode 100644 schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql create mode 100644 schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql new file mode 100644 index 00000000..e69de29b diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql new file mode 100644 index 00000000..81cd9947 --- /dev/null +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -0,0 +1,345 @@ +/* + +From Gentile Fusillo N.P. , Tremblay P-E, Gaensicke B.T. et..al, 2019MNRAS.482.4570G + +Downloaded from https://cdsarc.unistra.fr/viz-bin/cat/J/MNRAS/482/4570 + +J/MNRAS/482/4570 Gaia DR2 white dwarf candidates (Gentile Fusillo+, 2019) +================================================================================ +A Gaia Data Release 2 catalogue of white dwarfs and a comparison with SDSS. + Gentile Fusillo N.P., Tremblay P-E, Gaensicke B.T., Manser C.J., + Cunningham T., Cukanovaite E., Hollands M., Marsh T., Raddi R., + Jordan S., Toonen S., Geier S., Barstow M., Cummings J.D. + + =2019MNRAS.482.4570G (SIMBAD/NED BibCode) +================================================================================ +ADC_Keywords: Surveys ; Stars, white dwarf ; Parallaxes, trigonometric; + Photometry +Keywords: catalogues - surveys - white dwarfs + +Abstract: + We present a catalogue of white dwarf candidates selected from the + second data release of Gaia (DR2). We used a sample of + spectroscopically confirmed white dwarfs from the Sloan Digital Sky + Survey (SDSS) to map the entire space spanned by these objects in the + Gaia Hertzsprung-Russell diagram. + + We then defined a set of cuts in absolute magnitude, colour, and a + number of Gaia quality flags to remove the majority of contaminating + objects. Finally, we adopt a method analogous to the one presented in + our earlier SDSS photometric catalogues to calculate a probability of + being a white dwarf (PWD) for all Gaia sources which passed the + initial selection. The final catalogue is composed of 486641 stars + with calculated PWD from which it is possible to select a sample + of~260000 high-confidence white dwarf candidates in the magnitude + range 87000K, at high + Galactic latitudes (|b|>20deg). However, the completeness drops at + low Galactic latitudes, and the magnitude limit of the catalogue + varies significantly across the sky as a function of Gaia's scanning + law. We also provide the list of objects within our sample with + available SDSS spectroscopy. + +Description: + The main catalogue provides 486,641 stars selected from Gaia DR2 with + calculated probabilities of being a white dwarf (PWD). The PWD values + can used to reliably select high-confidence white dwarf candidates + with a flexible compromise between completeness and level of potential + contamination. As a generic guideline selecting objects with PWD>0.75 + recovers 96 per cent of the spectroscopically confirmed white dwarfs + from SDSS and only 1 per cent of the contaminant (non white dwarfs) + objects. + + All Gaia sources in the catalogue have also been cross matched with + SDSS DR14 taking into account the difference in epoch of observation + and proper motions. Whether available we include SDSS ugriz + photometry. In a separate table we provide informations on all the + available SDSS spectra for the Gaia sources in the main catalogue. + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +gaia2wd.dat 1008 486641 DR2 white dwarf candidates, corrected version + (gaia_dr2_white_dwarf_candidates.dat) +gaiasdss.dat 344 37259 Available SDSS spectra of candidates, corrected + version (gaia-sdss_white_dwarf_catalogue.dat) +-------------------------------------------------------------------------------- + +See also: + B/wd : Spectroscopically identified white dwarfs (McCook+, 2014) + I/345 : Gaia DR2 (Gaia Collaboration, 2018) + V/147 : The SDSS Photometric Catalogue, Release 12 (Alam+, 2015) + J/MNRAS/465/2849 : Gaia DR1 mass-radius relation of WD (Tremblay+ 2017) + +Byte-by-byte Description of file: gaia2wd.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 23 A23 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 25- 52 A28 --- DR2Name Unique Gaia source designation (designation) + 54- 72 I19 --- Source Unique Gaia source identifier (source_id) + 74- 96 F23.19 deg RAdeg Gaia DR2 barycentric right ascension (ICRS) + at epoch J2015.5 (ra) + 98- 117 F20.18 mas e_RAdeg Standard error of right ascension (ra_error) + 119- 140 E22.18 deg DEdeg Gaia DR2 barycentric declination (ICRS) + at epoch J2015.5 (dec) + 142- 161 F20.18 mas e_DEdeg Standard error of declination (dec_error) + 163- 183 F21.17 mas Plx Absolute stellar parallax of the source + at J2015.5 (parallax) + 185- 204 F20.18 mas e_Plx Standard error of parallax (parallax_error) + 206- 227 E22.19 mas/yr pmRA Proper motion in right ascension + (pmRAxcosDE) (pmra) + 229- 248 F20.18 mas/yr e_pmRA Standard error of proper motion in + right ascension (pmra_error) + 250- 271 E22.19 mas/yr pmDE Proper motion in declination (pmdec) + 273- 292 F20.18 mas/yr e_pmDE Standard error of proper motion + in declination (pmdec_error) + 294- 315 F22.19 mas epsi Measure of the residuals in the astrometric + solution for the source + (astrometric_excess_noise) + 317- 327 F11.9 mas amax Five-dimensional equivalent to the + semi-major axis of the Gaia position error + ellipse (astrometric_sigma5d_max) + 329- 350 F22.14 e-/s FG Gaia G-band mean flux (phot_g_mean_flux) + 352- 374 F23.17 e-/s e_FG Error on G-band mean flux + (phot_g_mean_flux_error) + 376- 385 F10.7 mag Gmag Gaia G-band mean magnitude (Vega scale) + (phot_g_mean_mag) + 387- 409 F23.15 e-/s FBP Integrated GBP mean flux (phot_bp_mean_flux) + 411- 435 F25.18 e-/s e_FBP Error on GBP-band mean flux + (phot_bp_mean_flux_error) + 437- 446 F10.7 mag BPmag Gaia GBP-band mean magnitude (Vega scale) + (phot_bp_mean_mag) + 448- 470 F23.15 e-/s FRP Integrated GRP mean flux (phot_rp_mean_flux) + 472- 497 F26.19 e-/s e_FRP Error on GRP-band mean flux + (phot_rp_mean_flux_error) + 499- 508 F10.7 mag RPmag Gaia GRP-band mean magnitude (Vega scale) + (phot_rp_mean_mag) + 510- 520 F11.9 --- E(BR/RP) GBP/GRP excess factor estimated from the + comparison of the sum of integrated GBP and + GRP fluxes with respect to the flux in the + G-band (phot_bp_rp_excess_factor) + 522- 542 E21.19 deg GLON Galactic longitude (l) + 544- 565 E22.19 deg GLAT Galactic latitude (b) + 567- 578 F12.4 --- Density The number of Gaia sources per square degree + around this object (density) + 580- 602 F23.19 mag AG Extinction in the Gaia G-band band derived + from E(B-V) values from Schlafly and + Finkbeiner (2011ApJ...737..103S) (AG) + 604- 622 A19 --- SDSS SDSS object name if available + (JHHMMSS,.ss+DDMMSS.s, J2000) (SDSS_name) + 624- 641 F18.15 mag umag ? SDSS u band magnitude (umag) + 643- 664 F22.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 666- 683 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 685- 706 F22.19 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 708- 725 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 727- 747 F21.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 749- 766 F18.15 mag imag ? SDSS i band magnitude (imag) + 768- 789 F22.19 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 791- 808 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 810- 830 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 832- 852 E21.19 --- Pwd [0/1]? The probability of being a white + dwarf (Pwd) + 854 I1 --- f_Pwd [0/1] If 1 it indicates the PWD value + could be unreliable (Pwd_flag) + 856- 868 F13.6 K TeffH ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-H model atmospheres (Teff_H) + 870- 882 F13.6 K e_TeffH ? Uncertainty on Teff_H (eTeff_H) + 884- 891 F8.6 --- loggH ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-H model atmospheres (log_g_H) + 893- 900 E8.6 --- e_loggH ? Uncertainty on log_g_H (elog_g_H) + 902- 909 F8.6 Msun MassH ? Stellar mass resulting from the adopted + mass-radius relation (mass_H) + 911- 919 E9.6 Msun e_MassH ? Uncertainty on the mass (emass_H) + 921- 931 E11.6 --- chi2H ? chi2 value of the fit (pure-H) (chi2_H) + 933- 944 F12.6 K TeffHe ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-He model atmospheres (Teff_He) + 946- 959 F14.6 K e_TeffHe ? Uncertainty on Teff_He (eTeff_He) + 961- 968 F8.6 --- loggHe ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-He model atmospheres (log_g_He) + 970- 977 E8.6 --- e_loggHe ? Uncertainty on log_g_He (elog_g_He) + 979- 986 F8.6 Msun MassHe ? Stellar mass resulting from the adopted + mass-radius relation (mass_He) + 988- 996 E9.6 Msun e_MassHe ? Uncertainty on the mass (emass_He) + + 998-1008 E11.6 --- chisqHe ? chi2 value of the fit (pure-H) (chisq_He) +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: gaiasdss.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 22 A22 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 23 A1 --- n_WD [a] Multiplicity index on WD + 24- 42 I19 --- Source Unique Gaia source identifier (source_id) + 44- 62 A19 --- SDSS SDSS object name (JHHMMSS,.ss+DDMMSS.s, J2000) + (SDSS_name) + 64- 83 F20.16 deg RAdeg Right ascension (J2000) of the spectrum source + from SDSS DR14 (SDSS_ra) + 85- 98 E14.10 deg DEdeg Declination (J2000) of the spectrum source + from SDSS DR14 (SDSS_dec) + 100-117 F18.15 mag umag ? SDSS u band magnitude (umag) + 119-139 F21.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 141-158 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 160-179 F20.18 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 181-198 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 200-219 F20.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 221-238 F18.15 mag imag ? SDSS i band magnitude (imag) + 240-260 F21.18 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 262-279 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 281-301 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 303-306 I4 --- Plate Identifier of the plate used in the + observation of the spectrum (Plate) + 308-312 I5 --- MJD Modified Julian date of the observation of + the spectrum (mjd) + 314-317 I4 --- Fiber Identifier of the fiber used in the + observation of the spectrum (fiberID) + 319-333 F15.11 --- S/N Signal-to-noise ratio of the spectrum + calculated in the range 4500-5500 A (S/N) + 335-344 A10 --- SpClass Classification of the object based on a visual + inspection of the SDSS spectrum + (spectral_class) (1) +-------------------------------------------------------------------------------- +Note (1): spectral classes are as follows: + CV = Cataclysmic Variable star + DA = white dwarfs with H lines only + DAB/DAB = white dwarfs with H lines and neutral He lines + DABZ/DBAZ/DZAB/DZBA = white dwarfs with H lines, neutral He lines and + Ca H & K lines + DAO/DO = white dwarfs with H lines and He II lines + DAZ/DZA = white dwarfs with H lines and Ca H & K lines + DB = white dwarfs with neutral He lines only + DBZ/DZB = white dwarfs with neutral He lines and Ca H & K lines + DC = white dwarfs with feature-less spectra + DQ = white dwarfs with molecular C absorptions + hotDQ = white dwarfs with atomic C absorption + DQpec = Peculiar white dwarfs with molecular C absorptions + DZ = white dwarfs with Ca H & K lines + DAH = white dwarfs with Zeeman split H lines + DBH = white dwarfs with Zeeman split He lines + DZH = white dwarfs with Zeeman split Ca H & K lines + MWD = Magnetic white dwarfs with unidentified absorption lines + WDPec = Peculiar magnetic white dwarf + PG1159 = Pre-white dwarf star + DB+MS/DA+MS/DC+M = White Dwarf - Main Sequence binary + STAR = non white dwarf stellar object + QSO = Quasar + UNK = Spectra which we were unable to classify + Unreliable = Unreliable spectra +-------------------------------------------------------------------------------- + +Acknowledgements: + Nicola Gentile Fusillo, n.gentile-fusillo(at)warwick.ac.uk + +History: + * 01-Feb-2019: on-line version + * 01-Apr-2019: corrected version (from author) + +================================================================================ +(End) Patricia Vannier [CDS] 31-Jan-2019 + +Modifications to datamodels: + +- All columns to lower case. +- All column except coordinates have been converted from float64 to float32. +- source -> source_id. +- RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec +- pm_de, e_pmde -> pm_dec, e_pmdec +- E(BR/RP) -> e_br_rp +- (S/N) -> snr + +*/ + + +CREATE TABLE catalogdb.wd_dr2 ( + wd TEXT, + dr2name TEXT, + source_id INTEGER PRIMARY KEY, + ra DOUBLE PRECISION, + e_ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + e_dec DOUBLE PRECISION, + plx REAL, + e_plx REAL, + pmra DOUBLE PRECISION, + e_pmra DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + e_pmdec DOUBLE PRECISION, + epsi REAL, + amax REAL, + fg REAL, + e_fg REAL, + gmag REAL, + fbp REAL, + e_fbp REAL, + bpmag REAL, + frp REAL, + e_frp REAL, + rpmag REAL, + e_br_rp REAL, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + density REAL, + ag REAL, + sdss TEXT, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + pwd REAL, + f_pwd INTEGER, + teffh REAL, + e_teffh REAL, + loggh REAL, + e_loggh REAL, + massh REAL, + e_massh REAL, + chi2h REAL, + teffhe REAL, + e_teffhe REAL, + logghe REAL, + e_logghe REAL, + masshe REAL, + e_masshe REAL, + chisqhe REAL +) WITHOUT OIDS; + + +CREATE TABLE catalogdb.wd_dr2_sdss ( + wd TEXT, + n_wd TEXT, + source_id BIGINT PRIMARY KEY, + sdss TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + plate INTEGER, + mjd INTEGER, + fiber INTEGER, + snr REAL, + spclass TEXT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py new file mode 100644 index 00000000..c393d1b1 --- /dev/null +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-17 +# @Filename: wd_dr2_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table +import numpy + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + replacements = {'source': 'source_id', + 'radeg': 'ra', + 'dedeg': 'dec', + 'e_RAdeg': 'e_ra', + 'e_DEdeg': 'e_dec', + 'pm_de': 'pm_dec', + 'e_pmde': 'e_pmdec', + 'e(br/rp)': 'e_br_rp', + '(s/n)': 'snr'} + + path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/') + + for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz', 'wd_dr2'), + ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'wd_dr2_sdss')): + + data = astropy.table.Table.read(path_ + file_) + for column_name in data.colnames: + new_column_name = column_name.lower() + if new_column_name in replacements: + new_column_name = replacements[new_column_name] + data.rename_column(column_name, new_column_name) + + copy_data(data, database, table, schema='catalogdb') + + +if __name__ == '__main__': + + main() From 415fcf8f556f83855ddf75fb7804f64c0f56e835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 22:09:04 -0600 Subject: [PATCH 066/197] Change permissions --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py old mode 100644 new mode 100755 From 63aa54dc8f46ee83f36cea6804fa12451b0cdd00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 21:09:23 -0700 Subject: [PATCH 067/197] Move table creation to correct file --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 345 ++++++++++++++++++ .../sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 345 ------------------ 2 files changed, 345 insertions(+), 345 deletions(-) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql index e69de29b..81cd9947 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -0,0 +1,345 @@ +/* + +From Gentile Fusillo N.P. , Tremblay P-E, Gaensicke B.T. et..al, 2019MNRAS.482.4570G + +Downloaded from https://cdsarc.unistra.fr/viz-bin/cat/J/MNRAS/482/4570 + +J/MNRAS/482/4570 Gaia DR2 white dwarf candidates (Gentile Fusillo+, 2019) +================================================================================ +A Gaia Data Release 2 catalogue of white dwarfs and a comparison with SDSS. + Gentile Fusillo N.P., Tremblay P-E, Gaensicke B.T., Manser C.J., + Cunningham T., Cukanovaite E., Hollands M., Marsh T., Raddi R., + Jordan S., Toonen S., Geier S., Barstow M., Cummings J.D. + + =2019MNRAS.482.4570G (SIMBAD/NED BibCode) +================================================================================ +ADC_Keywords: Surveys ; Stars, white dwarf ; Parallaxes, trigonometric; + Photometry +Keywords: catalogues - surveys - white dwarfs + +Abstract: + We present a catalogue of white dwarf candidates selected from the + second data release of Gaia (DR2). We used a sample of + spectroscopically confirmed white dwarfs from the Sloan Digital Sky + Survey (SDSS) to map the entire space spanned by these objects in the + Gaia Hertzsprung-Russell diagram. + + We then defined a set of cuts in absolute magnitude, colour, and a + number of Gaia quality flags to remove the majority of contaminating + objects. Finally, we adopt a method analogous to the one presented in + our earlier SDSS photometric catalogues to calculate a probability of + being a white dwarf (PWD) for all Gaia sources which passed the + initial selection. The final catalogue is composed of 486641 stars + with calculated PWD from which it is possible to select a sample + of~260000 high-confidence white dwarf candidates in the magnitude + range 87000K, at high + Galactic latitudes (|b|>20deg). However, the completeness drops at + low Galactic latitudes, and the magnitude limit of the catalogue + varies significantly across the sky as a function of Gaia's scanning + law. We also provide the list of objects within our sample with + available SDSS spectroscopy. + +Description: + The main catalogue provides 486,641 stars selected from Gaia DR2 with + calculated probabilities of being a white dwarf (PWD). The PWD values + can used to reliably select high-confidence white dwarf candidates + with a flexible compromise between completeness and level of potential + contamination. As a generic guideline selecting objects with PWD>0.75 + recovers 96 per cent of the spectroscopically confirmed white dwarfs + from SDSS and only 1 per cent of the contaminant (non white dwarfs) + objects. + + All Gaia sources in the catalogue have also been cross matched with + SDSS DR14 taking into account the difference in epoch of observation + and proper motions. Whether available we include SDSS ugriz + photometry. In a separate table we provide informations on all the + available SDSS spectra for the Gaia sources in the main catalogue. + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +gaia2wd.dat 1008 486641 DR2 white dwarf candidates, corrected version + (gaia_dr2_white_dwarf_candidates.dat) +gaiasdss.dat 344 37259 Available SDSS spectra of candidates, corrected + version (gaia-sdss_white_dwarf_catalogue.dat) +-------------------------------------------------------------------------------- + +See also: + B/wd : Spectroscopically identified white dwarfs (McCook+, 2014) + I/345 : Gaia DR2 (Gaia Collaboration, 2018) + V/147 : The SDSS Photometric Catalogue, Release 12 (Alam+, 2015) + J/MNRAS/465/2849 : Gaia DR1 mass-radius relation of WD (Tremblay+ 2017) + +Byte-by-byte Description of file: gaia2wd.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 23 A23 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 25- 52 A28 --- DR2Name Unique Gaia source designation (designation) + 54- 72 I19 --- Source Unique Gaia source identifier (source_id) + 74- 96 F23.19 deg RAdeg Gaia DR2 barycentric right ascension (ICRS) + at epoch J2015.5 (ra) + 98- 117 F20.18 mas e_RAdeg Standard error of right ascension (ra_error) + 119- 140 E22.18 deg DEdeg Gaia DR2 barycentric declination (ICRS) + at epoch J2015.5 (dec) + 142- 161 F20.18 mas e_DEdeg Standard error of declination (dec_error) + 163- 183 F21.17 mas Plx Absolute stellar parallax of the source + at J2015.5 (parallax) + 185- 204 F20.18 mas e_Plx Standard error of parallax (parallax_error) + 206- 227 E22.19 mas/yr pmRA Proper motion in right ascension + (pmRAxcosDE) (pmra) + 229- 248 F20.18 mas/yr e_pmRA Standard error of proper motion in + right ascension (pmra_error) + 250- 271 E22.19 mas/yr pmDE Proper motion in declination (pmdec) + 273- 292 F20.18 mas/yr e_pmDE Standard error of proper motion + in declination (pmdec_error) + 294- 315 F22.19 mas epsi Measure of the residuals in the astrometric + solution for the source + (astrometric_excess_noise) + 317- 327 F11.9 mas amax Five-dimensional equivalent to the + semi-major axis of the Gaia position error + ellipse (astrometric_sigma5d_max) + 329- 350 F22.14 e-/s FG Gaia G-band mean flux (phot_g_mean_flux) + 352- 374 F23.17 e-/s e_FG Error on G-band mean flux + (phot_g_mean_flux_error) + 376- 385 F10.7 mag Gmag Gaia G-band mean magnitude (Vega scale) + (phot_g_mean_mag) + 387- 409 F23.15 e-/s FBP Integrated GBP mean flux (phot_bp_mean_flux) + 411- 435 F25.18 e-/s e_FBP Error on GBP-band mean flux + (phot_bp_mean_flux_error) + 437- 446 F10.7 mag BPmag Gaia GBP-band mean magnitude (Vega scale) + (phot_bp_mean_mag) + 448- 470 F23.15 e-/s FRP Integrated GRP mean flux (phot_rp_mean_flux) + 472- 497 F26.19 e-/s e_FRP Error on GRP-band mean flux + (phot_rp_mean_flux_error) + 499- 508 F10.7 mag RPmag Gaia GRP-band mean magnitude (Vega scale) + (phot_rp_mean_mag) + 510- 520 F11.9 --- E(BR/RP) GBP/GRP excess factor estimated from the + comparison of the sum of integrated GBP and + GRP fluxes with respect to the flux in the + G-band (phot_bp_rp_excess_factor) + 522- 542 E21.19 deg GLON Galactic longitude (l) + 544- 565 E22.19 deg GLAT Galactic latitude (b) + 567- 578 F12.4 --- Density The number of Gaia sources per square degree + around this object (density) + 580- 602 F23.19 mag AG Extinction in the Gaia G-band band derived + from E(B-V) values from Schlafly and + Finkbeiner (2011ApJ...737..103S) (AG) + 604- 622 A19 --- SDSS SDSS object name if available + (JHHMMSS,.ss+DDMMSS.s, J2000) (SDSS_name) + 624- 641 F18.15 mag umag ? SDSS u band magnitude (umag) + 643- 664 F22.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 666- 683 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 685- 706 F22.19 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 708- 725 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 727- 747 F21.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 749- 766 F18.15 mag imag ? SDSS i band magnitude (imag) + 768- 789 F22.19 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 791- 808 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 810- 830 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 832- 852 E21.19 --- Pwd [0/1]? The probability of being a white + dwarf (Pwd) + 854 I1 --- f_Pwd [0/1] If 1 it indicates the PWD value + could be unreliable (Pwd_flag) + 856- 868 F13.6 K TeffH ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-H model atmospheres (Teff_H) + 870- 882 F13.6 K e_TeffH ? Uncertainty on Teff_H (eTeff_H) + 884- 891 F8.6 --- loggH ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-H model atmospheres (log_g_H) + 893- 900 E8.6 --- e_loggH ? Uncertainty on log_g_H (elog_g_H) + 902- 909 F8.6 Msun MassH ? Stellar mass resulting from the adopted + mass-radius relation (mass_H) + 911- 919 E9.6 Msun e_MassH ? Uncertainty on the mass (emass_H) + 921- 931 E11.6 --- chi2H ? chi2 value of the fit (pure-H) (chi2_H) + 933- 944 F12.6 K TeffHe ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-He model atmospheres (Teff_He) + 946- 959 F14.6 K e_TeffHe ? Uncertainty on Teff_He (eTeff_He) + 961- 968 F8.6 --- loggHe ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-He model atmospheres (log_g_He) + 970- 977 E8.6 --- e_loggHe ? Uncertainty on log_g_He (elog_g_He) + 979- 986 F8.6 Msun MassHe ? Stellar mass resulting from the adopted + mass-radius relation (mass_He) + 988- 996 E9.6 Msun e_MassHe ? Uncertainty on the mass (emass_He) + + 998-1008 E11.6 --- chisqHe ? chi2 value of the fit (pure-H) (chisq_He) +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: gaiasdss.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 22 A22 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 23 A1 --- n_WD [a] Multiplicity index on WD + 24- 42 I19 --- Source Unique Gaia source identifier (source_id) + 44- 62 A19 --- SDSS SDSS object name (JHHMMSS,.ss+DDMMSS.s, J2000) + (SDSS_name) + 64- 83 F20.16 deg RAdeg Right ascension (J2000) of the spectrum source + from SDSS DR14 (SDSS_ra) + 85- 98 E14.10 deg DEdeg Declination (J2000) of the spectrum source + from SDSS DR14 (SDSS_dec) + 100-117 F18.15 mag umag ? SDSS u band magnitude (umag) + 119-139 F21.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 141-158 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 160-179 F20.18 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 181-198 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 200-219 F20.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 221-238 F18.15 mag imag ? SDSS i band magnitude (imag) + 240-260 F21.18 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 262-279 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 281-301 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 303-306 I4 --- Plate Identifier of the plate used in the + observation of the spectrum (Plate) + 308-312 I5 --- MJD Modified Julian date of the observation of + the spectrum (mjd) + 314-317 I4 --- Fiber Identifier of the fiber used in the + observation of the spectrum (fiberID) + 319-333 F15.11 --- S/N Signal-to-noise ratio of the spectrum + calculated in the range 4500-5500 A (S/N) + 335-344 A10 --- SpClass Classification of the object based on a visual + inspection of the SDSS spectrum + (spectral_class) (1) +-------------------------------------------------------------------------------- +Note (1): spectral classes are as follows: + CV = Cataclysmic Variable star + DA = white dwarfs with H lines only + DAB/DAB = white dwarfs with H lines and neutral He lines + DABZ/DBAZ/DZAB/DZBA = white dwarfs with H lines, neutral He lines and + Ca H & K lines + DAO/DO = white dwarfs with H lines and He II lines + DAZ/DZA = white dwarfs with H lines and Ca H & K lines + DB = white dwarfs with neutral He lines only + DBZ/DZB = white dwarfs with neutral He lines and Ca H & K lines + DC = white dwarfs with feature-less spectra + DQ = white dwarfs with molecular C absorptions + hotDQ = white dwarfs with atomic C absorption + DQpec = Peculiar white dwarfs with molecular C absorptions + DZ = white dwarfs with Ca H & K lines + DAH = white dwarfs with Zeeman split H lines + DBH = white dwarfs with Zeeman split He lines + DZH = white dwarfs with Zeeman split Ca H & K lines + MWD = Magnetic white dwarfs with unidentified absorption lines + WDPec = Peculiar magnetic white dwarf + PG1159 = Pre-white dwarf star + DB+MS/DA+MS/DC+M = White Dwarf - Main Sequence binary + STAR = non white dwarf stellar object + QSO = Quasar + UNK = Spectra which we were unable to classify + Unreliable = Unreliable spectra +-------------------------------------------------------------------------------- + +Acknowledgements: + Nicola Gentile Fusillo, n.gentile-fusillo(at)warwick.ac.uk + +History: + * 01-Feb-2019: on-line version + * 01-Apr-2019: corrected version (from author) + +================================================================================ +(End) Patricia Vannier [CDS] 31-Jan-2019 + +Modifications to datamodels: + +- All columns to lower case. +- All column except coordinates have been converted from float64 to float32. +- source -> source_id. +- RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec +- pm_de, e_pmde -> pm_dec, e_pmdec +- E(BR/RP) -> e_br_rp +- (S/N) -> snr + +*/ + + +CREATE TABLE catalogdb.wd_dr2 ( + wd TEXT, + dr2name TEXT, + source_id INTEGER PRIMARY KEY, + ra DOUBLE PRECISION, + e_ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + e_dec DOUBLE PRECISION, + plx REAL, + e_plx REAL, + pmra DOUBLE PRECISION, + e_pmra DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + e_pmdec DOUBLE PRECISION, + epsi REAL, + amax REAL, + fg REAL, + e_fg REAL, + gmag REAL, + fbp REAL, + e_fbp REAL, + bpmag REAL, + frp REAL, + e_frp REAL, + rpmag REAL, + e_br_rp REAL, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + density REAL, + ag REAL, + sdss TEXT, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + pwd REAL, + f_pwd INTEGER, + teffh REAL, + e_teffh REAL, + loggh REAL, + e_loggh REAL, + massh REAL, + e_massh REAL, + chi2h REAL, + teffhe REAL, + e_teffhe REAL, + logghe REAL, + e_logghe REAL, + masshe REAL, + e_masshe REAL, + chisqhe REAL +) WITHOUT OIDS; + + +CREATE TABLE catalogdb.wd_dr2_sdss ( + wd TEXT, + n_wd TEXT, + source_id BIGINT PRIMARY KEY, + sdss TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + plate INTEGER, + mjd INTEGER, + fiber INTEGER, + snr REAL, + spclass TEXT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql index 81cd9947..e69de29b 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -1,345 +0,0 @@ -/* - -From Gentile Fusillo N.P. , Tremblay P-E, Gaensicke B.T. et..al, 2019MNRAS.482.4570G - -Downloaded from https://cdsarc.unistra.fr/viz-bin/cat/J/MNRAS/482/4570 - -J/MNRAS/482/4570 Gaia DR2 white dwarf candidates (Gentile Fusillo+, 2019) -================================================================================ -A Gaia Data Release 2 catalogue of white dwarfs and a comparison with SDSS. - Gentile Fusillo N.P., Tremblay P-E, Gaensicke B.T., Manser C.J., - Cunningham T., Cukanovaite E., Hollands M., Marsh T., Raddi R., - Jordan S., Toonen S., Geier S., Barstow M., Cummings J.D. - - =2019MNRAS.482.4570G (SIMBAD/NED BibCode) -================================================================================ -ADC_Keywords: Surveys ; Stars, white dwarf ; Parallaxes, trigonometric; - Photometry -Keywords: catalogues - surveys - white dwarfs - -Abstract: - We present a catalogue of white dwarf candidates selected from the - second data release of Gaia (DR2). We used a sample of - spectroscopically confirmed white dwarfs from the Sloan Digital Sky - Survey (SDSS) to map the entire space spanned by these objects in the - Gaia Hertzsprung-Russell diagram. - - We then defined a set of cuts in absolute magnitude, colour, and a - number of Gaia quality flags to remove the majority of contaminating - objects. Finally, we adopt a method analogous to the one presented in - our earlier SDSS photometric catalogues to calculate a probability of - being a white dwarf (PWD) for all Gaia sources which passed the - initial selection. The final catalogue is composed of 486641 stars - with calculated PWD from which it is possible to select a sample - of~260000 high-confidence white dwarf candidates in the magnitude - range 87000K, at high - Galactic latitudes (|b|>20deg). However, the completeness drops at - low Galactic latitudes, and the magnitude limit of the catalogue - varies significantly across the sky as a function of Gaia's scanning - law. We also provide the list of objects within our sample with - available SDSS spectroscopy. - -Description: - The main catalogue provides 486,641 stars selected from Gaia DR2 with - calculated probabilities of being a white dwarf (PWD). The PWD values - can used to reliably select high-confidence white dwarf candidates - with a flexible compromise between completeness and level of potential - contamination. As a generic guideline selecting objects with PWD>0.75 - recovers 96 per cent of the spectroscopically confirmed white dwarfs - from SDSS and only 1 per cent of the contaminant (non white dwarfs) - objects. - - All Gaia sources in the catalogue have also been cross matched with - SDSS DR14 taking into account the difference in epoch of observation - and proper motions. Whether available we include SDSS ugriz - photometry. In a separate table we provide informations on all the - available SDSS spectra for the Gaia sources in the main catalogue. - -File Summary: --------------------------------------------------------------------------------- - FileName Lrecl Records Explanations --------------------------------------------------------------------------------- -ReadMe 80 . This file -gaia2wd.dat 1008 486641 DR2 white dwarf candidates, corrected version - (gaia_dr2_white_dwarf_candidates.dat) -gaiasdss.dat 344 37259 Available SDSS spectra of candidates, corrected - version (gaia-sdss_white_dwarf_catalogue.dat) --------------------------------------------------------------------------------- - -See also: - B/wd : Spectroscopically identified white dwarfs (McCook+, 2014) - I/345 : Gaia DR2 (Gaia Collaboration, 2018) - V/147 : The SDSS Photometric Catalogue, Release 12 (Alam+, 2015) - J/MNRAS/465/2849 : Gaia DR1 mass-radius relation of WD (Tremblay+ 2017) - -Byte-by-byte Description of file: gaia2wd.dat --------------------------------------------------------------------------------- - Bytes Format Units Label Explanations --------------------------------------------------------------------------------- - 1- 23 A23 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) - (white_dwarf_name) - 25- 52 A28 --- DR2Name Unique Gaia source designation (designation) - 54- 72 I19 --- Source Unique Gaia source identifier (source_id) - 74- 96 F23.19 deg RAdeg Gaia DR2 barycentric right ascension (ICRS) - at epoch J2015.5 (ra) - 98- 117 F20.18 mas e_RAdeg Standard error of right ascension (ra_error) - 119- 140 E22.18 deg DEdeg Gaia DR2 barycentric declination (ICRS) - at epoch J2015.5 (dec) - 142- 161 F20.18 mas e_DEdeg Standard error of declination (dec_error) - 163- 183 F21.17 mas Plx Absolute stellar parallax of the source - at J2015.5 (parallax) - 185- 204 F20.18 mas e_Plx Standard error of parallax (parallax_error) - 206- 227 E22.19 mas/yr pmRA Proper motion in right ascension - (pmRAxcosDE) (pmra) - 229- 248 F20.18 mas/yr e_pmRA Standard error of proper motion in - right ascension (pmra_error) - 250- 271 E22.19 mas/yr pmDE Proper motion in declination (pmdec) - 273- 292 F20.18 mas/yr e_pmDE Standard error of proper motion - in declination (pmdec_error) - 294- 315 F22.19 mas epsi Measure of the residuals in the astrometric - solution for the source - (astrometric_excess_noise) - 317- 327 F11.9 mas amax Five-dimensional equivalent to the - semi-major axis of the Gaia position error - ellipse (astrometric_sigma5d_max) - 329- 350 F22.14 e-/s FG Gaia G-band mean flux (phot_g_mean_flux) - 352- 374 F23.17 e-/s e_FG Error on G-band mean flux - (phot_g_mean_flux_error) - 376- 385 F10.7 mag Gmag Gaia G-band mean magnitude (Vega scale) - (phot_g_mean_mag) - 387- 409 F23.15 e-/s FBP Integrated GBP mean flux (phot_bp_mean_flux) - 411- 435 F25.18 e-/s e_FBP Error on GBP-band mean flux - (phot_bp_mean_flux_error) - 437- 446 F10.7 mag BPmag Gaia GBP-band mean magnitude (Vega scale) - (phot_bp_mean_mag) - 448- 470 F23.15 e-/s FRP Integrated GRP mean flux (phot_rp_mean_flux) - 472- 497 F26.19 e-/s e_FRP Error on GRP-band mean flux - (phot_rp_mean_flux_error) - 499- 508 F10.7 mag RPmag Gaia GRP-band mean magnitude (Vega scale) - (phot_rp_mean_mag) - 510- 520 F11.9 --- E(BR/RP) GBP/GRP excess factor estimated from the - comparison of the sum of integrated GBP and - GRP fluxes with respect to the flux in the - G-band (phot_bp_rp_excess_factor) - 522- 542 E21.19 deg GLON Galactic longitude (l) - 544- 565 E22.19 deg GLAT Galactic latitude (b) - 567- 578 F12.4 --- Density The number of Gaia sources per square degree - around this object (density) - 580- 602 F23.19 mag AG Extinction in the Gaia G-band band derived - from E(B-V) values from Schlafly and - Finkbeiner (2011ApJ...737..103S) (AG) - 604- 622 A19 --- SDSS SDSS object name if available - (JHHMMSS,.ss+DDMMSS.s, J2000) (SDSS_name) - 624- 641 F18.15 mag umag ? SDSS u band magnitude (umag) - 643- 664 F22.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) - 666- 683 F18.15 mag gmag ? SDSS g band magnitude (gmag) - 685- 706 F22.19 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) - 708- 725 F18.15 mag rmag ? SDSS r band magnitude (rmag) - 727- 747 F21.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) - 749- 766 F18.15 mag imag ? SDSS i band magnitude (imag) - 768- 789 F22.19 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) - 791- 808 F18.15 mag zmag ? SDSS z band magnitude (zmag) - 810- 830 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) - 832- 852 E21.19 --- Pwd [0/1]? The probability of being a white - dwarf (Pwd) - 854 I1 --- f_Pwd [0/1] If 1 it indicates the PWD value - could be unreliable (Pwd_flag) - 856- 868 F13.6 K TeffH ? Effective temperature from fitting the - dereddened G,GBP, and GRP absolute fluxes - with pure-H model atmospheres (Teff_H) - 870- 882 F13.6 K e_TeffH ? Uncertainty on Teff_H (eTeff_H) - 884- 891 F8.6 --- loggH ? Surface gravity from fitting the - dereddened G,GBP,and GRP absolute fluxes - with pure-H model atmospheres (log_g_H) - 893- 900 E8.6 --- e_loggH ? Uncertainty on log_g_H (elog_g_H) - 902- 909 F8.6 Msun MassH ? Stellar mass resulting from the adopted - mass-radius relation (mass_H) - 911- 919 E9.6 Msun e_MassH ? Uncertainty on the mass (emass_H) - 921- 931 E11.6 --- chi2H ? chi2 value of the fit (pure-H) (chi2_H) - 933- 944 F12.6 K TeffHe ? Effective temperature from fitting the - dereddened G,GBP, and GRP absolute fluxes - with pure-He model atmospheres (Teff_He) - 946- 959 F14.6 K e_TeffHe ? Uncertainty on Teff_He (eTeff_He) - 961- 968 F8.6 --- loggHe ? Surface gravity from fitting the - dereddened G,GBP,and GRP absolute fluxes - with pure-He model atmospheres (log_g_He) - 970- 977 E8.6 --- e_loggHe ? Uncertainty on log_g_He (elog_g_He) - 979- 986 F8.6 Msun MassHe ? Stellar mass resulting from the adopted - mass-radius relation (mass_He) - 988- 996 E9.6 Msun e_MassHe ? Uncertainty on the mass (emass_He) - - 998-1008 E11.6 --- chisqHe ? chi2 value of the fit (pure-H) (chisq_He) --------------------------------------------------------------------------------- - -Byte-by-byte Description of file: gaiasdss.dat --------------------------------------------------------------------------------- - Bytes Format Units Label Explanations --------------------------------------------------------------------------------- - 1- 22 A22 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) - (white_dwarf_name) - 23 A1 --- n_WD [a] Multiplicity index on WD - 24- 42 I19 --- Source Unique Gaia source identifier (source_id) - 44- 62 A19 --- SDSS SDSS object name (JHHMMSS,.ss+DDMMSS.s, J2000) - (SDSS_name) - 64- 83 F20.16 deg RAdeg Right ascension (J2000) of the spectrum source - from SDSS DR14 (SDSS_ra) - 85- 98 E14.10 deg DEdeg Declination (J2000) of the spectrum source - from SDSS DR14 (SDSS_dec) - 100-117 F18.15 mag umag ? SDSS u band magnitude (umag) - 119-139 F21.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) - 141-158 F18.15 mag gmag ? SDSS g band magnitude (gmag) - 160-179 F20.18 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) - 181-198 F18.15 mag rmag ? SDSS r band magnitude (rmag) - 200-219 F20.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) - 221-238 F18.15 mag imag ? SDSS i band magnitude (imag) - 240-260 F21.18 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) - 262-279 F18.15 mag zmag ? SDSS z band magnitude (zmag) - 281-301 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) - 303-306 I4 --- Plate Identifier of the plate used in the - observation of the spectrum (Plate) - 308-312 I5 --- MJD Modified Julian date of the observation of - the spectrum (mjd) - 314-317 I4 --- Fiber Identifier of the fiber used in the - observation of the spectrum (fiberID) - 319-333 F15.11 --- S/N Signal-to-noise ratio of the spectrum - calculated in the range 4500-5500 A (S/N) - 335-344 A10 --- SpClass Classification of the object based on a visual - inspection of the SDSS spectrum - (spectral_class) (1) --------------------------------------------------------------------------------- -Note (1): spectral classes are as follows: - CV = Cataclysmic Variable star - DA = white dwarfs with H lines only - DAB/DAB = white dwarfs with H lines and neutral He lines - DABZ/DBAZ/DZAB/DZBA = white dwarfs with H lines, neutral He lines and - Ca H & K lines - DAO/DO = white dwarfs with H lines and He II lines - DAZ/DZA = white dwarfs with H lines and Ca H & K lines - DB = white dwarfs with neutral He lines only - DBZ/DZB = white dwarfs with neutral He lines and Ca H & K lines - DC = white dwarfs with feature-less spectra - DQ = white dwarfs with molecular C absorptions - hotDQ = white dwarfs with atomic C absorption - DQpec = Peculiar white dwarfs with molecular C absorptions - DZ = white dwarfs with Ca H & K lines - DAH = white dwarfs with Zeeman split H lines - DBH = white dwarfs with Zeeman split He lines - DZH = white dwarfs with Zeeman split Ca H & K lines - MWD = Magnetic white dwarfs with unidentified absorption lines - WDPec = Peculiar magnetic white dwarf - PG1159 = Pre-white dwarf star - DB+MS/DA+MS/DC+M = White Dwarf - Main Sequence binary - STAR = non white dwarf stellar object - QSO = Quasar - UNK = Spectra which we were unable to classify - Unreliable = Unreliable spectra --------------------------------------------------------------------------------- - -Acknowledgements: - Nicola Gentile Fusillo, n.gentile-fusillo(at)warwick.ac.uk - -History: - * 01-Feb-2019: on-line version - * 01-Apr-2019: corrected version (from author) - -================================================================================ -(End) Patricia Vannier [CDS] 31-Jan-2019 - -Modifications to datamodels: - -- All columns to lower case. -- All column except coordinates have been converted from float64 to float32. -- source -> source_id. -- RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec -- pm_de, e_pmde -> pm_dec, e_pmdec -- E(BR/RP) -> e_br_rp -- (S/N) -> snr - -*/ - - -CREATE TABLE catalogdb.wd_dr2 ( - wd TEXT, - dr2name TEXT, - source_id INTEGER PRIMARY KEY, - ra DOUBLE PRECISION, - e_ra DOUBLE PRECISION, - dec DOUBLE PRECISION, - e_dec DOUBLE PRECISION, - plx REAL, - e_plx REAL, - pmra DOUBLE PRECISION, - e_pmra DOUBLE PRECISION, - pmdec DOUBLE PRECISION, - e_pmdec DOUBLE PRECISION, - epsi REAL, - amax REAL, - fg REAL, - e_fg REAL, - gmag REAL, - fbp REAL, - e_fbp REAL, - bpmag REAL, - frp REAL, - e_frp REAL, - rpmag REAL, - e_br_rp REAL, - glon DOUBLE PRECISION, - glat DOUBLE PRECISION, - density REAL, - ag REAL, - sdss TEXT, - umag REAL, - e_umag REAL, - gmag REAL, - e_gmag REAL, - rmag REAL, - e_rmag REAL, - imag REAL, - e_imag REAL, - zmag REAL, - e_zmag REAL, - pwd REAL, - f_pwd INTEGER, - teffh REAL, - e_teffh REAL, - loggh REAL, - e_loggh REAL, - massh REAL, - e_massh REAL, - chi2h REAL, - teffhe REAL, - e_teffhe REAL, - logghe REAL, - e_logghe REAL, - masshe REAL, - e_masshe REAL, - chisqhe REAL -) WITHOUT OIDS; - - -CREATE TABLE catalogdb.wd_dr2_sdss ( - wd TEXT, - n_wd TEXT, - source_id BIGINT PRIMARY KEY, - sdss TEXT, - ra DOUBLE PRECISION, - dec DOUBLE PRECISION, - umag REAL, - e_umag REAL, - gmag REAL, - e_gmag REAL, - rmag REAL, - e_rmag REAL, - imag REAL, - e_imag REAL, - zmag REAL, - e_zmag REAL, - plate INTEGER, - mjd INTEGER, - fiber INTEGER, - snr REAL, - spclass TEXT -) WITHOUT OIDS; From 22cad31a63b73552e6a02a54fd9a3a75d6e90dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 21:16:12 -0700 Subject: [PATCH 068/197] Fix some replacements --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 7 +++--- .../sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 25 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql index 81cd9947..6b8bea83 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -254,6 +254,7 @@ Modifications to datamodels: - source -> source_id. - RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec - pm_de, e_pmde -> pm_dec, e_pmdec +- FG, e_FG Gmag -> fg_gaia, e_fg_gaia, g_gaia_mag - E(BR/RP) -> e_br_rp - (S/N) -> snr @@ -276,9 +277,9 @@ CREATE TABLE catalogdb.wd_dr2 ( e_pmdec DOUBLE PRECISION, epsi REAL, amax REAL, - fg REAL, - e_fg REAL, - gmag REAL, + fg_gaia REAL, + e_fg_gaia REAL, + g_gaia_mag REAL, fbp REAL, e_fbp REAL, bpmag REAL, diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py index c393d1b1..ad42200d 100755 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -17,15 +17,18 @@ def main(): assert database.connected - replacements = {'source': 'source_id', - 'radeg': 'ra', - 'dedeg': 'dec', + replacements = {'Source': 'source_id', + 'RAdeg': 'ra', + 'DEdeg': 'dec', 'e_RAdeg': 'e_ra', 'e_DEdeg': 'e_dec', - 'pm_de': 'pm_dec', - 'e_pmde': 'e_pmdec', - 'e(br/rp)': 'e_br_rp', - '(s/n)': 'snr'} + 'pmDE': 'pmdec', + 'e_pmDE': 'e_pmdec', + 'E(BR/RP)': 'e_br_rp', + '(S/N)': 'snr', + 'FG': 'fg_gaia', + 'e_FG': 'e_fg_gaia', + 'Gmag': 'g_gaia_mag'} path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/') @@ -33,10 +36,12 @@ def main(): ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'wd_dr2_sdss')): data = astropy.table.Table.read(path_ + file_) + for column_name in data.colnames: - new_column_name = column_name.lower() - if new_column_name in replacements: - new_column_name = replacements[new_column_name] + if column_name in replacements: + new_column_name = replacements[column_name] + else: + new_column_name = column_name.lower() data.rename_column(column_name, new_column_name) copy_data(data, database, table, schema='catalogdb') From 060f778c1f6f258011c8bf46c691066cfb8cebde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 22:04:55 -0700 Subject: [PATCH 069/197] Add source_id from dr2name --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 7 +++++-- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql index 6b8bea83..30d1adad 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -251,7 +251,8 @@ Modifications to datamodels: - All columns to lower case. - All column except coordinates have been converted from float64 to float32. -- source -> source_id. +- Added a source_id column that is constructed taking the integer of the +- dr2name designation (to be used for foreign keys). - RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec - pm_de, e_pmde -> pm_dec, e_pmdec - FG, e_FG Gmag -> fg_gaia, e_fg_gaia, g_gaia_mag @@ -264,7 +265,8 @@ Modifications to datamodels: CREATE TABLE catalogdb.wd_dr2 ( wd TEXT, dr2name TEXT, - source_id INTEGER PRIMARY KEY, + source_id BIGINT PRIMARY KEY, + source INTEGER, ra DOUBLE PRECISION, e_ra DOUBLE PRECISION, dec DOUBLE PRECISION, @@ -325,6 +327,7 @@ CREATE TABLE catalogdb.wd_dr2_sdss ( wd TEXT, n_wd TEXT, source_id BIGINT PRIMARY KEY, + source INTEGER, sdss TEXT, ra DOUBLE PRECISION, dec DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py index ad42200d..5c8157e7 100755 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -17,8 +17,7 @@ def main(): assert database.connected - replacements = {'Source': 'source_id', - 'RAdeg': 'ra', + replacements = {'RAdeg': 'ra', 'DEdeg': 'dec', 'e_RAdeg': 'e_ra', 'e_DEdeg': 'e_dec', @@ -44,6 +43,11 @@ def main(): new_column_name = column_name.lower() data.rename_column(column_name, new_column_name) + name_split = numpy.core.defchararray.split(data['dr2name']) + source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) + + data.add_column(astropy.table.Column(source_id, 'source_id'), data.index_column('source')) + copy_data(data, database, table, schema='catalogdb') From 78aebb1f6bdf6ce6eb3564813aca634c61e8e3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 22:24:49 -0700 Subject: [PATCH 070/197] Fix primary key in gaiasdss --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 2 +- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql index 30d1adad..b07e1faf 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -324,9 +324,9 @@ CREATE TABLE catalogdb.wd_dr2 ( CREATE TABLE catalogdb.wd_dr2_sdss ( + pk SERIAL PRIMARY KEY, wd TEXT, n_wd TEXT, - source_id BIGINT PRIMARY KEY, source INTEGER, sdss TEXT, ra DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py index 5c8157e7..b274e223 100755 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -43,10 +43,14 @@ def main(): new_column_name = column_name.lower() data.rename_column(column_name, new_column_name) - name_split = numpy.core.defchararray.split(data['dr2name']) - source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) - - data.add_column(astropy.table.Column(source_id, 'source_id'), data.index_column('source')) + if 'gaia2wd' in file_: + name_split = numpy.core.defchararray.split(data['dr2name']) + source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) + data.add_column(astropy.table.Column(source_id, 'source_id'), + data.index_column('source')) + else: + pk = numpy.arange(len(data)) + 1 + data.add_column(astropy.table.Column(pk, 'pk'), 0) copy_data(data, database, table, schema='catalogdb') From 5893e439c8df45dbfe53d17b9e3e5b35136366eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 23:48:29 -0700 Subject: [PATCH 071/197] Rename wd_dr2 to gaia_dr2_wd --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql | 4 ++-- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql index b07e1faf..bba188d2 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -262,7 +262,7 @@ Modifications to datamodels: */ -CREATE TABLE catalogdb.wd_dr2 ( +CREATE TABLE catalogdb.gaia_dr2_wd ( wd TEXT, dr2name TEXT, source_id BIGINT PRIMARY KEY, @@ -323,7 +323,7 @@ CREATE TABLE catalogdb.wd_dr2 ( ) WITHOUT OIDS; -CREATE TABLE catalogdb.wd_dr2_sdss ( +CREATE TABLE catalogdb.gaia_dr2_wd_sdss ( pk SERIAL PRIMARY KEY, wd TEXT, n_wd TEXT, diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py index b274e223..1b79bde7 100755 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -31,8 +31,8 @@ def main(): path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/') - for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz', 'wd_dr2'), - ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'wd_dr2_sdss')): + for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz', 'gaia_dr2_wd'), + ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'gaia_dr2_wd_sdss')): data = astropy.table.Table.read(path_ + file_) From 71fe8a899e33ad6347c4a92873513dae94642233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 23:48:51 -0700 Subject: [PATCH 072/197] Strip wd column on load --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py index 1b79bde7..eef12050 100755 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -43,6 +43,8 @@ def main(): new_column_name = column_name.lower() data.rename_column(column_name, new_column_name) + data['wd'][:] = numpy.core.defchararray.strip(data['wd'].data) + if 'gaia2wd' in file_: name_split = numpy.core.defchararray.split(data['dr2name']) source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) From cb6003105d0f0804eaa765e89e6a277a13f0b23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 23:49:08 -0700 Subject: [PATCH 073/197] q3c indices for gaia_dr2_wd --- schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql index e69de29b..0a7e4a9e 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -0,0 +1,8 @@ + +CREATE INDEX on catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_q3c_ang2ipix_idx on catalogdb.gaia_dr2_wd; +ANALYZE catalogdb.gaia_dr2_wd; + +CREATE INDEX on catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_sdss_q3c_ang2ipix_idx on catalogdb.gaia_dr2_wd_sdss; +ANALYZE catalogdb.gaia_dr2_wd_sdss; From 2e0b2d486e25a759b33ebfd67d4b9b355fb45823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 17 Mar 2020 23:49:18 -0700 Subject: [PATCH 074/197] ORM for gaia_dr2_wd --- python/sdssdb/peewee/sdss5db/catalogdb.py | 175 ++++++++++++++-------- 1 file changed, 109 insertions(+), 66 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index e1d36fbd..60a2098a 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -577,72 +577,6 @@ class Meta: schema = 'catalogdb' -class GaiaDR2WDCandidatesV1(SDSS5dbModel): - - ag = DoubleField(null=True) - astrometric_excess_noise = DoubleField(null=True) - astrometric_sigma5d_max = DoubleField(null=True) - b = DoubleField(index=True, null=True) - chi2 = DoubleField(null=True) - chisq_he = DoubleField(null=True) - dec = DoubleField(index=True, null=True) - dec_error = DoubleField(null=True) - density = DoubleField(null=True) - designation = TextField(null=True) - e_gmag = DoubleField(null=True) - e_imag = DoubleField(null=True) - e_rmag = DoubleField(null=True) - e_umag = DoubleField(null=True) - e_zmag = DoubleField(null=True) - elog_g = DoubleField(null=True) - elog_g_he = DoubleField(null=True) - emass = DoubleField(null=True) - emass_he = DoubleField(null=True) - eteff = DoubleField(null=True) - eteff_he = DoubleField(null=True) - gmag = DoubleField(null=True) - imag = DoubleField(null=True) - l = DoubleField(index=True, null=True) - log_g = DoubleField(null=True) - log_g_he = DoubleField(null=True) - mass = DoubleField(null=True) - mass_he = DoubleField(null=True) - parallax = DoubleField(null=True) - parallax_error = DoubleField(null=True) - phot_bp_mean_flux = DoubleField(null=True) - phot_bp_mean_flux_error = DoubleField(null=True) - phot_bp_mean_mag = DoubleField(null=True) - phot_bp_rp_excess_factor = DoubleField(null=True) - phot_g_mean_flux = DoubleField(index=True, null=True) - phot_g_mean_flux_error = DoubleField(null=True) - phot_g_mean_mag = DoubleField(index=True, null=True) - phot_rp_mean_flux = DoubleField(null=True) - phot_rp_mean_flux_error = DoubleField(null=True) - phot_rp_mean_mag = DoubleField(null=True) - pmdec = DoubleField(null=True) - pmdec_error = DoubleField(null=True) - pmra = DoubleField(null=True) - pmra_error = DoubleField(null=True) - pwd = FloatField(null=True) - pwd_correction = FloatField(null=True) - ra = DoubleField(index=True, null=True) - ra_error = DoubleField(null=True) - rmag = DoubleField(null=True) - sdss_name = TextField(null=True) - teff = DoubleField(null=True) - teff_he = DoubleField(null=True) - umag = DoubleField(null=True) - white_dwarf_name = TextField(null=True) - zmag = DoubleField(null=True) - - def gaia_source(self): - return GaiaDR2Source.select().get_id(self.source_id) - - class Meta: - table_name = 'gaia_dr2_wd_candidates_v1' - schema = 'catalogdb' - - class GaiaDR2SDSSDR9BestNeighbour(SDSS5dbModel): angular_distance = DoubleField(null=True) @@ -2391,4 +2325,113 @@ class Meta: schema = 'catalogdb' +class Gaia_DR2_WD_SDSS(SDSS5dbModel): + + pk = AutoField(primary_key=True) + wd = TextField(null=True) + n_wd = TextField(null=True) + source = IntegerField(null=True) + sdss = TextField(null=True) + ra = DoubleField(null=True) + dec = DoubleField(null=True) + umag = FloatField(null=True) + e_umag = FloatField(null=True) + gmag = FloatField(null=True) + e_gmag = FloatField(null=True) + rmag = FloatField(null=True) + e_rmag = FloatField(null=True) + imag = FloatField(null=True) + e_imag = FloatField(null=True) + zmag = FloatField(null=True) + e_zmag = FloatField(null=True) + plate = IntegerField(null=True) + mjd = IntegerField(null=True) + fiber = IntegerField(null=True) + snr = FloatField(null=True) + spclass = TextField(null=True) + + @property + def specobj(self): + """Returns the matching record in `.SDSSDR16SpecObj`.""" + + return SDSSDR16SpecObj.get(SDSSDR16SpecObj.plate == self.plate, + SDSSDR16SpecObj.mjd == self.mjd, + SDSSDR16SpecObj.fiberid == self.fiber) + + class Meta: + table_name = 'gaia_dr2_wd_sdss' + schema = 'catalogdb' + + +class Gaia_DR2_WD(SDSS5dbModel): + + wd = TextField(null=True) + dr2name = TextField(null=True) + source_id = BigAutoField(primary_key=True) + source = IntegerField(null=True) + ra = DoubleField(null=True) + e_ra = DoubleField(null=True) + dec = DoubleField(null=True) + e_dec = DoubleField(null=True) + plx = FloatField(null=True) + e_plx = FloatField(null=True) + pmra = DoubleField(null=True) + e_pmra = DoubleField(null=True) + pmdec = DoubleField(null=True) + e_pmdec = DoubleField(null=True) + epsi = FloatField(null=True) + amax = FloatField(null=True) + fg_gaia = FloatField(null=True) + e_fg_gaia = FloatField(null=True) + g_gaia_mag = FloatField(null=True) + fbp = FloatField(null=True) + e_fbp = FloatField(null=True) + bpmag = FloatField(null=True) + frp = FloatField(null=True) + e_frp = FloatField(null=True) + rpmag = FloatField(null=True) + e_br_rp = FloatField(null=True) + glon = DoubleField(null=True) + glat = DoubleField(null=True) + density = FloatField(null=True) + ag = FloatField(null=True) + sdss = TextField(null=True) + umag = FloatField(null=True) + e_umag = FloatField(null=True) + gmag = FloatField(null=True) + e_gmag = FloatField(null=True) + rmag = FloatField(null=True) + e_rmag = FloatField(null=True) + imag = FloatField(null=True) + e_imag = FloatField(null=True) + zmag = FloatField(null=True) + e_zmag = FloatField(null=True) + pwd = FloatField(null=True) + f_pwd = IntegerField(null=True) + teffh = FloatField(null=True) + e_teffh = FloatField(null=True) + loggh = FloatField(null=True) + e_loggh = FloatField(null=True) + massh = FloatField(null=True) + e_massh = FloatField(null=True) + chi2h = FloatField(null=True) + teffhe = FloatField(null=True) + e_teffhe = FloatField(null=True) + logghe = FloatField(null=True) + e_logghe = FloatField(null=True) + masshe = FloatField(null=True) + e_masshe = FloatField(null=True) + chisqhe = FloatField(null=True) + + @property + def sdss_spectra(self): + """Returns records from `.Gaia_DR2_WD_SDSS` with matching ``wd``.""" + + return Gaia_DR2_WD_SDSS.select().where(Gaia_DR2_WD_SDSS.wd == self.wd) + + class Meta: + table_name = 'gaia_dr2_wd' + schema = 'catalogdb' + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) From 39dc0a456e61d8bf0dd45dce2276cd78451ac84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 00:00:34 -0700 Subject: [PATCH 075/197] Add indices for specobj mjd, plate, and fiberid --- .../sdssSpecObj/dr16/sdss_dr16_specObj.sql | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index c29330de..dd70583e 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -406,9 +406,18 @@ CREATE TABLE catalogdb.sdss_dr16_specobj( alter table catalogdb.sdss_dr16_specobj add primary key(specObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_bestObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (bestObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fluxObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (fluxObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_targetObjID_index ON catalogdb.sdss_dr16_specobj using BTREE (targetObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_bestObjID_index + ON catalogdb.sdss_dr16_specobj using BTREE (bestObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fluxObjID_index + ON catalogdb.sdss_dr16_specobj using BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_targetObjID_index + ON catalogdb.sdss_dr16_specobj using BTREE (targetObjID); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_mjd_index + ON catalogdb.sdss_dr16_specobj using BTREE (mjd); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_plate_index + ON catalogdb.sdss_dr16_specobj using BTREE (plate); +CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fiberid_index + ON catalogdb.sdss_dr16_specobj using BTREE (fiberid); CREATE INDEX on catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); From e683e8b84570debb0d721cc56bf7cced3ccbf2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 18:47:33 -0700 Subject: [PATCH 076/197] Move LS files under dr8 and add ls_id pk --- python/sdssdb/peewee/sdss5db/catalogdb.py | 8 ++++---- .../legacy_survey/{ => dr8}/legacy_survey.sql | 0 .../{ => dr8}/legacy_survey_generate_csv.py | 0 .../legacy_survey/dr8/legacy_survey_index.sql | 12 ++++++++++++ .../legacy_survey/{ => dr8}/legacy_survey_load | 0 .../catalogdb/legacy_survey/legacy_survey_index.sql | 6 ------ 6 files changed, 16 insertions(+), 10 deletions(-) rename schema/sdss5db/catalogdb/legacy_survey/{ => dr8}/legacy_survey.sql (100%) rename schema/sdss5db/catalogdb/legacy_survey/{ => dr8}/legacy_survey_generate_csv.py (100%) create mode 100644 schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql rename schema/sdss5db/catalogdb/legacy_survey/{ => dr8}/legacy_survey_load (100%) delete mode 100644 schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 60a2098a..e3d6c96d 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -1947,9 +1947,10 @@ class Meta: class LegacySurveyDR8(SDSS5dbModel): - objid = BigIntegerField(primary_key=True) - release = IntegerField(null=True) - brickid = BigIntegerField(null=True) + lsid = BigIntegerField(primary_key=True) + objid = BigIntegerField() + release = IntegerField() + brickid = BigIntegerField() brickname = TextField(null=True) type = TextField(null=True) ra = DoubleField(null=True) @@ -2071,7 +2072,6 @@ class LegacySurveyDR8(SDSS5dbModel): class Meta: table_name = 'legacy_survey_dr8' schema = 'catalogdb' - primary_key = False class GaiaUnwiseAgn(SDSS5dbModel): diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql similarity index 100% rename from schema/sdss5db/catalogdb/legacy_survey/legacy_survey.sql rename to schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_generate_csv.py similarity index 100% rename from schema/sdss5db/catalogdb/legacy_survey/legacy_survey_generate_csv.py rename to schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_generate_csv.py diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql new file mode 100644 index 00000000..3962c7ff --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -0,0 +1,12 @@ +-- Add the lsid primary key. +ALTER TABLE catalogdb.legacy_survey_dr8 ADD COLUMN lsid BIGINT; +UPDATE catalogdb.legacy_survey_dr8 SET lsid = objid + brickid * 2^16 + release * 2^40; + +ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (lsid); + +CREATE INDEX CONCURRENTLY legacy_survey_dr8_lsid + ON catalogdb.legacy_survey_dr8 using BTREE (lsid); + +CREATE INDEX on catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey_dr8; +ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load similarity index 100% rename from schema/sdss5db/catalogdb/legacy_survey/legacy_survey_load rename to schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load diff --git a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql deleted file mode 100644 index 7180b319..00000000 --- a/schema/sdss5db/catalogdb/legacy_survey/legacy_survey_index.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Add the serial PK after copying the data. -ALTER TABLE catalogdb.legacy_survey ADD COLUMN pk BIGSERIAL PRIMARY KEY; - -CREATE INDEX on catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey_dr8; -ANALYZE catalogdb.legacy_survey_dr8; From b16b0cb62201b93e14f9afac1f5c69c0f57630f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 18:47:47 -0700 Subject: [PATCH 077/197] Deal with masked values when copying data --- python/sdssdb/utils/ingest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index 1c9e80f8..8afaa48d 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -258,6 +258,8 @@ def copy_data(data, connection, table_name, schema=None, chunk_size=10000, for col_value in row: if numpy.isscalar(col_value): row_data.append(str(col_value)) + elif numpy.ma.is_masked(col_value): + row_data.append('') else: if col_value.dtype.base.str[1] == 'S': col_value = col_value.astype('U') From 6be99e72880d4176d5fef708c0602bfdfa6fd216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 19:09:07 -0700 Subject: [PATCH 078/197] Tycho2 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 44 ++- schema/sdss5db/catalogdb/tycho2/tycho2.sql | 369 ++++++++++++++++++ .../sdss5db/catalogdb/tycho2/tycho2_index.sql | 4 + .../sdss5db/catalogdb/tycho2/tycho2_load.py | 59 +++ 4 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 schema/sdss5db/catalogdb/tycho2/tycho2.sql create mode 100644 schema/sdss5db/catalogdb/tycho2/tycho2_index.sql create mode 100755 schema/sdss5db/catalogdb/tycho2/tycho2_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index e3d6c96d..651ed0d6 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -499,7 +499,7 @@ class GaiaDR2Source(SDSS5dbModel): flame_flags = BigIntegerField(null=True) frame_rotator_object_type = IntegerField(null=True) g_rp = FloatField(null=True) - l = DoubleField(index=True, null=True) + l = DoubleField(index=True, null=True) # noqa lum_percentile_lower = FloatField(null=True) lum_percentile_upper = FloatField(null=True) lum_val = FloatField(null=True) @@ -2434,4 +2434,46 @@ class Meta: schema = 'catalogdb' +class Tycho2(SDSS5dbModel): + + id_tycho = BigAutoField(primary_key=True) + name = TextField(null=True) + pflag = CharField(null=True) + ramdeg = DoubleField(null=True) + demdeg = DoubleField(null=True) + pmra = DoubleField(null=True) + pmde = DoubleField(null=True) + e_ramdeg = DoubleField(null=True) + e_demdeg = DoubleField(null=True) + e_pmra = DoubleField(null=True) + e_pmde = DoubleField(null=True) + epram = DoubleField(null=True) + epdem = DoubleField(null=True) + num = IntegerField(null=True) + q_ramdeg = DoubleField(null=True) + q_demdeg = DoubleField(null=True) + q_pmra = DoubleField(null=True) + q_pmde = DoubleField(null=True) + btmag = DoubleField(null=True) + e_btmag = DoubleField(null=True) + vtmag = DoubleField(null=True) + e_vtmag = DoubleField(null=True) + prox = IntegerField(null=True) + tyc = CharField(null=True) + hip = IntegerField(null=True) + ccdm = TextField(null=True) + radeg = DoubleField(null=True) + dedeg = DoubleField(null=True) + epra_1990 = DoubleField(null=True) + epde_1990 = DoubleField(null=True) + e_radeg = DoubleField(null=True) + e_dedeg = DoubleField(null=True) + posflg = CharField(null=True) + corr = DoubleField(null=True) + + class Meta: + table_name = 'tycho2' + schema = 'catalogdb' + + _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql new file mode 100644 index 00000000..858459d6 --- /dev/null +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -0,0 +1,369 @@ +/* + +The Tycho-2 catalogue including the supplemental table 1. + +/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/tycho2/Tycho-2 + +Downloaded from ftp://cdsarc.u-strasbg.fr/pub/cats/I/259/ + +Note 1: we add an id_tycho column defined as in the Gaia cross-matching + (http://bit.ly/2TZD4Eg): (TYC1*1000000.0d0)+(TYC2*10.0d0)+(TYC3*1.0d0). + +Note 2: The columns HIP and CCDM are missing the delimiter in the files. This + is fixed in the loading process. + +I/259 The Tycho-2 Catalogue (Hog+ 2000) +================================================================================ +The Tycho-2 Catalogue of the 2.5 Million Brightest Stars + Hog E., Fabricius C., Makarov V.V., Urban S., Corbin T., + Wycoff G., Bastian U., Schwekendiek P., Wicenec A. + + =2000A&A...355L..27H +================================================================================ +ADC_Keywords: Positional data ; Proper motions ; Surveys ; + Photometry ; Stars, double and multiple +Mission_Name: Hipparcos +Keywords: astrometry - stars: fundamental parameters - catalogs + +Abstract: + The Tycho-2 Catalogue is an astrometric reference catalogue containing + positions and proper motions as well as two-colour photometric data + for the 2.5 million brightest stars in the sky. The Tycho-2 positions + and magnitudes are based on precisely the same observations as the + original Tycho Catalogue (hereafter Tycho-1; see Cat. )) + collected by the star mapper of the ESA Hipparcos satellite, but + Tycho-2 is much bigger and slightly more precise, owing to a more + advanced reduction technique. Components of double stars with + separations down to 0.8 arcsec are included. Proper motions precise to + about 2.5 mas/yr are given as derived from a comparison with the + Astrographic Catalogue and 143 other ground-based astrometric + catalogues, all reduced to the Hipparcos celestial coordinate system. + Tycho-2 supersedes in most applications Tycho-1, as well as the ACT + (Cat. ) and TRC (Cat. ) catalogues based on Tycho-1. + Supplement-1 lists stars from the Hipparcos and Tycho-1 Catalogues + which are not in Tycho-2. Supplement-2 lists 1146 Tycho-1 stars which + are probably either false or heavily disturbed. + + For more information, please consult the Tycho-2 home page: + http://www.astro.ku.dk/~erik/Tycho-2 + +Catalogue Characteristics: + + The principal characteristics of the Tycho-2 Catalogue are + summarized below. By means of proper motions the positions + are transferred to the year 2000.0, the epoch of the catalogue. + The median values of internal standard errors are given. + ------------------------------------------------------------ + Mean satellite observation epoch ~J1991.5 + Epoch of the Tycho-2 Catalogue J2000.0 + Reference system ICRS + coincidence with ICRS (1) +/-0.6 mas + deviation from inertial (1) +/-0.25 mas/yr + Number of entries 2,539,913 + Astrometric standard errors (2) + V_T_ < 9 mag 7 mas + all stars, positions 60 mas + all stars, proper motions 2.5 mas/yr + Photometric std. errors (3) on V_T_ + V_T_ < 9 mag 0.013 mag + all stars 0.10 mag + Star density + b= 0 deg 150 stars/sq.deg. + b= +/-30 deg 50 stars/sq.deg. + b= +/-90 deg 25 stars/sq.deg. + Completeness to 90 per cent V ~ 11.5 mag + Completeness to 99 per cent V ~ 11.0 mag + Number of Tycho observations ~300 10^6^ + ------------------------------------------------------------ + Note (1): about all 3 axes + Note (2): + ratio of external to internal standard errors is ~1.0 + for positions and for proper motions. Systematic errors + are less than 1 mas and 0.5 mas/yr + Note (3): + ratio of photometric external to internal standard errors + at V_T_ > 9 mag is below 1.5 + ------------------------------------------------------------ + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +tyc2.dat 206 2539913 *The Tycho-2 main catalogue +suppl_1.dat 122 17588 The Tycho-2 supplement-1 +suppl_2.dat 115 1146 The Tycho-2 supplement-2 +index.dat 42 9538 Index to Tycho-2 and supplement-1 +guide.ps 120 7837 Guide to Tycho-2 (postscript) +guide.pdf 1 428724 Guide to Tycho-2 (pdf) +-------------------------------------------------------------------------------- +Note on tyc2.dat: + This huge file is split into 20 smaller files named tyc2.dat.00, + tyc2.dat.01, ... tyc2.dat.18, each containing 127000 lines, + and tyc2.dat.19 which contains the last 126913 lines. +-------------------------------------------------------------------------------- + +See also: + I/239 : The Hipparcos and Tycho Catalogues (ESA 1997) + I/211 : CCDM (Components of Double and Multiple stars) (Dommanget+ 1994) + I/246 : The ACT Reference Catalog (Urban+ 1997) + I/250 : The Tycho Reference Catalogue (Hog+ 1998) + http://www.astro.ku.dk/~erik/Tycho-2 : Tycho-2 home page + +Nomenclature Notes: + The TYC identifier is constructed from the GSC region number (TYC1), + the running number within the region (TYC2) and a component identifier + (TYC3) which is normally 1. Some non-GSC running numbers were + constructed for the first Tycho Catalogue and for Tycho-2. + The recommended star designation contains a hyphen between the TYC + numbers, e.g. TYC 1-13-1. + +Field separator in the data files: + In the data files, the fields in each record are separated by + a vertical bar, |. In this connection the TYC identifier (TYC1, TYC2 + and TYC3) constitutes one field and the pair of a HIP number with a + CCDM identifier is also considered one field. + + +Byte-by-byte Description of file: tyc2.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 4 I4 --- TYC1 [1,9537]+= TYC1 from TYC or GSC (1) + 6- 10 I5 --- TYC2 [1,12121] TYC2 from TYC or GSC (1) + 12 I1 --- TYC3 [1,3] TYC3 from TYC (1) + 14 A1 --- pflag [ PX] mean position flag (2) + 16- 27 F12.8 deg RAmdeg []? Mean Right Asc, ICRS, epoch=J2000 (3) + 29- 40 F12.8 deg DEmdeg []? Mean Decl, ICRS, at epoch=J2000 (3) + 42- 48 F7.1 mas/yr pmRA ? Proper motion in RA*cos(dec) (12) + 50- 56 F7.1 mas/yr pmDE ? Proper motion in Dec (12) + 58- 60 I3 mas e_RAmdeg [3,183]? s.e. RA*cos(dec),at mean epoch (5) + 62- 64 I3 mas e_DEmdeg [1,184]? s.e. of Dec at mean epoch (5) + 66- 69 F4.1 mas/yr e_pmRA [0.2,11.5]? s.e. prop mot in RA*cos(dec)(5) + 71- 74 F4.1 mas/yr e_pmDE [0.2,10.3]? s.e. of proper motion in Dec(5) + 76- 82 F7.2 yr EpRAm [1915.95,1992.53]? mean epoch of RA (4) + 84- 90 F7.2 yr EpDEm [1911.94,1992.01]? mean epoch of Dec (4) + 92- 93 I2 --- Num [2,36]? Number of positions used + 95- 97 F3.1 --- q_RAmdeg [0.0,9.9]? Goodness of fit for mean RA (6) + 99-101 F3.1 --- q_DEmdeg [0.0,9.9]? Goodness of fit for mean Dec (6) + 103-105 F3.1 --- q_pmRA [0.0,9.9]? Goodness of fit for pmRA (6) + 107-109 F3.1 --- q_pmDE [0.0,9.9]? Goodness of fit for pmDE (6) + 111-116 F6.3 mag BTmag [2.183,16.581]? Tycho-2 BT magnitude (7) + 118-122 F5.3 mag e_BTmag [0.014,1.977]? s.e. of BT (7) + 124-129 F6.3 mag VTmag [1.905,15.193]? Tycho-2 VT magnitude (7) + 131-135 F5.3 mag e_VTmag [0.009,1.468]? s.e. of VT (7) + 137-139 I3 0.1arcsec prox [3,999] proximity indicator (8) + 141 A1 --- TYC [T] Tycho-1 star (9) + 143-148 I6 --- HIP [1,120404]? Hipparcos number + 149-151 A3 --- CCDM CCDM component identifier for HIP stars(10) + 153-164 F12.8 deg RAdeg Observed Tycho-2 Right Ascension, ICRS + 166-177 F12.8 deg DEdeg Observed Tycho-2 Declination, ICRS + 179-182 F4.2 yr EpRA-1990 [0.81,2.13] epoch-1990 of RAdeg + 184-187 F4.2 yr EpDE-1990 [0.72,2.36] epoch-1990 of DEdeg + 189-193 F5.1 mas e_RAdeg s.e.RA*cos(dec), of observed Tycho-2 RA (5) + 195-199 F5.1 mas e_DEdeg s.e. of observed Tycho-2 Dec (5) + 201 A1 --- posflg [ DP] type of Tycho-2 solution (11) + 203-206 F4.1 --- corr [-1,1] correlation (RAdeg,DEdeg) +-------------------------------------------------------------------------------- + +Note (1): The TYC identifier is constructed from the GSC region number + (TYC1), the running number within the region (TYC2) and a component + identifier (TYC3) which is normally 1. Some non-GSC running numbers + were constructed for the first Tycho Catalogue and for Tycho-2. + The recommended star designation contains a hyphen between the + TYC numbers, e.g. TYC 1-13-1. + +Note (2): + ' ' = normal mean position and proper motion. + 'P' = the mean position, proper motion, etc., refer to the + photocentre of two Tycho-2 entries, where the BT magnitudes + were used in weighting the positions. + 'X' = no mean position, no proper motion. + +Note (3): + The mean position is a weighted mean for the catalogues contributing + to the proper motion determination. This mean has then been brought to + epoch 2000.0 by the computed proper motion. See Note(2) above for + details. Tycho-2 is one of the several catalogues used to determine + the mean position and proper motion. The observed Tycho-2 position is + given in the fields RAdeg and DEdeg. + +Note (4): + The mean epochs are given in Julian years. + +Note (5): + The errors are based on error models. + +Note (6): + This goodness of fit is the ratio of the scatter-based and the + model-based error. It is only defined when Num > 2. Values + exceeding 9.9 are truncated to 9.9. + +Note (7): + Blank when no magnitude is available. Either BTmag or VTmag is + always given. Approximate Johnson photometry may be obtained as: + V = VT -0.090*(BT-VT) + B-V = 0.850*(BT-VT) + Consult Sect 1.3 of Vol 1 of "The Hipparcos and Tycho Catalogues", + ESA SP-1200, 1997, for details. + +Note (8): + Distance in units of 100 mas to the nearest entry in the Tycho-2 + main catalogue or supplement. The distance is computed for the + epoch 1991.25. A value of 999 (i.e. 99.9 arcsec) is given if the + distance exceeds 99.9 arcsec. + +Note (9): + ' ' = no Tycho-1 star was found within 0.8 arcsec (quality 1-8) + or 2.4 arcsec (quality 9). + 'T' = this is a Tycho-1 star. The Tycho-1 identifier is given in the + beginning of the record. For Tycho-1 stars, resolved in + Tycho-2 as a close pair, both components are flagged as + a Tycho-1 star and the Tycho-1 TYC3 is assigned to the + brightest (VT) component. + The HIP-only stars given in Tycho-1 are not flagged as Tycho-1 stars. + +Note (10): + The CCDM component identifiers for double or multiple Hipparcos stars + contributing to this Tycho-2 entry. For photocentre solutions, all + components within 0.8 arcsec contribute. For double star solutions any + unresolved component within 0.8 arcsec contributes. For single star + solutions, the predicted signal from close stars were normally + subtracted in the analysis of the photon counts and such stars + therefore do not contribute to the solution. The components are given + in lexical order. + +Note (11): + ' ' = normal treatment, close stars were subtracted when possible. + 'D' = double star treatment. Two stars were found. The companion is + normally included as a separate Tycho-2 entry, but may have + been rejected. + 'P' = photocentre treatment, close stars were not subtracted. This + special treatment was applied to known or suspected doubles + which were not successfully (or reliably) resolved in the + Tycho-2 double star processing. + +Note (12): Some Hipparcos stars (having a positive number in the HIP column) + have no proper motions; these are virtually all in multiple systems. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: suppl_1.dat, suppl_2.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 4 I4 --- TYC1 [2,9529]+= TYC1 from TYC (1) + 6- 10 I5 --- TYC2 [1,12112] TYC2 from TYC (1) + 12 I1 --- TYC3 [1,4] TYC3 from TYC (1) + 14 A1 --- flag [HT] data from Hipparcos or Tycho-1 (2) + 16- 27 F12.8 deg RAdeg Right Asc, ICRS, at epoch=J1991.25 + 29- 40 F12.8 deg DEdeg Decl, ICRS, at epoch=J1991.25 + 42- 48 F7.1 mas/yr pmRA []? Proper motion in RA*cos(dec) + 50- 56 F7.1 mas/yr pmDE []? Proper motion in Dec + 58- 62 F5.1 mas e_RAdeg s.e. RA*cos(dec) + 64- 68 F5.1 mas e_DEdeg s.e. of Dec + 70- 74 F5.1 mas/yr e_pmRA []? s.e. prop mot in RA * cos(dec) + 76- 80 F5.1 mas/yr e_pmDE []? s.e. of proper motion in Dec + 82 A1 --- mflag [ BVH] Note about Tycho magnitudes (3) + 84- 89 F6.3 mag BTmag []? Tycho-1 BT magnitude (4) + 91- 95 F5.3 mag e_BTmag []? s.e. of BT (4) + 97-102 F6.3 mag VTmag []? Tycho-1 VT or Hp magnitude (4) + 104-108 F5.3 mag e_VTmag []? s.e. of VT (4) + 110-112 I3 0.1arcsec prox [1,999] proximity indicator (5) + 114 A1 --- TYC [ T] Tycho-1 star + 116-121 I6 --- HIP [1,120404]? Hipparcos number + 122 A1 --- CCDM CCDM component identifier for HIP stars +-------------------------------------------------------------------------------- + +Note (1): The TYC identifier is constructed from the GSC region number (TYC1), + the running number within the region (TYC2) and a component identifier + (TYC3) which is normally 1. The numbers are copied from Tycho-1. + (see the "Nomenclature Notes" section above) + +Note (2): + 'H' = data are from Hipparcos and include proper motion. + 'T' = data are from Tycho-1. No proper motion is given. + +Note (3): + ' ' = both BT and VT given. + 'B' = only BT given. + 'V' = only VT given. + 'H' = Hp is given instead of VT. BT is blank. + +Note (4): + Blank when no magnitude is available. + For Hipparcos stars with no VT, Hp is given instead of VT, and BT is blank. + +Note (5): + Distance in units of 100 mas to nearest Tycho-2 main or supplement + entry. (Computed for the epoch 1991.25). A value of 999 (i.e. 99.9 + arcsec) is given if the distance exceeds 99.9 arcsec. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: index.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 7 I7 --- rec_t2 + Tycho-2 rec. of 1st star in region (1) + 9- 14 I6 --- rec_s1 += Suppl-1 rec. of 1st star in region (1) + 16- 21 F6.2 deg RAmin [-0.01,] smallest RA in region (2) + 23- 28 F6.2 deg RAmax [,360.00] largest RA in region (2) + 30- 35 F6.2 deg DEmin smallest Dec in this region (2) + 37- 42 F6.2 deg DEmax largest Dec in this region (2) +-------------------------------------------------------------------------------- + +Note (1): The catalogue is sorted according to the GSC region numbers. + The line i of the index file gives the record number in Tycho-2 of + the first star in GSC region i. Line i+1 gives the record number +1 + of the last star in GSC region i. For Supplement-1, some regions are + empty and line i and line i+1 give the same record number. + +Note (2): a safe rounding was applied. Minimum values are always + rounded down and maximum values up. +-------------------------------------------------------------------------------- + +References: + Hog E., Fabricius C., Makarov V.V., Bastian U., Schwekendiek P., Wicenec A., + Urban S., Corbin T., Wycoff G., "Construction and verification of the + Tycho-2 Catalogue." =2000A&A...357..367H +================================================================================ +(End) C. Fabricius [Niels Bohr Institute] 19-Jan-2000 + +*/ + + +CREATE TABLE catalogdb.tycho2 ( + id_tycho BIGINT PRIMARY KEY, + name TEXT, + pflag CHARACTER(1), + ramdeg DOUBLE PRECISION, + demdeg DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmde DOUBLE PRECISION, + e_ramdeg DOUBLE PRECISION, + e_demdeg DOUBLE PRECISION, + e_pmra DOUBLE PRECISION, + e_pmde DOUBLE PRECISION, + epram DOUBLE PRECISION, + epdem DOUBLE PRECISION, + num INTEGER, + q_ramdeg DOUBLE PRECISION, + q_demdeg DOUBLE PRECISION, + q_pmra DOUBLE PRECISION, + q_pmde DOUBLE PRECISION, + btmag DOUBLE PRECISION, + e_btmag DOUBLE PRECISION, + vtmag DOUBLE PRECISION, + e_vtmag DOUBLE PRECISION, + prox INTEGER, + tyc CHARACTER(1), + hip INTEGER, + ccdm TEXT, + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + epra_1990 DOUBLE PRECISION, + epde_1990 DOUBLE PRECISION, + e_radeg DOUBLE PRECISION, + e_dedeg DOUBLE PRECISION, + posflg CHARACTER(1), + corr DOUBLE PRECISION +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql new file mode 100644 index 00000000..2a308ea8 --- /dev/null +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql @@ -0,0 +1,4 @@ + +CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); +CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.tycho2; +ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py new file mode 100755 index 00000000..42c19c14 --- /dev/null +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-18 +# @Filename: tycho2_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import pandas +import io + +from sdssdb.peewee.sdss5db import database + + +def main(): + + assert database.connected + + path_ = '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/' + path_ = '/Users/albireo/Downloads/Tycho-2/' + + for file_ in sorted(glob.glob(path_ + 'tyc2.dat.*.gz')): + + print(file_) + + data = pandas.read_csv(file_, delimiter='|', low_memory=False, + header=None, skipinitialspace=True) + + # Convert columns num to integer with NA. + data[12] = data[12].astype('Int32') + + # Split column into hip and ccdm + col = data[23].fillna('') + hip = col.map(lambda x: x[0:-3].strip()) + ccdm = col.map(lambda x: x[-3:].strip()) + + data[23] = ccdm + data.insert(23, 'hip', pandas.to_numeric(hip).astype('Int32')) + + # Add the id_tycho column + tyc1, tyc2, tyc3 = data[0].str.split(' ', expand=True).apply(pandas.to_numeric).T.values + id_tycho = (tyc1 * 1000000) + (tyc2 * 10) + (tyc3 * 1) + data.insert(0, 'id_tycho', id_tycho) + + # Write as CSV to data stream. + csv = io.StringIO() + data.to_csv(csv, header=False, index=False) + csv.seek(0) + + cursor = database.cursor() + cursor.copy_expert('COPY catalogdb.tycho2 FROM STDIN ' + 'WITH DELIMITER \',\' CSV;', csv) + database.commit() + + +if __name__ == '__main__': + + main() From fdc9893ec4625c5d33b97603a962125c882232b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 19:10:19 -0700 Subject: [PATCH 079/197] Add pandas and ipdb dependencies --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7e4ba5b2..8e470aac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,13 +55,14 @@ all = progressbar2>=3.46.1 pydot>=1.4.1 astropy>=4.0.0 + pandas>=1.0.0 dev = pytest>=3.0.7 pytest-cov>=2.4.0 pytest-sugar>=0.8.0 - pandas>=1.0.0 sdsstools[dev]>=0.1.5 ipython>=7.13.0 + ipdb>=0.13.2 docs = Sphinx>=1.8.0 sphinx_bootstrap_theme>=0.4.12 From 8e993aa94f0a31481dab104f85c0c59e13f0a918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 19:11:14 -0700 Subject: [PATCH 080/197] Change path for Tycho at utah --- schema/sdss5db/catalogdb/tycho2/tycho2_load.py | 1 - 1 file changed, 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py index 42c19c14..85b3997f 100755 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py @@ -18,7 +18,6 @@ def main(): assert database.connected path_ = '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/' - path_ = '/Users/albireo/Downloads/Tycho-2/' for file_ in sorted(glob.glob(path_ + 'tyc2.dat.*.gz')): From 018f3c3db4546b3ab7ec2a5f505e599a48aa4ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 19:12:20 -0700 Subject: [PATCH 081/197] Move sqlalchemy to install_requires --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8e470aac..1e52d146 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ install_requires = psycopg2-binary>=2.7.7 six>=1.12.0 peewee>=3.9.6 + sqlalchemy>=1.3.6 sdsstools>=0.1.5 [options.packages.find] @@ -51,7 +52,6 @@ sdssdb = [options.extras_require] all = - sqlalchemy>=1.3.6 progressbar2>=3.46.1 pydot>=1.4.1 astropy>=4.0.0 From 7ec3adcf87cc6d07a8c5518f0d4912f5df57eeb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 18 Mar 2020 19:28:38 -0700 Subject: [PATCH 082/197] noqa --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 651ed0d6..aa7ce46c 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -869,7 +869,7 @@ class SDSSDR13Photoobj(SDSS5dbModel): fieldid = BigIntegerField(null=True) flags = BigIntegerField(null=True) htmid = BigIntegerField(null=True) - l = DoubleField(index=True, null=True) + l = DoubleField(index=True, null=True) # noqa lnldev_g = FloatField(null=True) lnldev_i = FloatField(null=True) lnldev_r = FloatField(null=True) From 7500b7d2a250b14982513470dc3e533c15c4ada9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 12:46:41 -0700 Subject: [PATCH 083/197] Initial implementation of ReflectMeta --- docs/sphinx/api.rst | 5 +- docs/sphinx/conf.py | 2 +- python/sdssdb/connection.py | 17 ++- python/sdssdb/peewee/__init__.py | 122 +++++++++++++++++++++- python/sdssdb/peewee/sdss5db/catalogdb.py | 36 +------ setup.cfg | 2 +- 6 files changed, 140 insertions(+), 44 deletions(-) diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index 111d9a58..15fe9198 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -7,13 +7,16 @@ Database connections .. automodule:: sdssdb.connection :members: DatabaseConnection, PeeweeDatabaseConnection, SQLADatabaseConnection - :undoc-members: :show-inheritance: Peewee ------ +.. autoclass:: sdssdb.peewee.ReflectMeta + :members: + :show-inheritance: + .. autoclass:: sdssdb.peewee.BaseModel :members: :show-inheritance: diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 41f49fb9..9b993cac 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -49,7 +49,7 @@ # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.autosummary', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.mathjax', - 'sphinx.ext.intersphinx', 'sdsstools.releases'] + 'sphinx.ext.intersphinx', 'sdsstools.releases', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 871051f8..37521b9f 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -307,7 +307,17 @@ def become_user(self): if _peewee: class PeeweeDatabaseConnection(DatabaseConnection, PostgresqlDatabase): - """Peewee database connection implementation.""" + """Peewee database connection implementation. + + Attributes + ---------- + models : set + Models bound to this database. Only models that are bound using + `~sdssdb.peewee.BaseModel` are handled. + + """ + + models = set([]) def __init__(self, *args, **kwargs): @@ -340,6 +350,11 @@ def _conn(self, dbname, silent_on_fail=False, **params): PostgresqlDatabase.init(self, None) self.connected = False + if self.is_connection_usable(): + for model in self.models: + if hasattr(model, 'reflect'): + model.reflect() + return self.connected diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 6ff14b2b..bf2e5c77 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -1,18 +1,130 @@ +# isort:skip_file # flake8: noqa +import re + from sdssdb import _peewee if _peewee is False: raise ImportError('Peewee must be installed to use this module.') -import re - -from peewee import Model, fn +from peewee import Model, ModelBase, fn from playhouse.hybrid import hybrid_method +from playhouse.reflection import generate_models + + +class ReflectMeta(ModelBase): + """A metaclass that supports model reflection on demand. + + This metaclass expands Peewee's ``ModelBase`` to provide a hook for + declaring/expanding fields and indexes using the + :ref:`introspection system `. The feature is enabled + by a new attribute in :class:`peewee:Metadata` called ``use_reflection`` + (which is set to `False` by default). When set to `True` the metaclass + extends the model using the fields and indexes discovered + using reflection. It is possible to mix explicitely defined fields + with discovered ones; the latter never override the former. + + Normally `.ReflectMeta` is implemented by creating a base model that + is then used to defined the table models :: + + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + + class Meta: + primary_key = False + use_reflection = False + database = database + + class Tycho2(ReflectBaseModel): + + class Meta: + use_reflection = True + schema = 'catalogdb' + table_name = 'tycho2' + + Note that ``use_reflection`` is inheritable so if set to `True` in the base + class that will affect to all subclasses, except if it's overridden there. + It's also a good idea to set ``primary_key=False`` to prevent Peewee from + creating an ``id`` column automatically. + + If the database connection changes it's possible to call `.reflect` to + rediscover the reflected fields for the new connection. This will remove + all reflected fields (but not those explicitely added) and add the newly + discovered ones. + + If the database class is `.PeeweeDatabaseConnection`, the database will + call `.reflect` for each model bound to the database each time it connects. + This ensures that if the connection changes the reflected fields are + updated. Note that this will not work if using Peewee's + :class:`peewee:PostgresqlDatabase`. + + """ + + def __new__(cls, name, bases, attrs): + + Model = super(ReflectMeta, cls).__new__(cls, name, bases, attrs) + Model._meta.use_reflection = getattr(Model._meta, 'use_reflection', False) + + database = Model._meta.database + if database and hasattr(database, 'models'): + database.models.add(Model) + + if Model._meta.use_reflection and database and database.connected: + cls.reflect(Model) + + return Model + + def reflect(self): + """Adds fields and indexes to the model using reflection.""" + + # Don't do anything if this model doesn't want reflection. + if not hasattr(self._meta, 'use_reflection') or not self._meta.use_reflection: + return + + # First clear all previously reflected fields and indexes. + for field in self._meta.fields: + if hasattr(field, 'reflected') and field.reflected: + self._meta.remove_field(field) + + for index in self._meta.indexes: + if hasattr(index, 'reflected') and index.reflected: + self._meta.indexes.remove(index) + + + database = self._meta.database + if not database.is_connection_usable(): + raise peewee.DatabaseError('database not connected.') + + table_name = self._meta.table_name + schema = self._meta.schema + + ReflectedModel = generate_models(database, schema=schema, + table_names=table_name)[table_name] + + for field_name, field in ReflectedModel._meta.fields.items(): + + if field_name in self._meta.fields: + continue + + field.reflected = True + + if field.primary_key: + self._meta.set_primary_key(field_name, field) + else: + self._meta.add_field(field_name, field) + + for index in ReflectedModel._meta.indexes: + + # TODO: this probably can never work. Find a better way of determining + # if the index has been added manually. + if index in self._meta.indexes: + continue + + self._meta.indexes.append(index) -class BaseModel(Model): +class BaseModel(Model, metaclass=ReflectMeta): """A custom peewee `.Model` with enhanced representation and methods. By default it always prints ``pk``, ``name``, and ``label``, if found. @@ -60,7 +172,7 @@ def cone_search(self, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): return fn.q3c_ellipse_query(ra_attr, dec_attr, ra, dec, a, ratio, pa) @cone_search.expression - def cone_search(cls, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): + def cone_search(cls, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): # noqa """Returns a query with the rows inside a region on the sky. Defines a sky ellipse and returns the targets within. By default it diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index aa7ce46c..b9074646 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -2436,44 +2436,10 @@ class Meta: class Tycho2(SDSS5dbModel): - id_tycho = BigAutoField(primary_key=True) - name = TextField(null=True) - pflag = CharField(null=True) - ramdeg = DoubleField(null=True) - demdeg = DoubleField(null=True) - pmra = DoubleField(null=True) - pmde = DoubleField(null=True) - e_ramdeg = DoubleField(null=True) - e_demdeg = DoubleField(null=True) - e_pmra = DoubleField(null=True) - e_pmde = DoubleField(null=True) - epram = DoubleField(null=True) - epdem = DoubleField(null=True) - num = IntegerField(null=True) - q_ramdeg = DoubleField(null=True) - q_demdeg = DoubleField(null=True) - q_pmra = DoubleField(null=True) - q_pmde = DoubleField(null=True) - btmag = DoubleField(null=True) - e_btmag = DoubleField(null=True) - vtmag = DoubleField(null=True) - e_vtmag = DoubleField(null=True) - prox = IntegerField(null=True) - tyc = CharField(null=True) - hip = IntegerField(null=True) - ccdm = TextField(null=True) - radeg = DoubleField(null=True) - dedeg = DoubleField(null=True) - epra_1990 = DoubleField(null=True) - epde_1990 = DoubleField(null=True) - e_radeg = DoubleField(null=True) - e_dedeg = DoubleField(null=True) - posflg = CharField(null=True) - corr = DoubleField(null=True) - class Meta: table_name = 'tycho2' schema = 'catalogdb' + use_reflection = True _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) diff --git a/setup.cfg b/setup.cfg index 1e52d146..f2d691bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,7 @@ docs = sphinx_bootstrap_theme>=0.4.12 [isort] -line_length = 99 +line_length = 79 lines_after_imports = 2 use_parentheses = true balanced_wrapping = true From 430bff46bfd679267e4d1ae778545c1e77c2b757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 12:49:16 -0700 Subject: [PATCH 084/197] Add Meta to BaseModel --- python/sdssdb/peewee/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index bf2e5c77..3f23009a 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -136,6 +136,10 @@ class BaseModel(Model, metaclass=ReflectMeta): #: A list of fields (as strings) to be included in the ``__repr__`` print_fields = [] + class Meta: + primary_key = False + use_reflection = False + def __str__(self): """A custom str for the model repr.""" From bdcd629503119bfffd6a4e8407f447e68c8f6d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 13:30:09 -0700 Subject: [PATCH 085/197] Issue warning if reflection fails --- python/sdssdb/peewee/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 3f23009a..aefcd826 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -2,6 +2,7 @@ # flake8: noqa import re +import warnings from sdssdb import _peewee @@ -13,6 +14,8 @@ from playhouse.hybrid import hybrid_method from playhouse.reflection import generate_models +from sdssdb.core.exceptions import SdssdbUserWarning + class ReflectMeta(ModelBase): """A metaclass that supports model reflection on demand. @@ -99,8 +102,13 @@ def reflect(self): table_name = self._meta.table_name schema = self._meta.schema - ReflectedModel = generate_models(database, schema=schema, - table_names=table_name)[table_name] + try: + ReflectedModel = generate_models(database, schema=schema, + table_names=table_name)[table_name] + except KeyError: + warnings.warn('reflection failed for {}'.format(table_name), + SdssdbUserWarning) + return for field_name, field in ReflectedModel._meta.fields.items(): From 80d7c4c4d8951e2f202be98780044d740e8dcb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 14:34:11 -0700 Subject: [PATCH 086/197] Log failed reflections as debug --- python/sdssdb/peewee/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index aefcd826..48304117 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -14,7 +14,7 @@ from playhouse.hybrid import hybrid_method from playhouse.reflection import generate_models -from sdssdb.core.exceptions import SdssdbUserWarning +from sdssdb import log class ReflectMeta(ModelBase): @@ -104,10 +104,9 @@ def reflect(self): try: ReflectedModel = generate_models(database, schema=schema, - table_names=table_name)[table_name] + table_names=table_name)[table_name] except KeyError: - warnings.warn('reflection failed for {}'.format(table_name), - SdssdbUserWarning) + log.debug('reflection failed for {}'.format(table_name)) return for field_name, field in ReflectedModel._meta.fields.items(): From 6bfd86d35561bba71219d56763833d0afb15ef78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 14:34:31 -0700 Subject: [PATCH 087/197] Deal with FKs during reflection --- python/sdssdb/peewee/__init__.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 48304117..df83371a 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -9,7 +9,7 @@ if _peewee is False: raise ImportError('Peewee must be installed to use this module.') - +import peewee from peewee import Model, ModelBase, fn from playhouse.hybrid import hybrid_method from playhouse.reflection import generate_models @@ -66,18 +66,35 @@ class that will affect to all subclasses, except if it's overridden there. def __new__(cls, name, bases, attrs): + # Remove explicit foreign keys before we do reflection. + fks = [(name, attr) for name, attr in attrs.items() + if isinstance(attr, peewee.ForeignKeyField)] + + for name, __ in fks: + attrs.pop(name) + Model = super(ReflectMeta, cls).__new__(cls, name, bases, attrs) Model._meta.use_reflection = getattr(Model._meta, 'use_reflection', False) + Model._meta._fks = fks database = Model._meta.database - if database and hasattr(database, 'models'): - database.models.add(Model) + if database and hasattr(database, 'models') and Model not in database.models: + database.models.append(Model) - if Model._meta.use_reflection and database and database.connected: - cls.reflect(Model) + if Model._meta.use_reflection: + if database and database.connected: + cls.reflect(Model) + else: + cls._attach_fks(Model) return Model + def _attach_fks(self): + """Attach the explicit foreign keys.""" + + for name, fk in self._meta._fks: + self._meta.add_field(name, fk) + def reflect(self): """Adds fields and indexes to the model using reflection.""" @@ -121,6 +138,9 @@ def reflect(self): else: self._meta.add_field(field_name, field) + # Attach FKs now + self._attach_fks() + for index in ReflectedModel._meta.indexes: # TODO: this probably can never work. Find a better way of determining From ff4cfd2a5476679d3fd1b5410a18cd8404df0776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 14:34:46 -0700 Subject: [PATCH 088/197] Store models as list to preserve order --- python/sdssdb/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 37521b9f..30219a77 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -311,13 +311,13 @@ class PeeweeDatabaseConnection(DatabaseConnection, PostgresqlDatabase): Attributes ---------- - models : set + models : list Models bound to this database. Only models that are bound using `~sdssdb.peewee.BaseModel` are handled. """ - models = set([]) + models = list() def __init__(self, *args, **kwargs): From 79df80a69eaec99c08fa8c4b5173899adb0f7778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 14:35:02 -0700 Subject: [PATCH 089/197] Use reflection in catalogdb --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2080 +-------------------- 1 file changed, 39 insertions(+), 2041 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index b9074646..80c00eb4 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -10,554 +10,50 @@ # @Last modified time: 2019-09-23 00:54:42 -from peewee import (AutoField, BigAutoField, BigIntegerField, BooleanField, CharField, DateField, - DecimalField, DeferredThroughModel, DoubleField, FloatField, ForeignKeyField, - IntegerField, ManyToManyField, SmallIntegerField, TextField) -from playhouse.postgres_ext import ArrayField +from peewee import (BigAutoField, BigIntegerField, CharField, + DeferredThroughModel, DoubleField, FloatField, + ForeignKeyField, IntegerField, ManyToManyField, TextField) from . import SDSS5dbModel, database # noqa -class AllWise(SDSS5dbModel): +_GaiaDR2TmassBestNeighbourDeferred = DeferredThroughModel() - best_use_cntr = BigIntegerField(null=True) - cc_flags = CharField(null=True) - cntr = BigIntegerField(null=True) - coadd_id = CharField(null=True) - dec = DecimalField(index=True, null=True) - dec_pm = DecimalField(null=True) - designation = CharField(null=True) - det_bit = IntegerField(null=True) - elat = DecimalField(null=True) - elon = DecimalField(null=True) - ext_flg = IntegerField(null=True) - glat = DecimalField(index=True, null=True) - glon = DecimalField(index=True, null=True) - h_m_2mass = DecimalField(null=True) - h_msig_2mass = DecimalField(null=True) - htm20 = BigIntegerField(null=True) - j_m_2mass = DecimalField(null=True) - j_msig_2mass = DecimalField(null=True) - k_m_2mass = DecimalField(null=True) - k_msig_2mass = DecimalField(null=True) - moon_lev = CharField(null=True) - n_2mass = IntegerField(null=True) - na = IntegerField(null=True) - nb = IntegerField(null=True) - ngrp = IntegerField(null=True) - pa_2mass = DecimalField(null=True) - ph_qual = CharField(null=True) - pmcode = CharField(null=True) - pmdec = IntegerField(null=True) - pmra = IntegerField(null=True) - q12 = IntegerField(null=True) - q23 = IntegerField(null=True) - q34 = IntegerField(null=True) - r_2mass = DecimalField(null=True) - ra = DecimalField(index=True, null=True) - ra_pm = DecimalField(null=True) - rchi2 = FloatField(null=True) - rchi2_pm = FloatField(null=True) - rel = CharField(null=True) - rho12 = IntegerField(null=True) - rho23 = IntegerField(null=True) - rho34 = IntegerField(null=True) - satnum = CharField(null=True) - sigdec = DecimalField(null=True) - sigdec_pm = DecimalField(null=True) - sigpmdec = IntegerField(null=True) - sigpmra = IntegerField(null=True) - sigra = DecimalField(null=True) - sigra_pm = DecimalField(null=True) - sigradec = DecimalField(null=True) - sigradec_pm = DecimalField(null=True) - source_id = CharField(null=True) - spt_ind = IntegerField(null=True) - src = IntegerField(null=True) - tmass_key = IntegerField(null=True) - var_flg = CharField(null=True) - w1ba = DecimalField(null=True) - w1cc_map = IntegerField(null=True) - w1cc_map_str = CharField(null=True) - w1conf = DecimalField(null=True) - w1cov = DecimalField(null=True) - w1flg = IntegerField(null=True) - w1flg_1 = IntegerField(null=True) - w1flg_2 = IntegerField(null=True) - w1flg_3 = IntegerField(null=True) - w1flg_4 = IntegerField(null=True) - w1flg_5 = IntegerField(null=True) - w1flg_6 = IntegerField(null=True) - w1flg_7 = IntegerField(null=True) - w1flg_8 = IntegerField(null=True) - w1flux = FloatField(null=True) - w1gerr = DecimalField(null=True) - w1gflg = IntegerField(null=True) - w1gmag = DecimalField(null=True) - w1k = DecimalField(null=True) - w1m = IntegerField(null=True) - w1mag = DecimalField(null=True) - w1mag_1 = DecimalField(null=True) - w1mag_2 = DecimalField(null=True) - w1mag_3 = DecimalField(null=True) - w1mag_4 = DecimalField(null=True) - w1mag_5 = DecimalField(null=True) - w1mag_6 = DecimalField(null=True) - w1mag_7 = DecimalField(null=True) - w1mag_8 = DecimalField(null=True) - w1magp = DecimalField(null=True) - w1mcor = DecimalField(null=True) - w1mjdmax = DecimalField(null=True) - w1mjdmean = DecimalField(null=True) - w1mjdmin = DecimalField(null=True) - w1mlq = DecimalField(null=True) - w1mpro = DecimalField(null=True) - w1ndf = IntegerField(null=True) - w1nm = IntegerField(null=True) - w1pa = DecimalField(null=True) - w1rchi2 = FloatField(null=True) - w1rchi2_pm = FloatField(null=True) - w1rsemi = DecimalField(null=True) - w1sat = DecimalField(null=True) - w1sigflux = FloatField(null=True) - w1sigm = DecimalField(null=True) - w1sigm_1 = DecimalField(null=True) - w1sigm_2 = DecimalField(null=True) - w1sigm_3 = DecimalField(null=True) - w1sigm_4 = DecimalField(null=True) - w1sigm_5 = DecimalField(null=True) - w1sigm_6 = DecimalField(null=True) - w1sigm_7 = DecimalField(null=True) - w1sigm_8 = DecimalField(null=True) - w1sigmpro = DecimalField(null=True) - w1sigp1 = DecimalField(null=True) - w1sigp2 = DecimalField(null=True) - w1sigsk = DecimalField(null=True) - w1sky = DecimalField(null=True) - w1snr = DecimalField(null=True) - w2ba = DecimalField(null=True) - w2cc_map = IntegerField(null=True) - w2cc_map_str = CharField(null=True) - w2conf = DecimalField(null=True) - w2cov = DecimalField(null=True) - w2flg = IntegerField(null=True) - w2flg_1 = IntegerField(null=True) - w2flg_2 = IntegerField(null=True) - w2flg_3 = IntegerField(null=True) - w2flg_4 = IntegerField(null=True) - w2flg_5 = IntegerField(null=True) - w2flg_6 = IntegerField(null=True) - w2flg_7 = IntegerField(null=True) - w2flg_8 = IntegerField(null=True) - w2flux = FloatField(null=True) - w2gerr = DecimalField(null=True) - w2gflg = IntegerField(null=True) - w2gmag = DecimalField(null=True) - w2k = DecimalField(null=True) - w2m = IntegerField(null=True) - w2mag = DecimalField(null=True) - w2mag_1 = DecimalField(null=True) - w2mag_2 = DecimalField(null=True) - w2mag_3 = DecimalField(null=True) - w2mag_4 = DecimalField(null=True) - w2mag_5 = DecimalField(null=True) - w2mag_6 = DecimalField(null=True) - w2mag_7 = DecimalField(null=True) - w2mag_8 = DecimalField(null=True) - w2magp = DecimalField(null=True) - w2mcor = DecimalField(null=True) - w2mjdmax = DecimalField(null=True) - w2mjdmean = DecimalField(null=True) - w2mjdmin = DecimalField(null=True) - w2mlq = DecimalField(null=True) - w2mpro = DecimalField(null=True) - w2ndf = IntegerField(null=True) - w2nm = IntegerField(null=True) - w2pa = DecimalField(null=True) - w2rchi2 = FloatField(null=True) - w2rchi2_pm = FloatField(null=True) - w2rsemi = DecimalField(null=True) - w2sat = DecimalField(null=True) - w2sigflux = FloatField(null=True) - w2sigm = DecimalField(null=True) - w2sigm_1 = DecimalField(null=True) - w2sigm_2 = DecimalField(null=True) - w2sigm_3 = DecimalField(null=True) - w2sigm_4 = DecimalField(null=True) - w2sigm_5 = DecimalField(null=True) - w2sigm_6 = DecimalField(null=True) - w2sigm_7 = DecimalField(null=True) - w2sigm_8 = DecimalField(null=True) - w2sigmpro = DecimalField(null=True) - w2sigp1 = DecimalField(null=True) - w2sigp2 = DecimalField(null=True) - w2sigsk = DecimalField(null=True) - w2sky = DecimalField(null=True) - w2snr = DecimalField(null=True) - w3ba = DecimalField(null=True) - w3cc_map = IntegerField(null=True) - w3cc_map_str = CharField(null=True) - w3conf = DecimalField(null=True) - w3cov = DecimalField(null=True) - w3flg = IntegerField(null=True) - w3flg_1 = IntegerField(null=True) - w3flg_2 = IntegerField(null=True) - w3flg_3 = IntegerField(null=True) - w3flg_4 = IntegerField(null=True) - w3flg_5 = IntegerField(null=True) - w3flg_6 = IntegerField(null=True) - w3flg_7 = IntegerField(null=True) - w3flg_8 = IntegerField(null=True) - w3flux = FloatField(null=True) - w3gerr = DecimalField(null=True) - w3gflg = IntegerField(null=True) - w3gmag = DecimalField(null=True) - w3k = DecimalField(null=True) - w3m = IntegerField(null=True) - w3mag = DecimalField(null=True) - w3mag_1 = DecimalField(null=True) - w3mag_2 = DecimalField(null=True) - w3mag_3 = DecimalField(null=True) - w3mag_4 = DecimalField(null=True) - w3mag_5 = DecimalField(null=True) - w3mag_6 = DecimalField(null=True) - w3mag_7 = DecimalField(null=True) - w3mag_8 = DecimalField(null=True) - w3magp = DecimalField(null=True) - w3mcor = DecimalField(null=True) - w3mjdmax = DecimalField(null=True) - w3mjdmean = DecimalField(null=True) - w3mjdmin = DecimalField(null=True) - w3mlq = DecimalField(null=True) - w3mpro = DecimalField(null=True) - w3ndf = IntegerField(null=True) - w3nm = IntegerField(null=True) - w3pa = DecimalField(null=True) - w3rchi2 = FloatField(null=True) - w3rchi2_pm = FloatField(null=True) - w3rsemi = DecimalField(null=True) - w3sat = DecimalField(null=True) - w3sigflux = FloatField(null=True) - w3sigm = DecimalField(null=True) - w3sigm_1 = DecimalField(null=True) - w3sigm_2 = DecimalField(null=True) - w3sigm_3 = DecimalField(null=True) - w3sigm_4 = DecimalField(null=True) - w3sigm_5 = DecimalField(null=True) - w3sigm_6 = DecimalField(null=True) - w3sigm_7 = DecimalField(null=True) - w3sigm_8 = DecimalField(null=True) - w3sigmpro = DecimalField(null=True) - w3sigp1 = DecimalField(null=True) - w3sigp2 = DecimalField(null=True) - w3sigsk = DecimalField(null=True) - w3sky = DecimalField(null=True) - w3snr = DecimalField(null=True) - w4ba = DecimalField(null=True) - w4cc_map = IntegerField(null=True) - w4cc_map_str = CharField(null=True) - w4conf = DecimalField(null=True) - w4cov = DecimalField(null=True) - w4flg = IntegerField(null=True) - w4flg_1 = IntegerField(null=True) - w4flg_2 = IntegerField(null=True) - w4flg_3 = IntegerField(null=True) - w4flg_4 = IntegerField(null=True) - w4flg_5 = IntegerField(null=True) - w4flg_6 = IntegerField(null=True) - w4flg_7 = IntegerField(null=True) - w4flg_8 = IntegerField(null=True) - w4flux = FloatField(null=True) - w4gerr = DecimalField(null=True) - w4gflg = IntegerField(null=True) - w4gmag = DecimalField(null=True) - w4k = DecimalField(null=True) - w4m = IntegerField(null=True) - w4mag = DecimalField(null=True) - w4mag_1 = DecimalField(null=True) - w4mag_2 = DecimalField(null=True) - w4mag_3 = DecimalField(null=True) - w4mag_4 = DecimalField(null=True) - w4mag_5 = DecimalField(null=True) - w4mag_6 = DecimalField(null=True) - w4mag_7 = DecimalField(null=True) - w4mag_8 = DecimalField(null=True) - w4magp = DecimalField(null=True) - w4mcor = DecimalField(null=True) - w4mjdmax = DecimalField(null=True) - w4mjdmean = DecimalField(null=True) - w4mjdmin = DecimalField(null=True) - w4mlq = DecimalField(null=True) - w4mpro = DecimalField(null=True) - w4ndf = IntegerField(null=True) - w4nm = IntegerField(null=True) - w4pa = DecimalField(null=True) - w4rchi2 = FloatField(null=True) - w4rchi2_pm = FloatField(null=True) - w4rsemi = DecimalField(null=True) - w4sat = DecimalField(null=True) - w4sigflux = FloatField(null=True) - w4sigm = DecimalField(null=True) - w4sigm_1 = DecimalField(null=True) - w4sigm_2 = DecimalField(null=True) - w4sigm_3 = DecimalField(null=True) - w4sigm_4 = DecimalField(null=True) - w4sigm_5 = DecimalField(null=True) - w4sigm_6 = DecimalField(null=True) - w4sigm_7 = DecimalField(null=True) - w4sigm_8 = DecimalField(null=True) - w4sigmpro = DecimalField(null=True) - w4sigp1 = DecimalField(null=True) - w4sigp2 = DecimalField(null=True) - w4sigsk = DecimalField(null=True) - w4sky = DecimalField(null=True) - w4snr = DecimalField(null=True) - wx = DecimalField(null=True) - wy = DecimalField(null=True) - x = DecimalField(null=True) - xscprox = DecimalField(null=True) - y = DecimalField(null=True) - z = DecimalField(null=True) + +class AllWise(SDSS5dbModel): class Meta: table_name = 'allwise' schema = 'catalogdb' - primary_key = False + use_reflection = True class ErositaAGNMock(SDSS5dbModel): - allwise_designation = TextField(null=True) - ero_dec = DoubleField(index=True, null=True) - ero_det_like_0 = FloatField(null=True) - ero_detuid = TextField(null=True) - ero_flux = FloatField(index=True, null=True) - ero_flux_err = FloatField(null=True) - ero_ra = DoubleField(index=True, null=True) - ero_radec_err = FloatField(null=True) - ero_souuid = TextField(null=True) - gaia_dr2_source_id = BigIntegerField(null=True) - nsc_dr1_id = TextField(null=True) - pk = BigAutoField() - sdss_dr14_objid = BigIntegerField(null=True) - t_exp = DoubleField(null=True) - target_dec = DoubleField(index=True, null=True) - target_mag_r = FloatField(null=True) - target_magerr_r = FloatField(null=True) - target_match_prob = FloatField(null=True) - target_origin = IntegerField(null=True) - target_pmdec = FloatField(null=True) - target_pmra = FloatField(null=True) - target_priority = IntegerField(index=True, null=True) - target_ra = DoubleField(index=True, null=True) - class Meta: table_name = 'erosita_agn_mock' schema = 'catalogdb' + use_reflection = True class ErositaClustersMock(SDSS5dbModel): - allwise_designation = TextField(null=True) - ero_dec = DoubleField(index=True, null=True) - ero_det_like_0 = FloatField(null=True) - ero_detuid = TextField(null=True) - ero_ext = FloatField(null=True) - ero_ext_err = FloatField(null=True) - ero_ext_like = FloatField(null=True) - ero_flux = FloatField(index=True, null=True) - ero_flux_err = FloatField(null=True) - ero_ra = DoubleField(index=True, null=True) - ero_radec_err = FloatField(null=True) - ero_souuid = TextField(null=True) - gaia_dr2_source_id = BigIntegerField(null=True) - nsc_dr1_id = TextField(null=True) - pk = BigAutoField() - sdss_dr14_objid = BigIntegerField(null=True) - t_exp = DoubleField(null=True) - target_dec = DoubleField(index=True, null=True) - target_mag_r = FloatField(null=True) - target_magerr_r = FloatField(null=True) - target_match_prob = FloatField(null=True) - target_origin = IntegerField(null=True) - target_pmdec = FloatField(null=True) - target_pmra = FloatField(null=True) - target_priority = IntegerField(index=True, null=True) - target_ra = DoubleField(index=True, null=True) - class Meta: table_name = 'erosita_clusters_mock' schema = 'catalogdb' + use_reflection = True class TwoMassPsc(SDSS5dbModel): - a = CharField(null=True) - b_m_opt = FloatField(null=True) - bl_flg = CharField(null=True) - cc_flg = CharField(index=True, null=True) - coadd = IntegerField(null=True) - coadd_key = IntegerField(null=True) - date = DateField(null=True) - decl = DoubleField(index=True, null=True) - designation = CharField(index=True, null=True) - dist_edge_ew = IntegerField(null=True) - dist_edge_flg = CharField(null=True) - dist_edge_ns = IntegerField(null=True) - dist_opt = FloatField(null=True) - dup_src = IntegerField(null=True) - err_ang = IntegerField(null=True) - err_maj = FloatField(null=True) - err_min = FloatField(null=True) - ext_key = IntegerField(null=True) - gal_contam = IntegerField(index=True, null=True) - glat = FloatField(null=True) - glon = FloatField(null=True) - h_cmsig = FloatField(null=True) - h_m = FloatField(index=True, null=True) - h_m_stdap = FloatField(null=True) - h_msig_stdap = FloatField(null=True) - h_msigcom = FloatField(null=True) - h_psfchi = FloatField(null=True) - h_snr = FloatField(null=True) - hemis = CharField(null=True) - j_cmsig = FloatField(null=True) - j_m = FloatField(index=True, null=True) - j_m_stdap = FloatField(null=True) - j_msig_stdap = FloatField(null=True) - j_msigcom = FloatField(null=True) - j_psfchi = FloatField(null=True) - j_snr = FloatField(null=True) - jdate = DoubleField(null=True) - k_cmsig = FloatField(null=True) - k_m = FloatField(index=True, null=True) - k_m_stdap = FloatField(null=True) - k_msig_stdap = FloatField(null=True) - k_msigcom = FloatField(null=True) - k_psfchi = FloatField(null=True) - k_snr = FloatField(null=True) - mp_flg = IntegerField(null=True) - ndet = CharField(null=True) - nopt_mchs = IntegerField(null=True) - ph_qual = CharField(index=True, null=True) - phi_opt = IntegerField(null=True) - prox = FloatField(null=True) - pts_key = AutoField() - pxcntr = IntegerField(null=True) - pxpa = IntegerField(null=True) - ra = DoubleField(index=True, null=True) - rd_flg = CharField(index=True, null=True) - scan = IntegerField(null=True) - scan_key = IntegerField(null=True) - use_src = IntegerField(null=True) - vr_m_opt = FloatField(null=True) - x_scan = FloatField(null=True) - class Meta: table_name = 'twomass_psc' schema = 'catalogdb' - - -_GaiaDR2TmassBestNeighbourDeferred = DeferredThroughModel() + use_reflection = True class GaiaDR2Source(SDSS5dbModel): - a_g_percentile_lower = FloatField(null=True) - a_g_percentile_upper = FloatField(null=True) - a_g_val = FloatField(null=True) - astrometric_chi2_al = FloatField(index=True, null=True) - astrometric_excess_noise = DoubleField(null=True) - astrometric_excess_noise_sig = DoubleField(null=True) - astrometric_gof_al = FloatField(null=True) - astrometric_matched_observations = IntegerField(null=True) - astrometric_n_bad_obs_al = IntegerField(null=True) - astrometric_n_good_obs_al = IntegerField(index=True, null=True) - astrometric_n_obs_ac = IntegerField(null=True) - astrometric_n_obs_al = IntegerField(null=True) - astrometric_params_solved = IntegerField(null=True) - astrometric_primary_flag = BooleanField(null=True) - astrometric_pseudo_colour = DoubleField(null=True) - astrometric_pseudo_colour_error = DoubleField(null=True) - astrometric_sigma5d_max = FloatField(null=True) - astrometric_weight_al = FloatField(null=True) - b = DoubleField(index=True, null=True) - bp_g = FloatField(null=True) - bp_rp = FloatField(null=True) - dec = DoubleField(index=True, null=True) - dec_error = DoubleField(null=True) - dec_parallax_corr = FloatField(null=True) - dec_pmdec_corr = FloatField(null=True) - dec_pmra_corr = FloatField(null=True) - designation = TextField(null=True) - duplicated_source = BooleanField(null=True) - e_bp_min_rp_percentile_lower = FloatField(null=True) - e_bp_min_rp_percentile_upper = FloatField(null=True) - e_bp_min_rp_val = FloatField(null=True) - ecl_lat = DoubleField(index=True, null=True) - ecl_lon = DoubleField(index=True, null=True) - flame_flags = BigIntegerField(null=True) - frame_rotator_object_type = IntegerField(null=True) - g_rp = FloatField(null=True) - l = DoubleField(index=True, null=True) # noqa - lum_percentile_lower = FloatField(null=True) - lum_percentile_upper = FloatField(null=True) - lum_val = FloatField(null=True) - matched_observations = IntegerField(null=True) - mean_varpi_factor_al = FloatField(null=True) - parallax = DoubleField(index=True, null=True) - parallax_error = DoubleField(null=True) - parallax_over_error = FloatField(index=True, null=True) - parallax_pmdec_corr = FloatField(null=True) - parallax_pmra_corr = FloatField(null=True) - phot_bp_mean_flux = DoubleField(null=True) - phot_bp_mean_flux_error = DoubleField(null=True) - phot_bp_mean_flux_over_error = FloatField(index=True, null=True) - phot_bp_mean_mag = FloatField(index=True, null=True) - phot_bp_n_obs = IntegerField(null=True) - phot_bp_rp_excess_factor = FloatField(index=True, null=True) - phot_g_mean_flux = DoubleField(index=True, null=True) - phot_g_mean_flux_error = DoubleField(null=True) - phot_g_mean_flux_over_error = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - phot_g_n_obs = IntegerField(null=True) - phot_proc_mode = IntegerField(null=True) - phot_rp_mean_flux = DoubleField(null=True) - phot_rp_mean_flux_error = DoubleField(null=True) - phot_rp_mean_flux_over_error = FloatField(index=True, null=True) - phot_rp_mean_mag = FloatField(index=True, null=True) - phot_rp_n_obs = IntegerField(null=True) - phot_variable_flag = TextField(null=True) - pmdec = DoubleField(null=True) - pmdec_error = DoubleField(null=True) - pmra = DoubleField(null=True) - pmra_error = DoubleField(null=True) - pmra_pmdec_corr = FloatField(null=True) - priam_flags = BigIntegerField(null=True) - ra = DoubleField(index=True, null=True) - ra_dec_corr = FloatField(null=True) - ra_error = DoubleField(null=True) - ra_parallax_corr = FloatField(null=True) - ra_pmdec_corr = FloatField(null=True) - ra_pmra_corr = FloatField(null=True) - radial_velocity = DoubleField(null=True) - radial_velocity_error = DoubleField(null=True) - radius_percentile_lower = FloatField(null=True) - radius_percentile_upper = FloatField(null=True) - radius_val = FloatField(null=True) - random_index = BigIntegerField(null=True) - ref_epoch = DoubleField(null=True) - rv_nb_transits = IntegerField(null=True) - rv_template_fe_h = FloatField(null=True) - rv_template_logg = FloatField(null=True) - rv_template_teff = FloatField(null=True) - solution_id = BigIntegerField(index=True, null=True) - source_id = BigAutoField() - teff_percentile_lower = FloatField(null=True) - teff_percentile_upper = FloatField(null=True) - teff_val = FloatField(null=True) - visibility_periods_used = IntegerField(index=True, null=True) - tmass_best_sources = ManyToManyField(TwoMassPsc, through_model=_GaiaDR2TmassBestNeighbourDeferred, backref='gaia_best_sources') @@ -565,1202 +61,150 @@ class GaiaDR2Source(SDSS5dbModel): class Meta: table_name = 'gaia_dr2_source' schema = 'catalogdb' + use_reflection = True class GaiaDR2Clean(SDSS5dbModel): - source_id = BigAutoField(column_name='source_id') source = ForeignKeyField(GaiaDR2Source, column_name='source_id', backref='gaia_clean') class Meta: table_name = 'gaia_dr2_clean' schema = 'catalogdb' + use_reflection = True class GaiaDR2SDSSDR9BestNeighbour(SDSS5dbModel): - angular_distance = DoubleField(null=True) - best_neighbour_multiplicity = IntegerField(null=True) - gaia_astrometric_params = IntegerField(null=True) - number_of_mates = IntegerField(null=True) - number_of_neighbours = IntegerField(null=True) - original_ext_source_id = TextField(null=True) - sdssdr9_oid = BigIntegerField(index=True, null=True) - source_id = BigAutoField() - class Meta: table_name = 'gaiadr2_sdssdr9_best_neighbour' schema = 'catalogdb' + use_reflection = True class GalacticGenesis(SDSS5dbModel): - g_dec = DoubleField(null=True) - g_ra = DoubleField(null=True) - gaiaid = BigAutoField() - h_m = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - t_dec = DoubleField(null=True) - t_ra = DoubleField(null=True) - twomass_desig = CharField(null=True, unique=True) - class Meta: table_name = 'galactic_genesis' schema = 'catalogdb' + use_reflection = True class GalacticGenesisBig(SDSS5dbModel): - g_dec = DoubleField(null=True) - g_ra = DoubleField(null=True) - gaiaid = BigAutoField() - h_m = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - t_dec = DoubleField(null=True) - t_ra = DoubleField(null=True) - twomass_desig = CharField(null=True) - class Meta: table_name = 'galactic_genesis_big' schema = 'catalogdb' + use_reflection = True class GUVCat(SDSS5dbModel): - avaspdec = DoubleField(null=True) - avaspra = DoubleField(null=True) - band = IntegerField(null=True) - chkobj_type = IntegerField(null=True) - corv = TextField(null=True) - dec = DoubleField(index=True, null=True) - difffuv = FloatField(null=True) - difffuvdist = FloatField(null=True) - diffnuv = FloatField(null=True) - diffnuvdist = FloatField(null=True) - e_bv = FloatField(null=True) - fexptime = FloatField(null=True) - fov_radius = FloatField(null=True) - fuv_a_world = FloatField(null=True) - fuv_artifact = IntegerField(null=True) - fuv_b_world = FloatField(null=True) - fuv_class_star = FloatField(null=True) - fuv_ellipticity = FloatField(null=True) - fuv_errtheta_j2000 = FloatField(null=True) - fuv_flags = IntegerField(null=True) - fuv_flux = FloatField(null=True) - fuv_fluxerr = FloatField(null=True) - fuv_fwhm_image = FloatField(null=True) - fuv_fwhm_world = FloatField(null=True) - fuv_kron_radius = FloatField(null=True) - fuv_mag = FloatField(null=True) - fuv_mag_aper_4 = FloatField(null=True) - fuv_mag_aper_6 = FloatField(null=True) - fuv_mag_auto = FloatField(null=True) - fuv_magerr = FloatField(null=True) - fuv_magerr_aper_4 = FloatField(null=True) - fuv_magerr_aper_6 = FloatField(null=True) - fuv_magerr_auto = FloatField(null=True) - fuv_ncat_flux_radius_3 = FloatField(null=True) - fuv_ncat_fwhm_image = FloatField(null=True) - fuv_poserr = FloatField(null=True) - fuv_pperr = FloatField(null=True) - fuv_theta_j2000 = FloatField(null=True) - fuv_weight = FloatField(null=True) - fuv_x_image = FloatField(null=True) - fuv_y_image = FloatField(null=True) - glat = DoubleField(null=True) - glon = DoubleField(null=True) - grank = IntegerField(null=True) - grankdist = IntegerField(null=True) - groupgid = TextField(null=True) - groupgiddist = TextField(null=True) - groupgidtot = TextField(null=True) - ib_poserr = FloatField(null=True) - img = IntegerField(null=True) - inlargeobj = TextField(null=True) - istherespectrum = IntegerField(null=True) - largeobjsize = FloatField(null=True) - mpstype = TextField(null=True) - nexptime = FloatField(null=True) - ngrank = IntegerField(null=True) - ngrankdist = BigIntegerField(null=True) - nuv_a_world = FloatField(null=True) - nuv_artifact = IntegerField(null=True) - nuv_b_world = FloatField(null=True) - nuv_class_star = FloatField(null=True) - nuv_ellipticity = FloatField(null=True) - nuv_errtheta_j2000 = FloatField(null=True) - nuv_flags = IntegerField(null=True) - nuv_flux = FloatField(null=True) - nuv_fluxerr = FloatField(null=True) - nuv_fwhm_image = FloatField(null=True) - nuv_fwhm_world = FloatField(null=True) - nuv_kron_radius = FloatField(null=True) - nuv_mag = FloatField(null=True) - nuv_mag_aper_4 = FloatField(null=True) - nuv_mag_aper_6 = FloatField(null=True) - nuv_mag_auto = FloatField(null=True) - nuv_magerr = FloatField(null=True) - nuv_magerr_aper_4 = FloatField(null=True) - nuv_magerr_aper_6 = FloatField(null=True) - nuv_magerr_auto = FloatField(null=True) - nuv_poserr = FloatField(null=True) - nuv_pperr = FloatField(null=True) - nuv_theta_j2000 = FloatField(null=True) - nuv_weight = FloatField(null=True) - nuv_x_image = FloatField(null=True) - nuv_y_image = FloatField(null=True) - objid = BigAutoField() - photoextractid = BigIntegerField(null=True) - primgid = BigIntegerField(null=True) - primgiddist = BigIntegerField(null=True) - prob = FloatField(null=True) - ra = DoubleField(index=True, null=True) - sep = FloatField(null=True) - sepas = FloatField(null=True) - sepasdist = FloatField(null=True) - subvisit = IntegerField(null=True) - tilenum = IntegerField(null=True) - type = IntegerField(null=True) - class Meta: table_name = 'guvcat' schema = 'catalogdb' + use_reflection = True class KeplerInput10(SDSS5dbModel): - kic_altid = BigIntegerField(null=True) - kic_altsource = BigIntegerField(null=True) - kic_aq = IntegerField(null=True) - kic_av = FloatField(null=True) - kic_blend = IntegerField(null=True) - kic_catkey = BigIntegerField(null=True) - kic_cq = TextField(null=True) - kic_d51mag = DoubleField(null=True) - kic_dec = DoubleField(index=True, null=True) - kic_degree_ra = DoubleField(null=True) - kic_ebminusv = FloatField(null=True) - kic_feh = FloatField(null=True) - kic_fov_flag = IntegerField(null=True) - kic_galaxy = IntegerField(null=True) - kic_gkcolor = DoubleField(null=True) - kic_glat = DoubleField(index=True, null=True) - kic_glon = DoubleField(index=True, null=True) - kic_gmag = DoubleField(index=True, null=True) - kic_grcolor = DoubleField(null=True) - kic_gredmag = DoubleField(null=True) - kic_hmag = DoubleField(index=True, null=True) - kic_imag = DoubleField(index=True, null=True) - kic_jkcolor = DoubleField(null=True) - kic_jmag = DoubleField(index=True, null=True) - kic_kepler_id = BigAutoField() - kic_kepmag = DoubleField(index=True, null=True) - kic_kmag = DoubleField(index=True, null=True) - kic_logg = FloatField(null=True) - kic_parallax = FloatField(null=True) - kic_pmdec = DoubleField(null=True) - kic_pmra = DoubleField(null=True) - kic_pmtotal = FloatField(null=True) - kic_pq = TextField(null=True) - kic_ra = DoubleField(index=True, null=True) - kic_radius = FloatField(null=True) - kic_rmag = DoubleField(index=True, null=True) - kic_scpid = BigIntegerField(null=True) - kic_scpkey = BigIntegerField(null=True) - kic_teff = IntegerField(null=True) - kic_tm_designation = TextField(null=True) - kic_tmid = BigIntegerField(null=True) - kic_umag = DoubleField(index=True, null=True) - kic_variable = IntegerField(null=True) - kic_zmag = DoubleField(index=True, null=True) - class Meta: table_name = 'kepler_input_10' schema = 'catalogdb' + use_reflection = True class SDSSDR13Photoobj(SDSS5dbModel): - aperflux7_g = FloatField(null=True) - aperflux7_i = FloatField(null=True) - aperflux7_r = FloatField(null=True) - aperflux7_u = FloatField(null=True) - aperflux7_z = FloatField(null=True) - b = DoubleField(index=True, null=True) - balkanid = IntegerField(null=True) - calibstatus_g = IntegerField(null=True) - calibstatus_i = IntegerField(null=True) - calibstatus_r = IntegerField(null=True) - calibstatus_u = IntegerField(null=True) - calibstatus_z = IntegerField(null=True) - camcol = IntegerField(null=True) - colc = FloatField(null=True) - colcerr = FloatField(null=True) - colv = FloatField(null=True) - colverr = FloatField(null=True) - dec = DoubleField(index=True, null=True) - devflux_g = FloatField(null=True) - devflux_i = FloatField(null=True) - devflux_r = FloatField(null=True) - devflux_u = FloatField(null=True) - devflux_z = FloatField(null=True) - devfluxivar_g = FloatField(null=True) - devfluxivar_i = FloatField(null=True) - devfluxivar_r = FloatField(null=True) - devfluxivar_u = FloatField(null=True) - devfluxivar_z = FloatField(null=True) - expflux_g = FloatField(null=True) - expflux_i = FloatField(null=True) - expflux_r = FloatField(null=True) - expflux_u = FloatField(null=True) - expflux_z = FloatField(null=True) - expfluxivar_g = FloatField(null=True) - expfluxivar_i = FloatField(null=True) - expfluxivar_r = FloatField(null=True) - expfluxivar_u = FloatField(null=True) - expfluxivar_z = FloatField(null=True) - extinction_g = FloatField(null=True) - extinction_i = FloatField(null=True) - extinction_r = FloatField(null=True) - extinction_u = FloatField(null=True) - extinction_z = FloatField(null=True) - fiber2flux_g = FloatField(null=True) - fiber2flux_i = FloatField(null=True) - fiber2flux_r = FloatField(null=True) - fiber2flux_u = FloatField(null=True) - fiber2flux_z = FloatField(null=True) - fiber2fluxivar_g = FloatField(null=True) - fiber2fluxivar_i = FloatField(null=True) - fiber2fluxivar_r = FloatField(null=True) - fiber2fluxivar_u = FloatField(null=True) - fiber2fluxivar_z = FloatField(null=True) - fiber2mag_g = FloatField(null=True) - fiber2mag_i = FloatField(null=True) - fiber2mag_r = FloatField(null=True) - fiber2mag_u = FloatField(null=True) - fiber2mag_z = FloatField(null=True) - fiber2magerr_g = FloatField(null=True) - fiber2magerr_i = FloatField(null=True) - fiber2magerr_r = FloatField(null=True) - fiber2magerr_u = FloatField(null=True) - fiber2magerr_z = FloatField(null=True) - fiberflux_g = FloatField(null=True) - fiberflux_i = FloatField(null=True) - fiberflux_r = FloatField(null=True) - fiberflux_u = FloatField(null=True) - fiberflux_z = FloatField(null=True) - fiberfluxivar_g = FloatField(null=True) - fiberfluxivar_i = FloatField(null=True) - fiberfluxivar_r = FloatField(null=True) - fiberfluxivar_u = FloatField(null=True) - fiberfluxivar_z = FloatField(null=True) - fibermag_g = FloatField(null=True) - fibermag_i = FloatField(null=True) - fibermag_r = FloatField(null=True) - fibermag_u = FloatField(null=True) - fibermag_z = FloatField(null=True) - fibermagerr_g = FloatField(null=True) - fibermagerr_i = FloatField(null=True) - fibermagerr_r = FloatField(null=True) - fibermagerr_u = FloatField(null=True) - fibermagerr_z = FloatField(null=True) - field = IntegerField(null=True) - fieldid = BigIntegerField(null=True) - flags = BigIntegerField(null=True) - htmid = BigIntegerField(null=True) - l = DoubleField(index=True, null=True) # noqa - lnldev_g = FloatField(null=True) - lnldev_i = FloatField(null=True) - lnldev_r = FloatField(null=True) - lnldev_u = FloatField(null=True) - lnldev_z = FloatField(null=True) - lnlexp_g = FloatField(null=True) - lnlexp_i = FloatField(null=True) - lnlexp_r = FloatField(null=True) - lnlexp_u = FloatField(null=True) - lnlexp_z = FloatField(null=True) - lnlstar_g = FloatField(null=True) - lnlstar_i = FloatField(null=True) - lnlstar_r = FloatField(null=True) - lnlstar_u = FloatField(null=True) - lnlstar_z = FloatField(null=True) - mrrcc_g = FloatField(null=True) - mrrcc_i = FloatField(null=True) - mrrcc_r = FloatField(null=True) - mrrcc_u = FloatField(null=True) - mrrcc_z = FloatField(null=True) - ndetect = IntegerField(null=True) - nmgypercount_g = FloatField(null=True) - nmgypercount_i = FloatField(null=True) - nmgypercount_r = FloatField(null=True) - nmgypercount_u = FloatField(null=True) - nmgypercount_z = FloatField(null=True) - obj = IntegerField(null=True) - objid = BigAutoField() - parentid = BigIntegerField(null=True) - petroflux_g = FloatField(null=True) - petroflux_i = FloatField(null=True) - petroflux_r = FloatField(null=True) - petroflux_u = FloatField(null=True) - petroflux_z = FloatField(null=True) - petrofluxivar_g = FloatField(null=True) - petrofluxivar_i = FloatField(null=True) - petrofluxivar_r = FloatField(null=True) - petrofluxivar_u = FloatField(null=True) - petrofluxivar_z = FloatField(null=True) - petromag_g = FloatField(null=True) - petromag_i = FloatField(null=True) - petromag_r = FloatField(null=True) - petromag_u = FloatField(null=True) - petromag_z = FloatField(null=True) - petromagerr_g = FloatField(null=True) - petromagerr_i = FloatField(null=True) - petromagerr_r = FloatField(null=True) - petromagerr_u = FloatField(null=True) - petromagerr_z = FloatField(null=True) - petror50_g = FloatField(null=True) - petror50_i = FloatField(null=True) - petror50_r = FloatField(null=True) - petror50_u = FloatField(null=True) - petror50_z = FloatField(null=True) - petror50err_g = FloatField(null=True) - petror50err_i = FloatField(null=True) - petror50err_r = FloatField(null=True) - petror50err_u = FloatField(null=True) - petror50err_z = FloatField(null=True) - petror90_g = FloatField(null=True) - petror90_i = FloatField(null=True) - petror90_r = FloatField(null=True) - petror90_u = FloatField(null=True) - petror90_z = FloatField(null=True) - petror90err_g = FloatField(null=True) - petror90err_i = FloatField(null=True) - petror90err_r = FloatField(null=True) - petror90err_u = FloatField(null=True) - petror90err_z = FloatField(null=True) - petrorad_g = FloatField(null=True) - petrorad_i = FloatField(null=True) - petrorad_r = FloatField(null=True) - petrorad_u = FloatField(null=True) - petrorad_z = FloatField(null=True) - petroraderr_g = FloatField(null=True) - petroraderr_i = FloatField(null=True) - petroraderr_r = FloatField(null=True) - petroraderr_u = FloatField(null=True) - petroraderr_z = FloatField(null=True) - psfflux_g = FloatField(null=True) - psfflux_i = FloatField(null=True) - psfflux_r = FloatField(null=True) - psfflux_u = FloatField(null=True) - psfflux_z = FloatField(null=True) - psffluxivar_g = FloatField(null=True) - psffluxivar_i = FloatField(null=True) - psffluxivar_r = FloatField(null=True) - psffluxivar_u = FloatField(null=True) - psffluxivar_z = FloatField(null=True) - psffwhm_g = FloatField(null=True) - psffwhm_i = FloatField(null=True) - psffwhm_r = FloatField(null=True) - psffwhm_u = FloatField(null=True) - psffwhm_z = FloatField(null=True) - psfmag_g = FloatField(index=True, null=True) - psfmag_i = FloatField(index=True, null=True) - psfmag_r = FloatField(index=True, null=True) - psfmag_u = FloatField(index=True, null=True) - psfmag_z = FloatField(index=True, null=True) - psfmagerr_g = FloatField(null=True) - psfmagerr_i = FloatField(null=True) - psfmagerr_r = FloatField(null=True) - psfmagerr_u = FloatField(null=True) - psfmagerr_z = FloatField(null=True) - ra = DoubleField(index=True, null=True) - rerun = IntegerField(null=True) - resolvestatus = IntegerField(null=True) - rowc = FloatField(null=True) - rowcerr = FloatField(null=True) - rowv = FloatField(null=True) - rowverr = FloatField(null=True) - run = IntegerField(null=True) - sky_g = FloatField(null=True) - sky_i = FloatField(null=True) - sky_r = FloatField(null=True) - sky_u = FloatField(null=True) - sky_z = FloatField(null=True) - skyivar_g = FloatField(null=True) - skyivar_i = FloatField(null=True) - skyivar_r = FloatField(null=True) - skyivar_u = FloatField(null=True) - skyivar_z = FloatField(null=True) - specobjid = DecimalField(null=True) - thingid = IntegerField(null=True) - type = IntegerField(null=True) - class Meta: table_name = 'sdss_dr13_photoobj' schema = 'catalogdb' + use_reflection = True class SDSSDR14APOGEEStar(SDSS5dbModel): - andflag = BigIntegerField(null=True) - apogee2_target1 = BigIntegerField(null=True) - apogee2_target2 = BigIntegerField(null=True) - apogee2_target3 = BigIntegerField(null=True) - apogee_id = CharField(null=True) - apogee_target1 = BigIntegerField(null=True) - apogee_target2 = BigIntegerField(null=True) - apogee_target3 = BigIntegerField(null=True) - apstar_id = CharField(primary_key=True) - apstar_version = CharField(null=True) - chi2_threshold = FloatField(null=True) - commiss = BigIntegerField(null=True) - dec = DoubleField(index=True, null=True) - extratarg = BigIntegerField(null=True) - field = CharField(null=True) - file = CharField(null=True) - glat = DoubleField(index=True, null=True) - glon = DoubleField(index=True, null=True) - htmid = BigIntegerField(null=True) - location_id = BigIntegerField(null=True) - max_h = FloatField(null=True) - max_jk = FloatField(null=True) - min_h = FloatField(null=True) - min_jk = FloatField(null=True) - nvisits = BigIntegerField(null=True) - ra = DoubleField(index=True, null=True) - reduction_id = CharField(null=True) - rv_autofwhm = FloatField(null=True) - rv_ccfwhm = FloatField(null=True) - rv_feh = FloatField(null=True) - rv_logg = FloatField(null=True) - rv_teff = FloatField(null=True) - snr = FloatField(null=True) - stablerv_chi2 = FloatField(null=True) - stablerv_chi2_prob = FloatField(null=True) - stablerv_rchi2 = FloatField(null=True) - starflag = BigIntegerField(null=True) - survey = CharField(null=True) - synthscatter = FloatField(null=True) - synthverr = FloatField(null=True) - synthverr_med = FloatField(null=True) - synthvhelio_avg = FloatField(null=True) - synthvscatter = FloatField(null=True) - target_id = CharField(null=True) - telescope = CharField(null=True) - verr = FloatField(null=True) - verr_med = FloatField(null=True) - vhelio_avg = FloatField(null=True) - vscatter = FloatField(null=True) - class Meta: table_name = 'sdss_dr14_apogeestar' schema = 'catalogdb' + use_reflection = True class SDSSDR14APOGEEStarVisit(SDSS5dbModel): - visit_id = CharField(primary_key=True) - apstar_id = CharField(index=True, null=True) - class Meta: table_name = 'sdss_dr14_apogeestarvisit' schema = 'catalogdb' + use_reflection = True class SDSSDR14APOGEEVisit(SDSS5dbModel): - apogee2_target1 = BigIntegerField(null=True) - apogee2_target2 = BigIntegerField(null=True) - apogee2_target3 = BigIntegerField(null=True) - apogee_id = CharField(null=True) - apogee_target1 = BigIntegerField(null=True) - apogee_target2 = BigIntegerField(null=True) - apogee_target3 = BigIntegerField(null=True) - apred_version = CharField(null=True) - bc = FloatField(null=True) - chisq = FloatField(null=True) - dateobs = CharField(null=True) - dec = DoubleField(index=True, null=True) - estvhelio = FloatField(null=True) - estvrel = FloatField(null=True) - estvrelerr = FloatField(null=True) - estvtype = BigIntegerField(null=True) - extratarg = BigIntegerField(null=True) - fiberid = BigIntegerField(null=True) - file = CharField(null=True) - glat = DoubleField(index=True, null=True) - glon = DoubleField(index=True, null=True) - jd = DoubleField(null=True) - location_id = BigIntegerField(null=True) - max_h = FloatField(null=True) - max_jk = FloatField(null=True) - min_h = FloatField(null=True) - min_jk = FloatField(null=True) - mjd = BigIntegerField(null=True) - plate = CharField(null=True) - ra = DoubleField(index=True, null=True) - reduction_id = CharField(null=True) - rv_alpha = FloatField(null=True) - rv_carb = FloatField(null=True) - rv_feh = FloatField(null=True) - rv_logg = FloatField(null=True) - rv_teff = FloatField(null=True) - snr = FloatField(null=True) - starflag = BigIntegerField(null=True) - survey = CharField(null=True) - synthfile = CharField(null=True) - synthvhelio = FloatField(null=True) - synthvrel = FloatField(null=True) - synthvrelerr = FloatField(null=True) - target_id = CharField(null=True) - telescope = CharField(null=True) - vhelio = FloatField(null=True) - visit_id = CharField(primary_key=True) - vrel = FloatField(null=True) - vrelerr = FloatField(null=True) - vtype = FloatField(null=True) - class Meta: table_name = 'sdss_dr14_apogeevisit' schema = 'catalogdb' + use_reflection = True class SDSSDR14ASCAPStar(SDSS5dbModel): - al_fe = FloatField(null=True) - al_fe_err = FloatField(null=True) - al_fe_flag = IntegerField(null=True) - alpha_m = FloatField(null=True) - alpha_m_err = FloatField(null=True) - alpha_m_flag = IntegerField(null=True) - apogee_id = CharField(index=True, null=True) - apstar_id = CharField(primary_key=True) - aspcap_chi2 = FloatField(null=True) - aspcap_class = CharField(null=True) - aspcap_id = CharField(null=True) - aspcap_version = CharField(null=True) - aspcapflag = BigIntegerField(null=True) - c_fe = FloatField(null=True) - c_fe_err = FloatField(null=True) - c_fe_flag = IntegerField(null=True) - ca_fe = FloatField(null=True) - ca_fe_err = FloatField(null=True) - ca_fe_flag = IntegerField(null=True) - ci_fe = FloatField(null=True) - ci_fe_err = FloatField(null=True) - ci_fe_flag = IntegerField(null=True) - co_fe = FloatField(null=True) - co_fe_err = FloatField(null=True) - co_fe_flag = IntegerField(null=True) - cr_fe = FloatField(null=True) - cr_fe_err = FloatField(null=True) - cr_fe_flag = IntegerField(null=True) - cu_fe = FloatField(null=True) - cu_fe_err = FloatField(null=True) - cu_fe_flag = IntegerField(null=True) - fe_h = FloatField(null=True) - fe_h_err = FloatField(null=True) - fe_h_flag = IntegerField(null=True) - felem_al_h = FloatField(null=True) - felem_al_h_err = FloatField(null=True) - felem_c_m = FloatField(null=True) - felem_c_m_err = FloatField(null=True) - felem_ca_m = FloatField(null=True) - felem_ca_m_err = FloatField(null=True) - felem_ci_m = FloatField(null=True) - felem_ci_m_err = FloatField(null=True) - felem_co_h = FloatField(null=True) - felem_co_h_err = FloatField(null=True) - felem_cr_h = FloatField(null=True) - felem_cr_h_err = FloatField(null=True) - felem_cu_h = FloatField(null=True) - felem_cu_h_err = FloatField(null=True) - felem_fe_h = FloatField(null=True) - felem_fe_h_err = FloatField(null=True) - felem_ge_h = FloatField(null=True) - felem_ge_h_err = FloatField(null=True) - felem_k_h = FloatField(null=True) - felem_k_h_err = FloatField(null=True) - felem_mg_m = FloatField(null=True) - felem_mg_m_err = FloatField(null=True) - felem_mn_h = FloatField(null=True) - felem_mn_h_err = FloatField(null=True) - felem_n_m = FloatField(null=True) - felem_n_m_err = FloatField(null=True) - felem_na_h = FloatField(null=True) - felem_na_h_err = FloatField(null=True) - felem_nd_h = FloatField(null=True) - felem_nd_h_err = FloatField(null=True) - felem_ni_h = FloatField(null=True) - felem_ni_h_err = FloatField(null=True) - felem_o_m = FloatField(null=True) - felem_o_m_err = FloatField(null=True) - felem_p_m = FloatField(null=True) - felem_p_m_err = FloatField(null=True) - felem_rb_h = FloatField(null=True) - felem_rb_h_err = FloatField(null=True) - felem_s_m = FloatField(null=True) - felem_s_m_err = FloatField(null=True) - felem_si_m = FloatField(null=True) - felem_si_m_err = FloatField(null=True) - felem_ti_m = FloatField(null=True) - felem_ti_m_err = FloatField(null=True) - felem_tiii_m = FloatField(null=True) - felem_tiii_m_err = FloatField(null=True) - felem_v_h = FloatField(null=True) - felem_v_h_err = FloatField(null=True) - felem_y_h = FloatField(null=True) - felem_y_h_err = FloatField(null=True) - fparam_alpha_m = FloatField(null=True) - fparam_c_m = FloatField(null=True) - fparam_logg = FloatField(null=True) - fparam_logvmicro = FloatField(null=True) - fparam_m_h = FloatField(null=True) - fparam_n_m = FloatField(null=True) - fparam_teff = FloatField(null=True) - ge_fe = FloatField(null=True) - ge_fe_err = FloatField(null=True) - ge_fe_flag = IntegerField(null=True) - k_fe = FloatField(null=True) - k_fe_err = FloatField(null=True) - k_fe_flag = IntegerField(null=True) - logg = FloatField(null=True) - logg_err = FloatField(null=True) - logg_flag = IntegerField(null=True) - m_h = FloatField(null=True) - m_h_err = FloatField(null=True) - m_h_flag = IntegerField(null=True) - mg_fe = FloatField(null=True) - mg_fe_err = FloatField(null=True) - mg_fe_flag = IntegerField(null=True) - mn_fe = FloatField(null=True) - mn_fe_err = FloatField(null=True) - mn_fe_flag = IntegerField(null=True) - n_fe = FloatField(null=True) - n_fe_err = FloatField(null=True) - n_fe_flag = IntegerField(null=True) - na_fe = FloatField(null=True) - na_fe_err = FloatField(null=True) - na_fe_flag = IntegerField(null=True) - nd_fe = FloatField(null=True) - nd_fe_err = FloatField(null=True) - nd_fe_flag = IntegerField(null=True) - ni_fe = FloatField(null=True) - ni_fe_err = FloatField(null=True) - ni_fe_flag = IntegerField(null=True) - o_fe = FloatField(null=True) - o_fe_err = FloatField(null=True) - o_fe_flag = IntegerField(null=True) - p_fe = FloatField(null=True) - p_fe_err = FloatField(null=True) - p_fe_flag = IntegerField(null=True) - param_c_m = FloatField(null=True) - param_c_m_flag = IntegerField(null=True) - param_n_m = FloatField(null=True) - param_n_m_flag = IntegerField(null=True) - rb_fe = FloatField(null=True) - rb_fe_err = FloatField(null=True) - rb_fe_flag = IntegerField(null=True) - results_version = CharField(null=True) - s_fe = FloatField(null=True) - s_fe_err = FloatField(null=True) - s_fe_flag = IntegerField(null=True) - si_fe = FloatField(null=True) - si_fe_err = FloatField(null=True) - si_fe_flag = IntegerField(null=True) - target_id = CharField(index=True, null=True) - teff = FloatField(null=True) - teff_err = FloatField(null=True) - teff_flag = IntegerField(null=True) - ti_fe = FloatField(null=True) - ti_fe_err = FloatField(null=True) - ti_fe_flag = IntegerField(null=True) - tiii_fe = FloatField(null=True) - tiii_fe_err = FloatField(null=True) - tiii_fe_flag = IntegerField(null=True) - v_fe = FloatField(null=True) - v_fe_err = FloatField(null=True) - v_fe_flag = IntegerField(null=True) - vmacro = FloatField(null=True) - vmicro = FloatField(null=True) - vsini = FloatField(null=True) - y_fe = FloatField(null=True) - y_fe_err = FloatField(null=True) - y_fe_flag = IntegerField(null=True) - class Meta: table_name = 'sdss_dr14_ascapstar' schema = 'catalogdb' + use_reflection = True class SDSSDR14CannonStar(SDSS5dbModel): - al_h = FloatField(null=True) - al_h_err = FloatField(null=True) - al_h_rawerr = FloatField(null=True) - alpha_m = FloatField(null=True) - alpha_m_err = FloatField(null=True) - alpha_m_rawerr = FloatField(null=True) - apogee_id = CharField(index=True, null=True) - c_h = FloatField(null=True) - c_h_err = FloatField(null=True) - c_h_rawerr = FloatField(null=True) - ca_h = FloatField(null=True) - ca_h_err = FloatField(null=True) - ca_h_rawerr = FloatField(null=True) - cannon_id = CharField(primary_key=True) - chi_sq = FloatField(null=True) - ci_h = FloatField(null=True) - ci_h_err = FloatField(null=True) - ci_h_rawerr = FloatField(null=True) - co_h = FloatField(null=True) - co_h_err = FloatField(null=True) - co_h_rawerr = FloatField(null=True) - cr_h = FloatField(null=True) - cr_h_err = FloatField(null=True) - cr_h_rawerr = FloatField(null=True) - fe_h = FloatField(null=True) - fe_h_err = FloatField(null=True) - fe_h_rawerr = FloatField(null=True) - field = CharField(null=True) - filename = CharField(null=True) - k_h = FloatField(null=True) - k_h_err = FloatField(null=True) - k_h_rawerr = FloatField(null=True) - location_id = BigIntegerField(null=True) - logg = FloatField(null=True) - logg_err = FloatField(null=True) - logg_rawerr = FloatField(null=True) - m_h = FloatField(null=True) - m_h_err = FloatField(null=True) - m_h_rawerr = FloatField(null=True) - mg_h = FloatField(null=True) - mg_h_err = FloatField(null=True) - mg_h_rawerr = FloatField(null=True) - mn_h = FloatField(null=True) - mn_h_err = FloatField(null=True) - mn_h_rawerr = FloatField(null=True) - n_h = FloatField(null=True) - n_h_err = FloatField(null=True) - n_h_rawerr = FloatField(null=True) - na_h = FloatField(null=True) - na_h_err = FloatField(null=True) - na_h_rawerr = FloatField(null=True) - ni_h = FloatField(null=True) - ni_h_err = FloatField(null=True) - ni_h_rawerr = FloatField(null=True) - o_h = FloatField(null=True) - o_h_err = FloatField(null=True) - o_h_rawerr = FloatField(null=True) - p_h = FloatField(null=True) - p_h_err = FloatField(null=True) - p_h_rawerr = FloatField(null=True) - r_chi_sq = FloatField(null=True) - s_h = FloatField(null=True) - s_h_err = FloatField(null=True) - s_h_rawerr = FloatField(null=True) - si_h = FloatField(null=True) - si_h_err = FloatField(null=True) - si_h_rawerr = FloatField(null=True) - teff = FloatField(null=True) - teff_err = FloatField(null=True) - teff_rawerr = FloatField(null=True) - ti_h = FloatField(null=True) - ti_h_err = FloatField(null=True) - ti_h_rawerr = FloatField(null=True) - tiii_h = FloatField(null=True) - tiii_h_err = FloatField(null=True) - tiii_h_rawerr = FloatField(null=True) - v_h = FloatField(null=True) - v_h_err = FloatField(null=True) - v_h_rawerr = FloatField(null=True) - class Meta: table_name = 'sdss_dr14_cannonstar' schema = 'catalogdb' + use_reflection = True class SDSSDR14SpecObj(SDSS5dbModel): - ancillary_target1 = BigIntegerField(null=True) - ancillary_target2 = BigIntegerField(null=True) - anyandmask = IntegerField(null=True) - anyormask = IntegerField(null=True) - bestobjid = BigIntegerField(index=True, null=True) - bluefiber = IntegerField(null=True) - boss_target1 = BigIntegerField(null=True) - bossprimary = IntegerField(null=True) - bossspecobjid = IntegerField(null=True) - chi68p = FloatField(null=True) - chunk = CharField(null=True) - class_ = CharField(column_name='class', null=True) - class_noqso = CharField(null=True) - class_person = CharField(null=True) - comments_person = CharField(null=True) - cx = DoubleField(null=True) - cy = DoubleField(null=True) - cz = DoubleField(null=True) - dec = DoubleField(index=True, null=True) - deredsn2 = FloatField(null=True) - designid = IntegerField(null=True) - dof = FloatField(null=True) - eboss_target0 = BigIntegerField(null=True) - eboss_target1 = BigIntegerField(null=True) - eboss_target2 = BigIntegerField(null=True) - eboss_target_id = BigIntegerField(null=True) - elodiebv = FloatField(null=True) - elodiedof = FloatField(null=True) - elodiefeh = FloatField(null=True) - elodiefilename = CharField(null=True) - elodielogg = FloatField(null=True) - elodieobject = CharField(null=True) - elodierchi2 = FloatField(null=True) - elodiesptype = CharField(null=True) - elodieteff = FloatField(null=True) - elodiez = FloatField(null=True) - elodiezerr = FloatField(null=True) - elodiezmodelerr = FloatField(null=True) - fiberid = IntegerField(null=True) - firstrelease = CharField(null=True) - fluxobjid = BigIntegerField(index=True, null=True) - fracnsighi_1 = FloatField(null=True) - fracnsighi_10 = FloatField(null=True) - fracnsighi_2 = FloatField(null=True) - fracnsighi_3 = FloatField(null=True) - fracnsighi_4 = FloatField(null=True) - fracnsighi_5 = FloatField(null=True) - fracnsighi_6 = FloatField(null=True) - fracnsighi_7 = FloatField(null=True) - fracnsighi_8 = FloatField(null=True) - fracnsighi_9 = FloatField(null=True) - fracnsiglo_1 = FloatField(null=True) - fracnsiglo_10 = FloatField(null=True) - fracnsiglo_2 = FloatField(null=True) - fracnsiglo_3 = FloatField(null=True) - fracnsiglo_4 = FloatField(null=True) - fracnsiglo_5 = FloatField(null=True) - fracnsiglo_6 = FloatField(null=True) - fracnsiglo_7 = FloatField(null=True) - fracnsiglo_8 = FloatField(null=True) - fracnsiglo_9 = FloatField(null=True) - fracnsigma_1 = FloatField(null=True) - fracnsigma_10 = FloatField(null=True) - fracnsigma_2 = FloatField(null=True) - fracnsigma_3 = FloatField(null=True) - fracnsigma_4 = FloatField(null=True) - fracnsigma_5 = FloatField(null=True) - fracnsigma_6 = FloatField(null=True) - fracnsigma_7 = FloatField(null=True) - fracnsigma_8 = FloatField(null=True) - fracnsigma_9 = FloatField(null=True) - htmid = BigIntegerField(null=True) - instrument = CharField(null=True) - lambdaeff = FloatField(null=True) - legacy_target1 = BigIntegerField(null=True) - legacy_target2 = BigIntegerField(null=True) - legacyprimary = IntegerField(null=True) - loadversion = IntegerField(null=True) - mjd = IntegerField(null=True) - npoly = FloatField(null=True) - plate = IntegerField(null=True) - plateid = DecimalField(null=True) - platerun = CharField(null=True) - platesn2 = FloatField(null=True) - primtarget = BigIntegerField(null=True) - programname = CharField(null=True) - ra = DoubleField(index=True, null=True) - rchi2 = FloatField(null=True) - rchi2diff = FloatField(null=True) - rchi2diff_noqso = FloatField(null=True) - run1d = CharField(null=True) - run2d = CharField(null=True) - scienceprimary = IntegerField(null=True) - sdssprimary = IntegerField(null=True) - sectarget = BigIntegerField(null=True) - segue1_target1 = BigIntegerField(null=True) - segue1_target2 = BigIntegerField(null=True) - segue1primary = IntegerField(null=True) - segue2_target1 = BigIntegerField(null=True) - segue2_target2 = BigIntegerField(null=True) - segue2primary = IntegerField(null=True) - segueprimary = IntegerField(null=True) - sn1_g = FloatField(null=True) - sn1_i = FloatField(null=True) - sn1_r = FloatField(null=True) - sn2_g = FloatField(null=True) - sn2_i = FloatField(null=True) - sn2_r = FloatField(null=True) - snmedian = FloatField(null=True) - snmedian_g = FloatField(null=True) - snmedian_i = FloatField(null=True) - snmedian_r = FloatField(null=True) - snmedian_u = FloatField(null=True) - snmedian_z = FloatField(null=True) - snturnoff = FloatField(null=True) - sourcetype = CharField(null=True) - special_target1 = BigIntegerField(null=True) - special_target2 = BigIntegerField(null=True) - specobjid = DecimalField(primary_key=True) - spectroflux_g = FloatField(null=True) - spectroflux_i = FloatField(null=True) - spectroflux_r = FloatField(null=True) - spectroflux_u = FloatField(null=True) - spectroflux_z = FloatField(null=True) - spectrofluxivar_g = FloatField(null=True) - spectrofluxivar_i = FloatField(null=True) - spectrofluxivar_r = FloatField(null=True) - spectrofluxivar_u = FloatField(null=True) - spectrofluxivar_z = FloatField(null=True) - spectrographid = IntegerField(null=True) - spectroskyflux_g = FloatField(null=True) - spectroskyflux_i = FloatField(null=True) - spectroskyflux_r = FloatField(null=True) - spectroskyflux_u = FloatField(null=True) - spectroskyflux_z = FloatField(null=True) - spectrosynflux_g = FloatField(null=True) - spectrosynflux_i = FloatField(null=True) - spectrosynflux_r = FloatField(null=True) - spectrosynflux_u = FloatField(null=True) - spectrosynflux_z = FloatField(null=True) - spectrosynfluxivar_g = FloatField(null=True) - spectrosynfluxivar_i = FloatField(null=True) - spectrosynfluxivar_r = FloatField(null=True) - spectrosynfluxivar_u = FloatField(null=True) - spectrosynfluxivar_z = FloatField(null=True) - subclass = CharField(null=True) - subclass_noqso = CharField(null=True) - survey = CharField(null=True) - targetobjid = BigIntegerField(index=True, null=True) - targettype = CharField(null=True) - tcolumn_0 = IntegerField(null=True) - tcolumn_1 = IntegerField(null=True) - tcolumn_2 = IntegerField(null=True) - tcolumn_3 = IntegerField(null=True) - tcolumn_4 = IntegerField(null=True) - tcolumn_5 = IntegerField(null=True) - tcolumn_6 = IntegerField(null=True) - tcolumn_7 = IntegerField(null=True) - tcolumn_8 = IntegerField(null=True) - tcolumn_9 = IntegerField(null=True) - tfile = CharField(null=True) - theta_0 = FloatField(null=True) - theta_1 = FloatField(null=True) - theta_2 = FloatField(null=True) - theta_3 = FloatField(null=True) - theta_4 = FloatField(null=True) - theta_5 = FloatField(null=True) - theta_6 = FloatField(null=True) - theta_7 = FloatField(null=True) - theta_8 = FloatField(null=True) - theta_9 = FloatField(null=True) - thing_id = IntegerField(null=True) - thing_id_targeting = BigIntegerField(null=True) - tile = IntegerField(null=True) - veldisp = FloatField(null=True) - veldispchi2 = FloatField(null=True) - veldispdof = IntegerField(null=True) - veldisperr = FloatField(null=True) - veldispnpix = IntegerField(null=True) - veldispz = FloatField(null=True) - veldispzerr = FloatField(null=True) - wavemax = FloatField(null=True) - wavemin = FloatField(null=True) - wcoverage = FloatField(null=True) - xfocal = FloatField(null=True) - yfocal = FloatField(null=True) - z = FloatField(null=True) - z_noqso = FloatField(null=True) - z_person = FloatField(null=True) - zerr = FloatField(null=True) - zerr_noqso = FloatField(null=True) - zoffset = FloatField(null=True) - zwarning = IntegerField(null=True) - zwarning_noqso = IntegerField(null=True) - class Meta: table_name = 'sdss_dr14_specobj' schema = 'catalogdb' + use_reflection = True class SDSSDR16SpecObj(SDSSDR14SpecObj): class Meta: table_name = 'sdss_dr16_specobj' - schema = 'catalogdb' class TIC_v8(SDSS5dbModel): - allwise = CharField(index=True, null=True) - apass = CharField(index=True, null=True) - bmag = FloatField(null=True) - bmagflag = CharField(null=True) - contratio = FloatField(null=True) - d = FloatField(null=True) - dec = DoubleField(null=True) - dec_orig = DoubleField(null=True) - disposition = CharField(null=True) - distflag = CharField(null=True) - duplicate_id = BigIntegerField(null=True) - e_bmag = FloatField(null=True) - e_d = FloatField(null=True) - e_dec = DoubleField(null=True) - e_dec_orig = DoubleField(null=True) - e_ebv = FloatField(null=True) - e_gaiabp = FloatField(null=True) - e_gaiamag = FloatField(null=True) - e_gaiarp = FloatField(null=True) - e_gmag = FloatField(null=True) - e_hmag = FloatField(null=True) - e_imag = FloatField(null=True) - e_jmag = FloatField(null=True) - e_kmag = FloatField(null=True) - e_logg = FloatField(null=True) - e_lum = FloatField(null=True) - e_mass = FloatField(null=True) - e_mh = FloatField(null=True) - e_plx = FloatField(null=True) - e_pmdec = FloatField(null=True) - e_pmra = FloatField(null=True) - e_ra = DoubleField(null=True) - e_ra_orig = DoubleField(null=True) - e_rad = FloatField(null=True) - e_rho = FloatField(null=True) - e_rmag = FloatField(null=True) - e_teff = FloatField(null=True) - e_tmag = FloatField(null=True) - e_umag = FloatField(null=True) - e_vmag = FloatField(null=True) - e_w1mag = FloatField(null=True) - e_w2mag = FloatField(null=True) - e_w3mag = FloatField(null=True) - e_w4mag = FloatField(null=True) - e_zmag = FloatField(null=True) - ebv = FloatField(null=True) - ebvflag = CharField(null=True) - eclat = DoubleField(null=True) - eclong = DoubleField(null=True) - eneg_dist = FloatField(null=True) - eneg_ebv = FloatField(null=True) - eneg_logg = FloatField(null=True) - eneg_lum = FloatField(null=True) - eneg_mass = FloatField(null=True) - eneg_rad = FloatField(null=True) - eneg_rho = FloatField(null=True) - eneg_teff = FloatField(null=True) - epos_dist = FloatField(null=True) - epos_ebv = FloatField(null=True) - epos_logg = FloatField(null=True) - epos_lum = FloatField(null=True) - epos_mass = FloatField(null=True) - epos_rad = FloatField(null=True) - epos_rho = FloatField(null=True) - epos_teff = FloatField(null=True) - gaia = CharField(index=True, null=True) - gaiabp = FloatField(null=True) - gaiamag = FloatField(null=True) - gaiaqflag = IntegerField(null=True) - gaiarp = FloatField(null=True) - gallat = DoubleField(null=True) - gallong = DoubleField(null=True) - gmag = FloatField(null=True) - hip = IntegerField(index=True, null=True) - hmag = FloatField(null=True) - id = BigIntegerField(null=True) - imag = FloatField(null=True) - jmag = FloatField(null=True) - kic = IntegerField(index=True, null=True) - kmag = FloatField(null=True) - logg = FloatField(null=True) - lum = FloatField(null=True) - lumclass = CharField(null=True) - mass = FloatField(null=True) - mh = FloatField(null=True) - numcont = IntegerField(null=True) - objid = BigIntegerField(null=True) - objtype = CharField(null=True) - parflag = CharField(null=True) - plx = FloatField(null=True) - pmdec = FloatField(null=True) - pmflag = CharField(null=True) - pmra = FloatField(null=True) - posflag = CharField(null=True) - priority = FloatField(null=True) - prox = FloatField(null=True) - ra = DoubleField(null=True) - ra_orig = DoubleField(null=True) - rad = FloatField(null=True) - raddflag = IntegerField(null=True) - rho = FloatField(null=True) - rmag = FloatField(null=True) - sdss = BigIntegerField(index=True, null=True) - spflag = CharField(null=True) - splits = CharField(null=True) - starchareflag = CharField(null=True) - teff = FloatField(null=True) - tefflag = CharField(null=True) - tessflag = CharField(null=True) - tmag = FloatField(null=True) - twomass = CharField(index=True, null=True) - twomflag = CharField(null=True) - tyc = CharField(index=True, null=True) - typesrc = CharField(null=True) - ucac = CharField(index=True, null=True) - umag = FloatField(null=True) - version = CharField(null=True) - vmag = FloatField(null=True) - vmagflag = CharField(null=True) - w1mag = FloatField(null=True) - w2mag = FloatField(null=True) - w3mag = FloatField(null=True) - w4mag = FloatField(null=True) - wdflag = IntegerField(null=True) - zmag = FloatField(null=True) - class Meta: table_name = 'tic_v8' schema = 'catalogdb' + use_reflection = True class TwoMassClean(SDSS5dbModel): - decl = DoubleField(null=True) - designation = CharField(primary_key=True) - h_m = FloatField(index=True, null=True) - ra = DoubleField(null=True) - twomassbrightneighbor = BooleanField(index=True, null=True) - psc = ForeignKeyField(TwoMassPsc, field='pts_key', column_name='pts_key', backref='tmass_clean') class Meta: table_name = 'twomass_clean' schema = 'catalogdb' + use_reflection = True class TwoMassCleanNoNeighbor(SDSS5dbModel): - decl = DoubleField(null=True) - designation = CharField(primary_key=True) - h_m = FloatField(index=True, null=True) - pts_key = IntegerField(null=True) - ra = DoubleField(null=True) - class Meta: table_name = 'twomass_clean_noneighbor' schema = 'catalogdb' + use_reflection = True class GaiaDR2TmassBestNeighbour(SDSS5dbModel): - angular_distance = DoubleField(null=True) - best_neighbour_multiplicity = IntegerField(null=True) - gaia_astrometric_params = IntegerField(null=True) - number_of_mates = IntegerField(null=True) - number_of_neighbours = IntegerField(null=True) - tmass_oid = BigIntegerField(index=True, null=True) - gaia_source = ForeignKeyField(GaiaDR2Source, column_name='source_id', backref='tmass_best_neighbour') @@ -1772,404 +216,50 @@ class GaiaDR2TmassBestNeighbour(SDSS5dbModel): class Meta: table_name = 'gaiadr2_tmass_best_neighbour' schema = 'catalogdb' + use_reflection = True class DR14QV44(SDSS5dbModel): - pk = AutoField(null=False, primary_key=True) - ancillary_target1 = BigIntegerField(null=True) - ancillary_target2 = BigIntegerField(null=True) - bi_civ = FloatField(null=True) - boss_target1 = BigIntegerField(null=True) - cc_flags = CharField(null=True) - col_number = IntegerField(null=True) - dec = FloatField(null=True) - eboss_target0 = BigIntegerField(null=True) - eboss_target1 = BigIntegerField(null=True) - eboss_target2 = BigIntegerField(null=True) - err_bi_civ = FloatField(null=True) - err_hmag = FloatField(null=True) - err_jmag = FloatField(null=True) - err_kmag = FloatField(null=True) - err_psfmag = ArrayField(null=True) - err_w1mag = FloatField(null=True) - err_w2mag = FloatField(null=True) - err_w3mag = FloatField(null=True) - err_w4mag = FloatField(null=True) - fiberid = IntegerField(null=True) - fiberid_duplicate = ArrayField(field_class=IntegerField, null=True) - field_number = IntegerField(null=True) - first_flux = FloatField(null=True) - first_matched = IntegerField(null=True) - first_snr = FloatField(null=True) - flux_0_2_12_0kev = FloatField(column_name='flux_0.2_12.0kev', null=True) - flux_0_2_12_0kev_err = FloatField(column_name='flux_0.2_12.0kev_err', null=True) - flux_0_2_2_0kev = FloatField(column_name='flux_0.2_2.0kev', null=True) - flux_0_2_2_0kev_err = FloatField(column_name='flux_0.2_2.0kev_err', null=True) - flux_2_0_12_0kev = FloatField(column_name='flux_2.0_12.0kev', null=True) - flux_2_0_12_0kev_err = FloatField(column_name='flux_2.0_12.0kev_err', null=True) - fuv = FloatField(null=True) - fuv_ivar = FloatField(null=True) - gal_ext = ArrayField(null=True) - galex_matched = FloatField(null=True) - hflux = FloatField(null=True) - hflux_err = FloatField(null=True) - hmag = FloatField(null=True) - hrdflag = IntegerField(null=True) - hsnr = FloatField(null=True) - ivar_psfflux = ArrayField(null=True) - jflux = FloatField(null=True) - jflux_err = FloatField(null=True) - jmag = FloatField(null=True) - jrdflag = IntegerField(null=True) - jsnr = FloatField(null=True) - kflux = FloatField(null=True) - kflux_err = FloatField(null=True) - kmag = FloatField(null=True) - krdflag = IntegerField(null=True) - ksnr = FloatField(null=True) - lum_0_2_12_0kev = FloatField(column_name='lum_0.2_12.0kev', null=True) - mi = FloatField(null=True) - mjd = IntegerField(null=True) - mjd_duplicate = ArrayField(field_class=IntegerField, null=True) - n_spec = IntegerField(null=True) - n_spec_boss = IntegerField(null=True) - n_spec_sdss = IntegerField(null=True) - nuv = FloatField(null=True) - nuv_ivar = FloatField(null=True) - obj_id = CharField(null=True) - ph_flags = CharField(null=True) - plate = IntegerField(null=True) - plate_duplicate = ArrayField(field_class=IntegerField, null=True) - psfflux = ArrayField(null=True) - psfmag = ArrayField(null=True) - ra = FloatField(null=True) - rass_counts = FloatField(null=True) - rass_counts_snr = FloatField(null=True) - rerun_number = CharField(null=True) - run_number = IntegerField(null=True) - sdss2first_sep = FloatField(null=True) - sdss2mass_sep = FloatField(null=True) - sdss2rosat_sep = FloatField(null=True) - sdss2wise_sep = FloatField(null=True) - sdss2xmm_sep = FloatField(null=True) - sdss_name = CharField(null=True) - source_z = CharField(null=True) - spectro = CharField(null=True) - spectro_duplicate = ArrayField(field_class=CharField, null=True) - thing_id = IntegerField(null=True) - ukidss_matched = FloatField(null=True) - w1chi2 = FloatField(null=True) - w1mag = FloatField(null=True) - w1snr = FloatField(null=True) - w2chi2 = FloatField(null=True) - w2mag = FloatField(null=True) - w2snr = FloatField(null=True) - w3chi2 = FloatField(null=True) - w3mag = FloatField(null=True) - w3snr = FloatField(null=True) - w4chi2 = FloatField(null=True) - w4mag = FloatField(null=True) - w4snr = FloatField(null=True) - yflux = FloatField(null=True) - yflux_err = FloatField(null=True) - z = FloatField(null=True) - z_err = FloatField(null=True) - z_mgii = FloatField(null=True) - z_pca = FloatField(null=True) - z_pca_er = FloatField(null=True) - z_pipe = FloatField(null=True) - z_pipe_err = FloatField(null=True) - z_vi = FloatField(null=True) - zwarning = IntegerField(null=True) - class Meta: table_name = 'dr14q_v4_4' schema = 'catalogdb' + use_reflection = True class unWISE(SDSS5dbModel): - unwise_objid = TextField(primary_key=True) - x_w1 = DoubleField(null=True) - x_w2 = DoubleField(null=True) - y_w1 = DoubleField(null=True) - y_w2 = DoubleField(null=True) - flux_w1 = FloatField(null=True) - flux_w2 = FloatField(null=True) - dx_w1 = FloatField(null=True) - dx_w2 = FloatField(null=True) - dy_w1 = FloatField(null=True) - dy_w2 = FloatField(null=True) - dflux_w1 = FloatField(null=True) - dflux_w2 = FloatField(null=True) - qf_w1 = FloatField(null=True) - qf_w2 = FloatField(null=True) - rchi2_w1 = FloatField(null=True) - rchi2_w2 = FloatField(null=True) - fracflux_w1 = FloatField(null=True) - fracflux_w2 = FloatField(null=True) - fluxlbs_w1 = FloatField(null=True) - fluxlbs_w2 = FloatField(null=True) - dfluxlbs_w1 = FloatField(null=True) - dfluxlbs_w2 = FloatField(null=True) - fwhm_w1 = FloatField(null=True) - fwhm_w2 = FloatField(null=True) - spread_model_w1 = FloatField(null=True) - spread_model_w2 = FloatField(null=True) - dspread_model_w1 = FloatField(null=True) - dspread_model_w2 = FloatField(null=True) - sky_w1 = FloatField(null=True) - sky_w2 = FloatField(null=True) - ra12_w1 = DoubleField(null=True) - ra12_w2 = DoubleField(null=True) - dec12_w1 = DoubleField(null=True) - dec12_w2 = DoubleField(null=True) - coadd_id = TextField(null=True) - unwise_detid_w1 = TextField(null=True) - unwise_detid_w2 = TextField(null=True) - nm_w1 = IntegerField(null=True) - nm_w2 = IntegerField(null=True) - primary12_w1 = IntegerField(null=True) - primary12_w2 = IntegerField(null=True) - flags_unwise_w1 = IntegerField(null=True) - flags_unwise_w2 = IntegerField(null=True) - flags_info_w1 = IntegerField(null=True) - flags_info_w2 = IntegerField(null=True) - ra = DoubleField(null=True) - dec = DoubleField(null=True) - primary_status = IntegerField(null=True) - class Meta: table_name = 'unwise' schema = 'catalogdb' + use_reflection = True class LegacySurveyDR8(SDSS5dbModel): - lsid = BigIntegerField(primary_key=True) - objid = BigIntegerField() - release = IntegerField() - brickid = BigIntegerField() - brickname = TextField(null=True) - type = TextField(null=True) - ra = DoubleField(null=True) - dec = DoubleField(null=True) - ra_ivar = FloatField(null=True) - dec_ivar = FloatField(null=True) - dchisq = ArrayField(field_class=FloatField, null=True) - ebv = FloatField(null=True) - flux_g = FloatField(null=True) - flux_r = FloatField(null=True) - flux_z = FloatField(null=True) - flux_w1 = FloatField(null=True) - flux_w2 = FloatField(null=True) - flux_w3 = FloatField(null=True) - flux_w4 = FloatField(null=True) - flux_ivar_g = FloatField(null=True) - flux_ivar_r = FloatField(null=True) - flux_ivar_z = FloatField(null=True) - flux_ivar_w1 = FloatField(null=True) - flux_ivar_w2 = FloatField(null=True) - flux_ivar_w3 = FloatField(null=True) - flux_ivar_w4 = FloatField(null=True) - mw_transmission_g = FloatField(null=True) - mw_transmission_r = FloatField(null=True) - mw_transmission_z = FloatField(null=True) - mw_transmission_w1 = FloatField(null=True) - mw_transmission_w2 = FloatField(null=True) - mw_transmission_w3 = FloatField(null=True) - mw_transmission_w4 = FloatField(null=True) - nobs_g = IntegerField(null=True) - nobs_r = IntegerField(null=True) - nobs_z = IntegerField(null=True) - nobs_w1 = IntegerField(null=True) - nobs_w2 = IntegerField(null=True) - nobs_w3 = IntegerField(null=True) - nobs_w4 = IntegerField(null=True) - rchisq_g = FloatField(null=True) - rchisq_r = FloatField(null=True) - rchisq_z = FloatField(null=True) - rchisq_w1 = FloatField(null=True) - rchisq_w2 = FloatField(null=True) - rchisq_w3 = FloatField(null=True) - rchisq_w4 = FloatField(null=True) - fracflux_g = FloatField(null=True) - fracflux_r = FloatField(null=True) - fracflux_z = FloatField(null=True) - fracflux_w1 = FloatField(null=True) - fracflux_w2 = FloatField(null=True) - fracflux_w3 = FloatField(null=True) - fracflux_w4 = FloatField(null=True) - fracmasked_g = FloatField(null=True) - fracmasked_r = FloatField(null=True) - fracmasked_z = FloatField(null=True) - fracin_g = FloatField(null=True) - fracin_r = FloatField(null=True) - fracin_z = FloatField(null=True) - anymask_g = IntegerField(null=True) - anymask_r = IntegerField(null=True) - anymask_z = IntegerField(null=True) - allmask_g = IntegerField(null=True) - allmask_r = IntegerField(null=True) - allmask_z = IntegerField(null=True) - wisemask_w1 = SmallIntegerField(null=True) - wisemask_w2 = SmallIntegerField(null=True) - psfsize_g = FloatField(null=True) - psfsize_r = FloatField(null=True) - psfsize_z = FloatField(null=True) - psfdepth_g = FloatField(null=True) - psfdepth_r = FloatField(null=True) - psfdepth_z = FloatField(null=True) - galdepth_g = FloatField(null=True) - galdepth_r = FloatField(null=True) - galdepth_z = FloatField(null=True) - psfdepth_w1 = FloatField(null=True) - psfdepth_w2 = FloatField(null=True) - wise_coadd_id = TextField(null=True) - fracdev = FloatField(null=True) - fracdev_ivar = FloatField(null=True) - shapedev_r = FloatField(null=True) - shapedev_r_ivar = FloatField(null=True) - shapedev_e1 = FloatField(null=True) - shapedev_e1_ivar = FloatField(null=True) - shapedev_e2 = FloatField(null=True) - shapedev_e2_ivar = FloatField(null=True) - shapeexp_r = FloatField(null=True) - shapeexp_r_ivar = FloatField(null=True) - shapeexp_e1 = FloatField(null=True) - shapeexp_e1_ivar = FloatField(null=True) - shapeexp_e2 = FloatField(null=True) - shapeexp_e2_ivar = FloatField(null=True) - fiberflux_g = FloatField(null=True) - fiberflux_r = FloatField(null=True) - fiberflux_z = FloatField(null=True) - fibertotflux_g = FloatField(null=True) - fibertotflux_r = FloatField(null=True) - fibertotflux_z = FloatField(null=True) - ref_cat = TextField(null=True) - ref_id = BigIntegerField(null=True) - ref_epoch = FloatField(null=True) - gaia_phot_g_mean_mag = FloatField(null=True) - gaia_phot_g_mean_flux_over_error = FloatField(null=True) - gaia_phot_bp_mean_mag = FloatField(null=True) - gaia_phot_bp_mean_flux_over_error = FloatField(null=True) - gaia_phot_rp_mean_mag = FloatField(null=True) - gaia_phot_rp_mean_flux_over_error = FloatField(null=True) - gaia_astrometric_excess_noise = FloatField(null=True) - gaia_duplicated_source = BooleanField(null=True) - gaia_phot_bp_rp_excess_factor = FloatField(null=True) - gaia_astrometric_sigma5d_max = FloatField(null=True) - gaia_astrometric_params_solved = SmallIntegerField(null=True) - parallax = FloatField(null=True) - parallax_ivar = FloatField(null=True) - pmra = FloatField(null=True) - pmra_ivar = FloatField(null=True) - pmdec = FloatField(null=True) - pmdec_ivar = FloatField(null=True) - maskbits = IntegerField(null=True) - class Meta: table_name = 'legacy_survey_dr8' schema = 'catalogdb' + use_reflection = True class GaiaUnwiseAgn(SDSS5dbModel): - gaia_sourceid = BigAutoField(primary_key=True) - ra = DoubleField(null=True) - dec = DoubleField(null=True) - plx = DoubleField(null=True) - plx_err = DoubleField(null=True) - pmra = DoubleField(null=True) - pmra_err = DoubleField(null=True) - pmdec = DoubleField(null=True) - pmdec_err = DoubleField(null=True) - plxsig = DoubleField(null=True) - pmsig = DoubleField(null=True) - ebv = DoubleField(null=True) - n_obs = IntegerField(null=True) - g = DoubleField(null=True) - bp = DoubleField(null=True) - rp = DoubleField(null=True) - w1 = DoubleField(null=True) - w2 = DoubleField(null=True) - bp_g = DoubleField(null=True) - bp_rp = DoubleField(null=True) - g_rp = DoubleField(null=True) - g_w1 = DoubleField(null=True) - gw_sep = DoubleField(null=True) - w1_w2 = DoubleField(null=True) - g_var = DoubleField(null=True) - bprp_ef = DoubleField(null=True) - aen = DoubleField(null=True) - gof = DoubleField(null=True) - cnt1 = IntegerField(null=True) - cnt2 = IntegerField(null=True) - cnt4 = IntegerField(null=True) - cnt8 = IntegerField(null=True) - cnt16 = IntegerField(null=True) - cnt32 = IntegerField(null=True) - phot_z = DoubleField(null=True) - prob_rf = DoubleField(null=True) - unwise = ForeignKeyField(unWISE, column_name='unwise_objid') gaia = ForeignKeyField(GaiaDR2Source) class Meta: table_name = 'gaia_unwise_agn' schema = 'catalogdb' + use_reflection = True class EbosstargetV5(SDSS5dbModel): - run = IntegerField(null=True) - camcol = IntegerField(null=True) - field = IntegerField(null=True) - id = IntegerField(null=True) - rerun = TextField(null=True) - fibermag = ArrayField(field_class=FloatField, null=True) - fiber2mag = ArrayField(field_class=FloatField, null=True) - calib_status = ArrayField(field_class=IntegerField, null=True) - ra = DoubleField(null=True) - dec = DoubleField(null=True) - epoch = FloatField(null=True) - pmra = FloatField(null=True) - pmdec = FloatField(null=True) - eboss_target1 = BigIntegerField(null=True) - eboss_target2 = BigIntegerField(null=True) - eboss_target_id = BigIntegerField(null=True) - thing_id_targeting = IntegerField(null=True) - objc_type = IntegerField(null=True) - objc_flags = IntegerField(null=True) - objc_flags2 = IntegerField(null=True) - flags = IntegerField(null=True) - flags2 = IntegerField(null=True) - psf_fwhm = ArrayField(field_class=FloatField, null=True) - psfflux = ArrayField(field_class=FloatField, null=True) - psfflux_ivar = ArrayField(field_class=FloatField, null=True) - extinction = ArrayField(field_class=FloatField, null=True) - fiberflux = ArrayField(field_class=FloatField, null=True) - fiberflux_ivar = ArrayField(field_class=FloatField, null=True) - fiber2flux = ArrayField(field_class=FloatField, null=True) - fiber2flux_ivar = ArrayField(field_class=FloatField, null=True) - modelflux = ArrayField(field_class=FloatField, null=True) - modelflux_ivar = ArrayField(field_class=FloatField, null=True) - modelmag = ArrayField(field_class=FloatField, null=True) - modelmag_ivar = ArrayField(field_class=FloatField, null=True) - resolve_status = IntegerField(null=True) - w1_mag = FloatField(null=True) - w1_mag_err = FloatField(null=True) - w1_nanomaggies = FloatField(null=True) - w1_nanomaggies_ivar = FloatField(null=True) - w2_nanomaggies = FloatField(null=True) - w2_nanomaggies_ivar = FloatField(null=True) - has_wise_phot = BooleanField(null=True) - objid_targeting = BigAutoField(primary_key=True) - class Meta: table_name = 'ebosstarget_v5' schema = 'catalogdb' + use_reflection = True # The following model (BhmSpidersGenericSuperset) does not need to be represented @@ -2179,6 +269,8 @@ class Meta: class BhmSpidersGenericSuperset(SDSS5dbModel): + # Not using reflection here to preserve Tom's notes. + pk = BigAutoField() # Parameters derived from eROSITA eSASS catalogue @@ -2257,7 +349,7 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): # erosita source, otherwise is optional des_dr1_coadd_object_id = BigIntegerField(index=True, null=True) - # Corresponds to the SDSS DR16 photoObj [[https://www.sdss.org/dr13/help/glossary/#ObjID]] + # Corresponds to the SDSS DR16 photoObj https://www.sdss.org/dr13/help/glossary/#ObjID # - must be used when SDSS photoObj was the primary source of counterpart # to an erosita source, otherwise is optional sdss_dr16_objid = BigIntegerField(index=True, null=True) @@ -2304,52 +396,14 @@ class Meta: class BhmCsc(SDSS5dbModel): - pk = BigAutoField() - - csc_version = TextField(null=True) - cxo_name = TextField(null=True) - - oir_ra = DoubleField(null=True) - oir_dec = DoubleField(null=True) - - mag_g = FloatField(null=True) - mag_r = FloatField(null=True) - mag_i = FloatField(null=True) - mag_z = FloatField(null=True) - mag_H = FloatField(null=True) - - spectrograph = CharField(index=True, null=True, max_length=6) - class Meta: table_name = 'bhm_csc' schema = 'catalogdb' + use_reflection = True class Gaia_DR2_WD_SDSS(SDSS5dbModel): - pk = AutoField(primary_key=True) - wd = TextField(null=True) - n_wd = TextField(null=True) - source = IntegerField(null=True) - sdss = TextField(null=True) - ra = DoubleField(null=True) - dec = DoubleField(null=True) - umag = FloatField(null=True) - e_umag = FloatField(null=True) - gmag = FloatField(null=True) - e_gmag = FloatField(null=True) - rmag = FloatField(null=True) - e_rmag = FloatField(null=True) - imag = FloatField(null=True) - e_imag = FloatField(null=True) - zmag = FloatField(null=True) - e_zmag = FloatField(null=True) - plate = IntegerField(null=True) - mjd = IntegerField(null=True) - fiber = IntegerField(null=True) - snr = FloatField(null=True) - spclass = TextField(null=True) - @property def specobj(self): """Returns the matching record in `.SDSSDR16SpecObj`.""" @@ -2361,68 +415,11 @@ def specobj(self): class Meta: table_name = 'gaia_dr2_wd_sdss' schema = 'catalogdb' + use_reflection = True class Gaia_DR2_WD(SDSS5dbModel): - wd = TextField(null=True) - dr2name = TextField(null=True) - source_id = BigAutoField(primary_key=True) - source = IntegerField(null=True) - ra = DoubleField(null=True) - e_ra = DoubleField(null=True) - dec = DoubleField(null=True) - e_dec = DoubleField(null=True) - plx = FloatField(null=True) - e_plx = FloatField(null=True) - pmra = DoubleField(null=True) - e_pmra = DoubleField(null=True) - pmdec = DoubleField(null=True) - e_pmdec = DoubleField(null=True) - epsi = FloatField(null=True) - amax = FloatField(null=True) - fg_gaia = FloatField(null=True) - e_fg_gaia = FloatField(null=True) - g_gaia_mag = FloatField(null=True) - fbp = FloatField(null=True) - e_fbp = FloatField(null=True) - bpmag = FloatField(null=True) - frp = FloatField(null=True) - e_frp = FloatField(null=True) - rpmag = FloatField(null=True) - e_br_rp = FloatField(null=True) - glon = DoubleField(null=True) - glat = DoubleField(null=True) - density = FloatField(null=True) - ag = FloatField(null=True) - sdss = TextField(null=True) - umag = FloatField(null=True) - e_umag = FloatField(null=True) - gmag = FloatField(null=True) - e_gmag = FloatField(null=True) - rmag = FloatField(null=True) - e_rmag = FloatField(null=True) - imag = FloatField(null=True) - e_imag = FloatField(null=True) - zmag = FloatField(null=True) - e_zmag = FloatField(null=True) - pwd = FloatField(null=True) - f_pwd = IntegerField(null=True) - teffh = FloatField(null=True) - e_teffh = FloatField(null=True) - loggh = FloatField(null=True) - e_loggh = FloatField(null=True) - massh = FloatField(null=True) - e_massh = FloatField(null=True) - chi2h = FloatField(null=True) - teffhe = FloatField(null=True) - e_teffhe = FloatField(null=True) - logghe = FloatField(null=True) - e_logghe = FloatField(null=True) - masshe = FloatField(null=True) - e_masshe = FloatField(null=True) - chisqhe = FloatField(null=True) - @property def sdss_spectra(self): """Returns records from `.Gaia_DR2_WD_SDSS` with matching ``wd``.""" @@ -2432,6 +429,7 @@ def sdss_spectra(self): class Meta: table_name = 'gaia_dr2_wd' schema = 'catalogdb' + use_reflection = True class Tycho2(SDSS5dbModel): From 855b087e8e05cff0bd648042439ab15d01ab11ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 15:06:32 -0700 Subject: [PATCH 090/197] Complete missing load info for TIC v8 --- schema/sdss5db/catalogdb/tic/v8/tic_v8.sql | 96 ++++++++++++++++++- .../sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 19 ++++ schema/sdss5db/catalogdb/tic/v8/tic_v8_load | 4 + 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql create mode 100644 schema/sdss5db/catalogdb/tic/v8/tic_v8_load diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql index 2a94772b..84fa2478 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql @@ -1,7 +1,99 @@ /* -schema for TESS Input Catalog: +schema for tess: -https://outerspace.stsci.edu/display/TESS/TIC+v8+and+CTL+v8.xx+Data+Release+Notes +/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tic/v8/tic_column_description.txt */ + +CREATE TABLE catalogdb.tic_v8 ( + ID bigint, + version text, + HIP integer, + TYC text, + UCAC text, + TWOMASS text, + SDSS bigint, + ALLWISE text, + GAIA text, + APASS text, + KIC integer, + objType text, + typeSrc text, + ra double precision, + dec double precision, + POSflag text, + pmRA real, + e_pmRA real, + pmDEC real, + e_pmDEC real, + PMflag text, + plx real, + e_plx real, + PARflag text, + gallong double precision, + gallat double precision, + eclong double precision, + eclat double precision, + Bmag real, + e_Bmag real, + Vmag real, + e_Vmag real, + umag real, + e_umag real, + gmag real, + e_gmag real, + rmag real, + e_rmag real, + imag real, + e_imag real, + zmag real, + e_zmag real, + Jmag real, + e_Jmag real, + Hmag real, + e_Hmag real, + Kmag real, + e_Kmag real, + TWOMflag text, + prox real, + w1mag real, + e_w1mag real, + w2mag real, + e_w2mag real, + w3mag real, + e_w3mag real, + w4mag real, + e_w4mag real, + GAIAmag real, + e_GAIAmag real, + Tmag real, + e_Tmag real, + TESSflag text, + SPFlag text, + Teff real, + e_Teff real, + logg real, + e_logg real, + MH real, + e_MH real, + rad real, + e_rad real, + mass real, + e_mass real, + rho real, + e_rho real, + lumclass text, + lum real, + e_lum real, + d real, + e_d real, + ebv real, + e_ebv real, + numcont integer, + contratio real, + disposition text, + duplicate_i bigint, + priority real, + objID bigint +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql new file mode 100644 index 00000000..6a15ec0d --- /dev/null +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -0,0 +1,19 @@ + +-- PK +ALTER TABLE catalogdb.dtic_v8 ADD PRIMARY KEY (id); + +-- Indices +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gallong); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gallat); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Bmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Vmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Kmag); diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_load b/schema/sdss5db/catalogdb/tic/v8/tic_v8_load new file mode 100644 index 00000000..9c16d9b6 --- /dev/null +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_load @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +ls /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tic/v8/csv/*.csv.gz | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.tic_v8 FROM program 'zcat {}' WITH CSV;\"" From 7682662de6231ac323ce9dbea04f4144bd480d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 16:11:12 -0600 Subject: [PATCH 091/197] Fix typo --- .../sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index 6a15ec0d..9c1bb3ed 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -1,19 +1,22 @@ -- PK -ALTER TABLE catalogdb.dtic_v8 ADD PRIMARY KEY (id); +ALTER TABLE catalogdb.tic_v8 ADD PRIMARY KEY (id); -- Indices -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gallong); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gallat); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Bmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Vmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (umag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (gmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (rmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (imag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (zmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Jmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Hmag); -CREATE INDEX CONCURRENTLY ON catalogdb.dtic_v8 USING BTREE (Kmag); + +CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.tic_v8; +ANALYZE catalogdb.tic_v8; + +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gallong); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gallat); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Bmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Vmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Kmag); From 3e37c2fbc60899cf701b247178088c34efee9e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:49:54 -0600 Subject: [PATCH 092/197] Add pk to dr14q --- schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql diff --git a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql b/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql new file mode 100644 index 00000000..707d367d --- /dev/null +++ b/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql @@ -0,0 +1,2 @@ + +ALTER TABLE catalogdb.dr14q_v4_4 ADD PRIMARY KEY (pk); From 99c1165a324ad834bdc79dcf06c8b7a95b82ccac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:50:52 -0600 Subject: [PATCH 093/197] Use regex to compare domain --- python/sdssdb/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 30219a77..45a32edc 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -14,12 +14,13 @@ import abc import importlib +import re import socket import six from pgpasslib import getpass -from sdssdb import config, log, _peewee, _sqla +from sdssdb import _peewee, _sqla, config, log if _peewee: @@ -117,7 +118,7 @@ def set_profile(self, profile=None, connect=True): # Tries to find a profile whose domain matches the hostname for profile in config: if 'domain' in config[profile] and config[profile]['domain'] is not None: - if hostname.endswith(config[profile]['domain']): + if re.match(config[profile]['domain'], hostname): self.profile = profile break From 6da547d6c29acebf34bca7e18f694bbf8c688c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:51:03 -0600 Subject: [PATCH 094/197] Fix call to log.warning --- python/sdssdb/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 45a32edc..e082230e 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -347,7 +347,7 @@ def _conn(self, dbname, silent_on_fail=False, **params): self.dbname = dbname except OperationalError: if not silent_on_fail: - log.warning(f'failed to connect to database {self.database!r}.', UserWarning) + log.warning(f'failed to connect to database {self.database!r}.') PostgresqlDatabase.init(self, None) self.connected = False From 4009665b97c5502f1aa5ba81da4d588da4e8128b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:55:57 -0600 Subject: [PATCH 095/197] Ignore connection params that are null --- python/sdssdb/connection.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index e082230e..2a8e8f4d 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -173,19 +173,10 @@ def connect(self, dbname=None, silent_on_fail=False, **connection_params): # Gets the necessary configuration values from the profile db_configuration = {} for item in ['user', 'host', 'port']: - if item in connection_params and connection_params[item] is not None: + if item in connection_params: db_configuration[item] = connection_params[item] else: profile_value = config[self.profile].get(item, None) - - # If the hostname is the same as the current domain, - # do not specify the host. This helps with the configuration - # of the PSQL security at Utah. - if item == 'host': - domain = socket.getfqdn() - if profile_value == domain: - continue - db_configuration[item] = profile_value dbname = dbname or self.dbname From 1817f9c3faf790474a0651ab7f1ca4fc1dda88fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:56:35 -0600 Subject: [PATCH 096/197] Don't do anything with explicitely decalred fks in ReflectMeta --- python/sdssdb/peewee/__init__.py | 49 +++++++++++++++++--------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index df83371a..b7cc75e7 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -62,39 +62,41 @@ class that will affect to all subclasses, except if it's overridden there. updated. Note that this will not work if using Peewee's :class:`peewee:PostgresqlDatabase`. - """ + By default, `.ReflectMeta` will add all the fields from the reflected + models, including foreign keys. Sometimes that is not desirable and it's + preferable to define the foreign keys explicitely. In that case it's + possible to disable the reflection of foreign keys by doing :: - def __new__(cls, name, bases, attrs): + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): - # Remove explicit foreign keys before we do reflection. - fks = [(name, attr) for name, attr in attrs.items() - if isinstance(attr, peewee.ForeignKeyField)] + class Meta: + primary_key = False + use_reflection = False + reflection_options = {'skip_foreign_keys': True} + database = database + + Foreign keys explicitely defined need to reference fields that exist so the + referenced columns need to be added manually. A caveat is that many-to-many + relationship need to be defined explicitely since it's not possible to set + the through model based on the reflected information. + + """ - for name, __ in fks: - attrs.pop(name) + def __new__(cls, name, bases, attrs): Model = super(ReflectMeta, cls).__new__(cls, name, bases, attrs) + Model._meta.use_reflection = getattr(Model._meta, 'use_reflection', False) - Model._meta._fks = fks database = Model._meta.database if database and hasattr(database, 'models') and Model not in database.models: database.models.append(Model) - if Model._meta.use_reflection: - if database and database.connected: - cls.reflect(Model) - else: - cls._attach_fks(Model) + if Model._meta.use_reflection and database and database.connected: + cls.reflect(Model) return Model - def _attach_fks(self): - """Attach the explicit foreign keys.""" - - for name, fk in self._meta._fks: - self._meta.add_field(name, fk) - def reflect(self): """Adds fields and indexes to the model using reflection.""" @@ -111,7 +113,6 @@ def reflect(self): if hasattr(index, 'reflected') and index.reflected: self._meta.indexes.remove(index) - database = self._meta.database if not database.is_connection_usable(): raise peewee.DatabaseError('database not connected.') @@ -131,6 +132,11 @@ def reflect(self): if field_name in self._meta.fields: continue + if (isinstance(field, peewee.ForeignKeyField) and + hasattr(self._meta, 'reflection_options') and + self._meta.reflection_options.get('skip_foreign_keys', False)): + continue + field.reflected = True if field.primary_key: @@ -138,9 +144,6 @@ def reflect(self): else: self._meta.add_field(field_name, field) - # Attach FKs now - self._attach_fks() - for index in ReflectedModel._meta.indexes: # TODO: this probably can never work. Find a better way of determining From 0090b85207a3899a28e3eaaf40d9fa522097356e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Mar 2020 22:56:56 -0600 Subject: [PATCH 097/197] Fix bug in str of fields wihtout primary key --- python/sdssdb/peewee/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index b7cc75e7..238b728f 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -173,8 +173,12 @@ class Meta: def __str__(self): """A custom str for the model repr.""" - pk_field = self._meta.primary_key.name - fields = ['{0}={1!r}'.format(pk_field, self.get_id())] + if self._meta.primary_key: + pk_field = self._meta.primary_key.name + fields = ['{0}={1!r}'.format(pk_field, self.get_id())] + else: + pk_field = None + fields = [] for extra_field in ['label', 'name']: if extra_field not in self.print_fields: From 5d1fc8d3c50adf393bca7b41982534953e63c31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 20 Mar 2020 01:28:36 -0600 Subject: [PATCH 098/197] Implement reflection and fks for catalogdb --- python/sdssdb/etc/sdssdb.yml | 4 +- python/sdssdb/peewee/sdss5db/__init__.py | 7 -- python/sdssdb/peewee/sdss5db/catalogdb.py | 146 +++++++++++----------- 3 files changed, 72 insertions(+), 85 deletions(-) diff --git a/python/sdssdb/etc/sdssdb.yml b/python/sdssdb/etc/sdssdb.yml index c91fe452..a41e5145 100644 --- a/python/sdssdb/etc/sdssdb.yml +++ b/python/sdssdb/etc/sdssdb.yml @@ -36,9 +36,9 @@ sdssadmin: operations: user: sdss - host: operations.sdss.org + host: null port: 5432 - domain: operations.sdss.org + domain: operations.sdss.* manga: user: sdss diff --git a/python/sdssdb/peewee/sdss5db/__init__.py b/python/sdssdb/peewee/sdss5db/__init__.py index 207a319a..8c2d058d 100644 --- a/python/sdssdb/peewee/sdss5db/__init__.py +++ b/python/sdssdb/peewee/sdss5db/__init__.py @@ -9,10 +9,3 @@ class SDSS5dbDatabaseConnection(PeeweeDatabaseConnection): database = SDSS5dbDatabaseConnection(autoconnect=True) - - -# Create a new base model class for the observatory and bind the database -class SDSS5dbModel(BaseModel): - - class Meta: - database = database diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 80c00eb4..a0461ebc 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -14,45 +14,54 @@ DeferredThroughModel, DoubleField, FloatField, ForeignKeyField, IntegerField, ManyToManyField, TextField) -from . import SDSS5dbModel, database # noqa +from . import BaseModel, database # noqa + + +class CatalogdbModel(BaseModel): + + class Meta: + database = database + use_reflection = True + reflection_options = {'skip_foreign_keys': True} + primary_key = False _GaiaDR2TmassBestNeighbourDeferred = DeferredThroughModel() -class AllWise(SDSS5dbModel): +class AllWise(CatalogdbModel): class Meta: table_name = 'allwise' schema = 'catalogdb' - use_reflection = True -class ErositaAGNMock(SDSS5dbModel): +class ErositaAGNMock(CatalogdbModel): class Meta: table_name = 'erosita_agn_mock' schema = 'catalogdb' - use_reflection = True -class ErositaClustersMock(SDSS5dbModel): +class ErositaClustersMock(CatalogdbModel): class Meta: table_name = 'erosita_clusters_mock' schema = 'catalogdb' - use_reflection = True -class TwoMassPsc(SDSS5dbModel): +class TwoMassPsc(CatalogdbModel): + + pts_key = IntegerField(primary_key=True) class Meta: table_name = 'twomass_psc' schema = 'catalogdb' - use_reflection = True -class GaiaDR2Source(SDSS5dbModel): +class GaiaDR2Source(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) tmass_best_sources = ManyToManyField(TwoMassPsc, through_model=_GaiaDR2TmassBestNeighbourDeferred, @@ -61,113 +70,102 @@ class GaiaDR2Source(SDSS5dbModel): class Meta: table_name = 'gaia_dr2_source' schema = 'catalogdb' - use_reflection = True -class GaiaDR2Clean(SDSS5dbModel): +class GaiaDR2Clean(CatalogdbModel): - source = ForeignKeyField(GaiaDR2Source, column_name='source_id', backref='gaia_clean') + source_id = ForeignKeyField(GaiaDR2Source, + field='source_id', + backref='gaia_clean', + lazy_load=True) class Meta: table_name = 'gaia_dr2_clean' schema = 'catalogdb' - use_reflection = True -class GaiaDR2SDSSDR9BestNeighbour(SDSS5dbModel): +class GaiaDR2SDSSDR9BestNeighbour(CatalogdbModel): class Meta: table_name = 'gaiadr2_sdssdr9_best_neighbour' schema = 'catalogdb' - use_reflection = True -class GalacticGenesis(SDSS5dbModel): +class GalacticGenesis(CatalogdbModel): class Meta: table_name = 'galactic_genesis' schema = 'catalogdb' - use_reflection = True -class GalacticGenesisBig(SDSS5dbModel): +class GalacticGenesisBig(CatalogdbModel): class Meta: table_name = 'galactic_genesis_big' schema = 'catalogdb' - use_reflection = True -class GUVCat(SDSS5dbModel): +class GUVCat(CatalogdbModel): class Meta: table_name = 'guvcat' schema = 'catalogdb' - use_reflection = True -class KeplerInput10(SDSS5dbModel): +class KeplerInput10(CatalogdbModel): class Meta: table_name = 'kepler_input_10' schema = 'catalogdb' - use_reflection = True -class SDSSDR13Photoobj(SDSS5dbModel): +class SDSSDR13Photoobj(CatalogdbModel): class Meta: table_name = 'sdss_dr13_photoobj' schema = 'catalogdb' - use_reflection = True -class SDSSDR14APOGEEStar(SDSS5dbModel): +class SDSSDR14APOGEEStar(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeestar' schema = 'catalogdb' - use_reflection = True -class SDSSDR14APOGEEStarVisit(SDSS5dbModel): +class SDSSDR14APOGEEStarVisit(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeestarvisit' schema = 'catalogdb' - use_reflection = True -class SDSSDR14APOGEEVisit(SDSS5dbModel): +class SDSSDR14APOGEEVisit(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeevisit' schema = 'catalogdb' - use_reflection = True -class SDSSDR14ASCAPStar(SDSS5dbModel): +class SDSSDR14ASCAPStar(CatalogdbModel): class Meta: table_name = 'sdss_dr14_ascapstar' schema = 'catalogdb' - use_reflection = True -class SDSSDR14CannonStar(SDSS5dbModel): +class SDSSDR14CannonStar(CatalogdbModel): class Meta: table_name = 'sdss_dr14_cannonstar' schema = 'catalogdb' - use_reflection = True -class SDSSDR14SpecObj(SDSS5dbModel): +class SDSSDR14SpecObj(CatalogdbModel): class Meta: table_name = 'sdss_dr14_specobj' schema = 'catalogdb' - use_reflection = True class SDSSDR16SpecObj(SDSSDR14SpecObj): @@ -176,90 +174,89 @@ class Meta: table_name = 'sdss_dr16_specobj' -class TIC_v8(SDSS5dbModel): +class TIC_v8(CatalogdbModel): class Meta: table_name = 'tic_v8' schema = 'catalogdb' - use_reflection = True -class TwoMassClean(SDSS5dbModel): +class TwoMassClean(CatalogdbModel): - psc = ForeignKeyField(TwoMassPsc, field='pts_key', - column_name='pts_key', backref='tmass_clean') + pts_key = ForeignKeyField(TwoMassPsc, + backref='tmass_clean', + lazy_load=True) class Meta: table_name = 'twomass_clean' schema = 'catalogdb' - use_reflection = True -class TwoMassCleanNoNeighbor(SDSS5dbModel): +class TwoMassCleanNoNeighbor(CatalogdbModel): class Meta: table_name = 'twomass_clean_noneighbor' schema = 'catalogdb' - use_reflection = True -class GaiaDR2TmassBestNeighbour(SDSS5dbModel): +class GaiaDR2TmassBestNeighbour(CatalogdbModel): - gaia_source = ForeignKeyField(GaiaDR2Source, - column_name='source_id', - backref='tmass_best_neighbour') - tmass_source = ForeignKeyField(TwoMassPsc, - column_name='tmass_pts_key', - field='pts_key', - backref='gaia_best_neighbour') + source_id = ForeignKeyField(GaiaDR2Source, + field='source_id', + column_name='source_id', + backref='+', + lazy_load=False) + + tmass_pts_key = ForeignKeyField(TwoMassPsc, + field='pts_key', + column_name='tmass_pts_key', + backref='+', + lazy_load=False) class Meta: table_name = 'gaiadr2_tmass_best_neighbour' schema = 'catalogdb' - use_reflection = True -class DR14QV44(SDSS5dbModel): +class DR14QV44(CatalogdbModel): class Meta: table_name = 'dr14q_v4_4' schema = 'catalogdb' - use_reflection = True -class unWISE(SDSS5dbModel): +class unWISE(CatalogdbModel): class Meta: table_name = 'unwise' schema = 'catalogdb' - use_reflection = True -class LegacySurveyDR8(SDSS5dbModel): +class LegacySurveyDR8(CatalogdbModel): class Meta: table_name = 'legacy_survey_dr8' schema = 'catalogdb' - use_reflection = True -class GaiaUnwiseAgn(SDSS5dbModel): +class GaiaUnwiseAgn(CatalogdbModel): - unwise = ForeignKeyField(unWISE, column_name='unwise_objid') - gaia = ForeignKeyField(GaiaDR2Source) + unwise_objid = ForeignKeyField(unWISE, + lazy_load=True) + + gaia = ForeignKeyField(GaiaDR2Source, + lazy_load=True) class Meta: table_name = 'gaia_unwise_agn' schema = 'catalogdb' - use_reflection = True -class EbosstargetV5(SDSS5dbModel): +class EbosstargetV5(CatalogdbModel): class Meta: table_name = 'ebosstarget_v5' schema = 'catalogdb' - use_reflection = True # The following model (BhmSpidersGenericSuperset) does not need to be represented @@ -267,7 +264,7 @@ class Meta: # BhmSpidersClustersSuperset which are real database tables. I am assuming that PeeWee # can handle such a scheme without problems. If not, then we will have to duplicate the -class BhmSpidersGenericSuperset(SDSS5dbModel): +class BhmSpidersGenericSuperset(CatalogdbModel): # Not using reflection here to preserve Tom's notes. @@ -378,6 +375,7 @@ class BhmSpidersGenericSuperset(SDSS5dbModel): class Meta: table_name = 'bhm_spiders_generic_superset' schema = 'catalogdb' + use_reflection = False # Note that following models are currently identical in form, but may well diverge in the future @@ -394,15 +392,14 @@ class Meta: table_name = 'bhm_spiders_clusters_superset' -class BhmCsc(SDSS5dbModel): +class BhmCsc(CatalogdbModel): class Meta: table_name = 'bhm_csc' schema = 'catalogdb' - use_reflection = True -class Gaia_DR2_WD_SDSS(SDSS5dbModel): +class Gaia_DR2_WD_SDSS(CatalogdbModel): @property def specobj(self): @@ -415,10 +412,9 @@ def specobj(self): class Meta: table_name = 'gaia_dr2_wd_sdss' schema = 'catalogdb' - use_reflection = True -class Gaia_DR2_WD(SDSS5dbModel): +class Gaia_DR2_WD(CatalogdbModel): @property def sdss_spectra(self): @@ -429,15 +425,13 @@ def sdss_spectra(self): class Meta: table_name = 'gaia_dr2_wd' schema = 'catalogdb' - use_reflection = True -class Tycho2(SDSS5dbModel): +class Tycho2(CatalogdbModel): class Meta: table_name = 'tycho2' schema = 'catalogdb' - use_reflection = True _GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) From 1cc4d66371d21e1f520a898c6d2254416d9386e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 20 Mar 2020 11:23:34 -0600 Subject: [PATCH 099/197] Fix tycho2 q3c cluster --- schema/sdss5db/catalogdb/tycho2/tycho2_index.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql index 2a308ea8..74c3f3cb 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql @@ -1,4 +1,4 @@ CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); -CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.tycho2; +CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; ANALYZE catalogdb.tycho2; From f2301099c7dbddc84e5cbc96c6bb05a2c9b86dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 20 Mar 2020 11:24:12 -0600 Subject: [PATCH 100/197] Add indexes to tmass_best_neighbour fks --- schema/sdss5db/catalogdb/foreignKeys.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index f4ea8756..aec2988a 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -9,12 +9,6 @@ https://www.postgresql.org/docs/8.2/static/sql-altertable.html */ -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour - ADD CONSTRAINT original_ext_source_id_fk - FOREIGN KEY (original_ext_source_id) - REFERENCES catalogdb.twomass_psc (designation) - ON UPDATE CASCADE ON DELETE CASCADE; - ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) @@ -27,12 +21,16 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (source_id ASC); + ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT tmass_pts_key_fk FOREIGN KEY (tmass_pts_key) REFERENCES catalogdb.twomass_psc (pts_key) ON UPDATE CASCADE ON DELETE CASCADE; +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); + ALTER TABLE catalogdb.sdss_dr14_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) From 3ca8bf6e2f002bbe767cabf245ad35afbd72a6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 20 Mar 2020 11:24:37 -0600 Subject: [PATCH 101/197] Upper case for gaia q3c index --- schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql index 127ee6b5..4c8e82e2 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql @@ -24,6 +24,6 @@ CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (phot_g_mean_ CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (solution_id); -create index on catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); -CLUSTER gaia_dr2_source_q3c_ang2ipix_idx on catalogdb.gaia_dr2_source; -analyze catalogdb.gaia_dr2_source; \ No newline at end of file +CREATE INDEX ON catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_source_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_source; +ANALYZE catalogdb.gaia_dr2_source; From 14d7e2fc19187b281637522ea70ae5498b36076a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 20 Mar 2020 15:02:31 -0600 Subject: [PATCH 102/197] Remove erosita mock --- python/sdssdb/peewee/sdss5db/catalogdb.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index a0461ebc..d23ee2cb 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -36,20 +36,6 @@ class Meta: schema = 'catalogdb' -class ErositaAGNMock(CatalogdbModel): - - class Meta: - table_name = 'erosita_agn_mock' - schema = 'catalogdb' - - -class ErositaClustersMock(CatalogdbModel): - - class Meta: - table_name = 'erosita_clusters_mock' - schema = 'catalogdb' - - class TwoMassPsc(CatalogdbModel): pts_key = IntegerField(primary_key=True) From 8b76ad8c4871eda875db45c663eff0b1cc55d7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 21 Mar 2020 12:26:51 -0600 Subject: [PATCH 103/197] Add Gaia QSO catalogue --- python/sdssdb/peewee/sdss5db/catalogdb.py | 7 ++- .../sdss5db/catalogdb/gaia_qso/gaia_qso.sql | 63 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index d23ee2cb..86cef2f4 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -420,4 +420,9 @@ class Meta: schema = 'catalogdb' -_GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) +class GaiaQSO(CatalogdbModel): + + class Meta: + table_name = 'gaia_qso' + schema = 'catalogdb' + diff --git a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql new file mode 100644 index 00000000..338d9aeb --- /dev/null +++ b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql @@ -0,0 +1,63 @@ +/* + +Title: Compilation of known QSOs for the Gaia mission (Liao+, 2019) +URL: http://vizier.u-strasbg.fr/viz-bin/VizieR?-source=J/other/RAA/19.29 + +The ;-separated file downloaded from Vizier has been modified by removing +all the comments except the header and then opening and saving it with +Pandas using the skipinitialspace=True option. + +Columns: + +Name (a23) Name (1) +RAJ2000 (F15.11) Right ascension (J2000.0) (2) +DEJ2000 (F15.11) Declination (J2000.0) (2) +e_RAJ2000 (F13.11) Error in Right ascension +e_DEJ2000 (F13.11) Error in Declination +z (F13.10) Redshift (3) +umag (F11.8) u photographic magnitude (4) +gmag (F11.8) g photographic magnitude (4) +rmag (F11.8) r photographic magnitude (4) +imag (F11.8) i photographic magnitude (4) +zmag (F11.8) z photographic magnitude (4) +Ref (a15) Reference catalog (5) +W1-W2 (F5.2) W1-W2 photographic magnitude (6) +W2-W3 (F5.2) W2-W3 photographic magnitude (6) +W1mag (F5.2) W1 photographic magnitude (6) + +*/ + +CREATE TABLE catalogdb.gaia_qso ( + name TEXT, + raj2000 DOUBLE PRECISION, + dej2000 DOUBLE PRECISION, + e_raj2000 DOUBLE PRECISION, + e_dej2000 DOUBLE PRECISION, + z DOUBLE PRECISION, + umag REAL, + gmag REAL, + rmag REAL, + imag REAL, + zmag REAL, + ref TEXT, + w1_w2 REAL, + w2_w3 REAL, + w1mag REAL +) WITHOUT OIDS; + +\copy catalogdb.gaia_qso FROM PROGRAM 'cat $CATALOGDB_DIR/gaia_qso/Gaia_QSO_Liao_2019.csv' WITH CSV HEADER; + +ALTER TABLE catalogdb.gaia_qso ADD COLUMN pk BIGSERIAL; + +CREATE INDEX ON catalogdb.gaia_qso (q3c_ang2ipix(raj2000, dej2000)); +CLUSTER gaia_qso_q3c_ang2ipix_idx ON catalogdb.gaia_qso; +ANALYZE catalogdb.gaia_qso; + +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (umag); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (gmag); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (rmag); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (imag); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (zmag); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w1_w2); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w2_w3); +CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w1mag); From d2df81c8bd3c80af6f65facb116a4797527ba7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 22 Mar 2020 17:23:41 -0600 Subject: [PATCH 104/197] Add some caveats to ReflectMeta --- python/sdssdb/peewee/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 238b728f..96c306cc 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -76,9 +76,17 @@ class Meta: database = database Foreign keys explicitely defined need to reference fields that exist so the - referenced columns need to be added manually. A caveat is that many-to-many - relationship need to be defined explicitely since it's not possible to set - the through model based on the reflected information. + referenced columns need to be added manually. + + Caveats: + + - Many-to-many relationships need to be defined explicitely since + it's not possible to set the through model based on the reflected + information. + + - When the primary key of a model is also a foreign key, it needs to be + defined explicitely. Otherwise the field will be added but not marked as + primary key. """ From ef6c8c498322cca338a87fdba2dcf95f9d055bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 22 Mar 2020 18:47:03 -0600 Subject: [PATCH 105/197] Mutiple changes to catalogdb schema and ORM --- python/sdssdb/peewee/sdss5db/catalogdb.py | 196 +++++++++++----- .../Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql | 7 +- .../catalogdb/allwise/source/allwise.sql | 7 +- .../allwise/source/allwise_index.sql | 23 +- .../bhm_csc/v0.1.0/bhm_csc_index.sql | 10 +- .../catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql | 4 + .../eRosita/v0.1.0/eRosita_index.sql | 10 +- .../v5/ebosstarget_v5_post_load.sql | 4 +- schema/sdss5db/catalogdb/foreignKeys.sql | 147 ++++++++++++ .../gaia/dr2/gaia_dr2_source_index.sql | 20 +- .../sdss5db/catalogdb/gaia_qso/gaia_qso.sql | 2 + .../catalogdb/guvcat/fov055/guvcat.sql | 7 +- .../catalogdb/guvcat/fov055/guvcat_index.sql | 8 +- .../catalogdb/kic/v10/kepler_input_10.sql | 2 +- .../kic/v10/kepler_input_10_index.sql | 35 ++- .../legacy_survey/dr8/legacy_survey_index.sql | 3 - .../dr14/sdss_dr14_apogeeStar.sql | 16 +- .../dr14/sdss_dr14_apogeeStarVisit.sql | 7 +- .../dr14/sdss_dr14_apogeeVisit.sql | 18 +- .../dr14/sdss_dr14_ascapStar.sql | 11 +- .../dr14/sdss_dr14_cannonStar.sql | 9 +- .../sdssPhotoObj/dr13/sdss_dr13_photoobj.sql | 7 +- .../dr13/sdss_dr13_photoobj_index.sql | 25 +- .../sdssSpecObj/dr14/sdss_dr14_specObj.sql | 21 +- .../sdssSpecObj/dr16/sdss_dr16_specObj.sql | 29 +-- schema/sdss5db/catalogdb/tic/v8/tic_v8.sql | 214 ++++++++++-------- .../sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 36 +-- .../catalogdb/twomass/psc/twomass_psc.sql | 5 +- .../twomass/psc/twomass_psc_index.sql | 30 ++- schema/sdss5db/catalogdb/tycho2/tycho2.sql | 2 +- .../sdss5db/catalogdb/unWISE/unWISE_index.sql | 4 +- .../sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 14 +- .../gaiaDR2/gaiadr2_tmass_best_neighbour.sql | 7 +- 33 files changed, 596 insertions(+), 344 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 86cef2f4..51cb162e 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -9,7 +9,6 @@ # @Last modified by: José Sánchez-Gallego (gallegoj@uw.edu) # @Last modified time: 2019-09-23 00:54:42 - from peewee import (BigAutoField, BigIntegerField, CharField, DeferredThroughModel, DoubleField, FloatField, ForeignKeyField, IntegerField, ManyToManyField, TextField) @@ -26,31 +25,36 @@ class Meta: primary_key = False -_GaiaDR2TmassBestNeighbourDeferred = DeferredThroughModel() +_Gaia_DR2_TwoMass_Best_Neighbour_Deferred = DeferredThroughModel() +_APOGEE_Star_Visit_Deferred = DeferredThroughModel() class AllWise(CatalogdbModel): + cntr = BigIntegerField(primary_key=True) + designation = TextField() + class Meta: table_name = 'allwise' schema = 'catalogdb' -class TwoMassPsc(CatalogdbModel): +class TwoMassPSC(CatalogdbModel): pts_key = IntegerField(primary_key=True) + designation = TextField() class Meta: table_name = 'twomass_psc' schema = 'catalogdb' -class GaiaDR2Source(CatalogdbModel): +class Gaia_DR2(CatalogdbModel): source_id = BigIntegerField(primary_key=True) - tmass_best_sources = ManyToManyField(TwoMassPsc, - through_model=_GaiaDR2TmassBestNeighbourDeferred, + tmass_best_sources = ManyToManyField(TwoMassPSC, + through_model=_Gaia_DR2_TwoMass_Best_Neighbour_Deferred, backref='gaia_best_sources') class Meta: @@ -58,9 +62,9 @@ class Meta: schema = 'catalogdb' -class GaiaDR2Clean(CatalogdbModel): +class Gaia_DR2_Clean(CatalogdbModel): - source_id = ForeignKeyField(GaiaDR2Source, + source_id = ForeignKeyField(Gaia_DR2, field='source_id', backref='gaia_clean', lazy_load=True) @@ -70,7 +74,7 @@ class Meta: schema = 'catalogdb' -class GaiaDR2SDSSDR9BestNeighbour(CatalogdbModel): +class Gaia_DR2_SDSS_DR9_Best_Neighbour(CatalogdbModel): class Meta: table_name = 'gaiadr2_sdssdr9_best_neighbour' @@ -98,78 +102,114 @@ class Meta: schema = 'catalogdb' -class KeplerInput10(CatalogdbModel): +class KeplerInput_DR10(CatalogdbModel): + + kic_kepler_id = BigIntegerField(primary_key=True) class Meta: table_name = 'kepler_input_10' schema = 'catalogdb' -class SDSSDR13Photoobj(CatalogdbModel): +class SDSS_DR13_PhotoObj(CatalogdbModel): + + objid = BigIntegerField(primary_key=True) class Meta: table_name = 'sdss_dr13_photoobj' schema = 'catalogdb' -class SDSSDR14APOGEEStar(CatalogdbModel): +class SDSS_DR14_APOGEE_Visit(CatalogdbModel): + + visit_id = TextField(primary_key=True) class Meta: - table_name = 'sdss_dr14_apogeestar' + table_name = 'sdss_dr14_apogeevisit' schema = 'catalogdb' -class SDSSDR14APOGEEStarVisit(CatalogdbModel): +class SDSS_DR14_APOGEE_Star(CatalogdbModel): + + apstar_id = TextField(primary_key=True) + apogee_id = TextField() + + visits = ManyToManyField(SDSS_DR14_APOGEE_Visit, + through_model=_APOGEE_Star_Visit_Deferred, + backref='stars') class Meta: - table_name = 'sdss_dr14_apogeestarvisit' + table_name = 'sdss_dr14_APOGEE_Star' schema = 'catalogdb' -class SDSSDR14APOGEEVisit(CatalogdbModel): +class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel): + + apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, + backref='+', + lazy_load=False) + + visit = ForeignKeyField(SDSS_DR14_APOGEE_Visit, + backref='+', + lazy_load=False) class Meta: - table_name = 'sdss_dr14_apogeevisit' + table_name = 'sdss_dr14_APOGEE_Starvisit' schema = 'catalogdb' -class SDSSDR14ASCAPStar(CatalogdbModel): +class SDSS_DR14_ASCAP_Star(CatalogdbModel): + + apstar_id = TextField(primary_key=True) + apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, + backref='ascap_stars') class Meta: table_name = 'sdss_dr14_ascapstar' schema = 'catalogdb' -class SDSSDR14CannonStar(CatalogdbModel): +class SDSS_DR14_Cannon_Star(CatalogdbModel): + + @property + def apstars(self): + """Returns the associated stars in ``SDSSDR14APOGEEStar``.""" + + return (SDSS_DR14_APOGEE_Star + .select() + .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id)) class Meta: table_name = 'sdss_dr14_cannonstar' schema = 'catalogdb' -class SDSSDR14SpecObj(CatalogdbModel): +class SDSS_DR14_SpecObj(CatalogdbModel): + + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='bestobjid', + object_id_name='bestobjid', + backref='specobj_dr14') class Meta: table_name = 'sdss_dr14_specobj' schema = 'catalogdb' -class SDSSDR16SpecObj(SDSSDR14SpecObj): +class SDSS_DR16_SpecObj(SDSS_DR14_SpecObj): - class Meta: - table_name = 'sdss_dr16_specobj' - - -class TIC_v8(CatalogdbModel): + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='bestobjid', + object_id_name='bestobjid', + backref='specobj_dr16') class Meta: - table_name = 'tic_v8' - schema = 'catalogdb' + table_name = 'sdss_dr16_specobj' -class TwoMassClean(CatalogdbModel): +class TwoMass_Clean(CatalogdbModel): - pts_key = ForeignKeyField(TwoMassPsc, + pts_key = ForeignKeyField(TwoMassPSC, backref='tmass_clean', lazy_load=True) @@ -178,22 +218,22 @@ class Meta: schema = 'catalogdb' -class TwoMassCleanNoNeighbor(CatalogdbModel): +class TwoMass_Clean_No_Neighbor(CatalogdbModel): class Meta: table_name = 'twomass_clean_noneighbor' schema = 'catalogdb' -class GaiaDR2TmassBestNeighbour(CatalogdbModel): +class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): - source_id = ForeignKeyField(GaiaDR2Source, + source_id = ForeignKeyField(Gaia_DR2, field='source_id', column_name='source_id', backref='+', lazy_load=False) - tmass_pts_key = ForeignKeyField(TwoMassPsc, + tmass_pts_key = ForeignKeyField(TwoMassPSC, field='pts_key', column_name='tmass_pts_key', backref='+', @@ -204,7 +244,7 @@ class Meta: schema = 'catalogdb' -class DR14QV44(CatalogdbModel): +class DR14_Q_V4_4(CatalogdbModel): class Meta: table_name = 'dr14q_v4_4' @@ -218,27 +258,45 @@ class Meta: schema = 'catalogdb' -class LegacySurveyDR8(CatalogdbModel): +class Legacy_Survey_DR8(CatalogdbModel): + + ref_cat = TextField() + ref_id = BigIntegerField() + + @property + def gaia(self): + """Returns the Gaia DR2 object or `None` if no match.""" + + if self.ref_cat != 'G2': + return None + + return Gaia_DR2.get(source_id=self.ref_id) class Meta: table_name = 'legacy_survey_dr8' schema = 'catalogdb' -class GaiaUnwiseAgn(CatalogdbModel): +class Gaia_unWISE_AGN(CatalogdbModel): - unwise_objid = ForeignKeyField(unWISE, - lazy_load=True) + gaia_sourceid = BigIntegerField(primary_key=True) - gaia = ForeignKeyField(GaiaDR2Source, - lazy_load=True) + unwise = ForeignKeyField(unWISE, field='unwise_objid', + column_name='unwise_objid', + object_id_name='unwise_objid', + backref='+') + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_sourceid', + object_id_name='gaia_sourceid_id', + backref='+') class Meta: table_name = 'gaia_unwise_agn' schema = 'catalogdb' -class EbosstargetV5(CatalogdbModel): +class eBOSS_Taarget_v5(CatalogdbModel): class Meta: table_name = 'ebosstarget_v5' @@ -250,7 +308,7 @@ class Meta: # BhmSpidersClustersSuperset which are real database tables. I am assuming that PeeWee # can handle such a scheme without problems. If not, then we will have to duplicate the -class BhmSpidersGenericSuperset(CatalogdbModel): +class BHM_Spiders_Generic_Superset(CatalogdbModel): # Not using reflection here to preserve Tom's notes. @@ -366,19 +424,19 @@ class Meta: # Note that following models are currently identical in form, but may well diverge in the future -class BhmSpidersAgnSuperset(BhmSpidersGenericSuperset): +class BHM_Spiders_AGN_Superset(BHM_Spiders_Generic_Superset): class Meta: table_name = 'bhm_spiders_agn_superset' -class BhmSpidersClustersSuperset(BhmSpidersGenericSuperset): +class BHM_Spiders_Clusters_Superset(BHM_Spiders_Generic_Superset): class Meta: table_name = 'bhm_spiders_clusters_superset' -class BhmCsc(CatalogdbModel): +class BHM_CSC(CatalogdbModel): class Meta: table_name = 'bhm_csc' @@ -389,11 +447,11 @@ class Gaia_DR2_WD_SDSS(CatalogdbModel): @property def specobj(self): - """Returns the matching record in `.SDSSDR16SpecObj`.""" + """Returns the matching record in `.SDSS_DR16_SpecObj`.""" - return SDSSDR16SpecObj.get(SDSSDR16SpecObj.plate == self.plate, - SDSSDR16SpecObj.mjd == self.mjd, - SDSSDR16SpecObj.fiberid == self.fiber) + return SDSS_DR16_SpecObj.get(SDSS_DR16_SpecObj.plate == self.plate, + SDSS_DR16_SpecObj.mjd == self.mjd, + SDSS_DR16_SpecObj.fiberid == self.fiber) class Meta: table_name = 'gaia_dr2_wd_sdss' @@ -403,7 +461,7 @@ class Meta: class Gaia_DR2_WD(CatalogdbModel): @property - def sdss_spectra(self): + def sdss(self): """Returns records from `.Gaia_DR2_WD_SDSS` with matching ``wd``.""" return Gaia_DR2_WD_SDSS.select().where(Gaia_DR2_WD_SDSS.wd == self.wd) @@ -415,6 +473,8 @@ class Meta: class Tycho2(CatalogdbModel): + name = TextField() + class Meta: table_name = 'tycho2' schema = 'catalogdb' @@ -426,3 +486,37 @@ class Meta: table_name = 'gaia_qso' schema = 'catalogdb' + +class TIC_v8(CatalogdbModel): + + tycho2 = ForeignKeyField(Tycho2, field='name', + column_name='tyc', object_id_name='tyc', + backref='tic') + + twomass_psc = ForeignKeyField(TwoMassPSC, field='designation', + column_name='twomass', object_id_name='twomass', + backref='tic') + + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, field='objid', + column_name='sdss_int', object_id_name='sdss_int', + backref='tic') + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_int', object_id_name='gaia_int', + backref='tic') + + allwise_target = ForeignKeyField(AllWise, field='designation', + column_name='allwise', object_id_name='allwise', + backref='tic') + + kepler_input = ForeignKeyField(KeplerInput_DR10, field='kic_kepler_id', + column_name='kic', object_id_name='kic', + backref='tic') + + class Meta: + table_name = 'tic_v8' + schema = 'catalogdb' + + +_Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) +_APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql index 5c63eccc..4a001b90 100644 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql @@ -1,4 +1,7 @@ -CREATE INDEX on catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); -CLUSTER gaia_unwise_agn_q3c_ang2ipix_idx on catalogdb.gaia_unwise_agn; +CREATE INDEX ON catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_unwise_agn_q3c_ang2ipix_idx ON catalogdb.gaia_unwise_agn; ANALYZE catalogdb.gaia_unwise_agn; + +CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (g); +CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (prob_rf); diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise.sql b/schema/sdss5db/catalogdb/allwise/source/allwise.sql index 0b6c7f6f..81b1a0d7 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise.sql @@ -20,7 +20,7 @@ CREATE TABLE catalogdb.allwise( elat decimal(9,7) , wx decimal(7,3) , wy decimal(7,3) , - cntr bigint , + cntr bigint PRIMARY KEY, source_id char(28) , coadd_id char(20) , src integer , @@ -307,8 +307,3 @@ CREATE TABLE catalogdb.allwise( spt_ind integer , htm20 bigint ); - - - - - diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql index ad46dafe..a595a220 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql @@ -8,17 +8,22 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; */ +-- Unique identification number for this object in the AllWISE Catalog/Reject Table. +-- This number is formed from the source_id, which is in turn formed from the +-- coadd_id and source number, src. + +ALTER TABLE catalogdb.allwise ADD PRIMARY KEY (cntr); --- Indices -alter table catalogdb.allwise add primary key(designation); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glat); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glon); +-- Indices -create index on catalogdb.allwise (q3c_ang2ipix(ra, dec)); -CLUSTER allwise_q3c_ang2ipix_idx on catalogdb.allwise; -analyze catalogdb.allwise; +CREATE INDEX ON catalogdb.allwise using BTREE (designation); +CREATE INDEX ON catalogdb.allwise using BTREE (ra); +CREATE INDEX ON catalogdb.allwise using BTREE (dec); +CREATE INDEX ON catalogdb.allwise using BTREE (glat); +CREATE INDEX ON catalogdb.allwise using BTREE (glon); +CREATE INDEX ON catalogdb.allwise (q3c_ang2ipix(ra, dec)); +CLUSTER allwise_q3c_ang2ipix_idx ON catalogdb.allwise; +ANALYZE catalogdb.allwise; diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql index 2212c251..68b0b63c 100644 --- a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql @@ -3,8 +3,8 @@ CREATE INDEX on catalogdb.bhm_csc (q3c_ang2ipix(oir_ra, oir_dec)); CLUSTER bhm_csc_q3c_ang2ipix_idx on catalogdb.bhm_csc; ANALYZE catalogdb.bhm_csc; -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_g); -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_r); -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_i); -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_z); -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc using BTREE (mag_h); +CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_g); +CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_r); +CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_i); +CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_z); +CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_h); diff --git a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql b/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql index 707d367d..111a26d1 100644 --- a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql +++ b/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql @@ -1,2 +1,6 @@ ALTER TABLE catalogdb.dr14q_v4_4 ADD PRIMARY KEY (pk); + +CREATE INDEX ON catalogdb.dr14q_v4_4 (q3c_ang2ipix(ra, dec)); +CLUSTER dr14q_v4_4_q3c_ang2ipix_idx ON catalogdb.dr14q_v4_4; +ANALYZE catalogdb.dr14q_v4_4; diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql index 87676fb9..a690f015 100644 --- a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql @@ -1,13 +1,13 @@ CREATE INDEX on catalogdb.bhm_spiders_agn_superset (q3c_ang2ipix(ero_ra, ero_dec)); -CLUSTER bhm_spiders_agn_superset_q3c_ang2ipix_idx on catalogdb.bhm_spiders_agn_superset; +CLUSTER bhm_spiders_agn_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_agn_superset; ANALYZE catalogdb.bhm_spiders_agn_superset; -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset using BTREE (ero_flux); +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ero_flux); -CREATE INDEX on catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); -CLUSTER bhm_spiders_clusters_superset_q3c_ang2ipix_idx on catalogdb.bhm_spiders_clusters_superset; +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CLUSTER bhm_spiders_clusters_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_clusters_superset; ANALYZE catalogdb.bhm_spiders_clusters_superset; -CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset using BTREE (ero_flux); +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ero_flux); diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql index 327dc0db..9835899f 100644 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql @@ -4,12 +4,10 @@ To be run after the data has been loaded. */ - ALTER TABLE catalogdb.ebosstarget_v5 ADD COLUMN pk BIGSERIAL PRIMARY KEY; CREATE INDEX on catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; ANALYZE catalogdb.ebosstarget_v5; -CREATE INDEX CONCURRENTLY ebosstarget_v5_objc_type_idx - ON catalogdb.ebosstarget_v5 using BTREE (objc_type); +CREATE INDEX ON catalogdb.ebosstarget_v5 USING BTREE (objc_type); diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index aec2988a..aba32cd6 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -9,12 +9,17 @@ https://www.postgresql.org/docs/8.2/static/sql-altertable.html */ +-- gaia_dr2_wd_candidates_v1 + ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; + +-- gaiadr2_tmass_best_neighbour + ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) @@ -31,8 +36,150 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); + +-- sdss_dr14_specobj + ALTER TABLE catalogdb.sdss_dr14_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); + +-- sdss_dr16_specobj + +ALTER TABLE catalogdb.sdss_dr16_specobj + ADD CONSTRAINT bestobjid_fk + FOREIGN KEY (bestobjid) + REFERENCES catalogdb.sdss_dr13_photoobj (objid) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); + + +-- gaia_unwise_agn + +ALTER TABLE catalogdb.gaia_unwise_agn + ADD CONSTRAINT source_id_fk + FOREIGN KEY (gaia_sourceid) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaia_unwise_agn + ADD CONSTRAINT unwise_objid_fk + FOREIGN KEY (unwise_objid) + REFERENCES catalogdb.unwise (unwise_objid) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); + + +-- sdss_dr14_apogeeStarVisit + +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit + ADD CONSTRAINT visit_id_fk + FOREIGN KEY (visit_id) + REFERENCES catalogdb.sdss_dr14_apogeeVisit (visit_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit + ADD CONSTRAINT apstar_id_fk + FOREIGN KEY (apstar_id) + REFERENCES catalogdb.sdss_dr14_apogeeStar (apstar_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- sdss_dr14_ascapStar + +ALTER TABLE catalogdb.sdss_dr14_ascapStar + ADD CONSTRAINT apstar_id_fk + FOREIGN KEY (apstar_id) + REFERENCES catalogdb.sdss_dr14_apogeeStar (apstar_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- tic_v8 + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT tyc_fk + FOREIGN KEY (tyc) + REFERENCES catalogdb.tycho2 (name) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT twomass_fk + FOREIGN KEY (twomass) + REFERENCES catalogdb.twomass_psc (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT sdss_int_fk + FOREIGN KEY (sdss_int) + REFERENCES catalogdb.sdss_dr13_photoobj (objid) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss_int); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT gaia_int_fk + FOREIGN KEY (gaia_int) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT allwise_fk + FOREIGN KEY (allwise) + REFERENCES catalogdb.allwise (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT kic_fk + FOREIGN KEY (kic) + REFERENCES catalogdb.kepler_input_10 (kic_kepler_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); + + +-- bhm_spiders_agn_superset, bhm_spiders_clusters_superset + +ALTER TABLE catalogdb.bhm_spiders_agn_superset + ADD CONSTRAINT gaia_dr2_source_id_fk + FOREIGN KEY (gaia_dr2_source_id) + REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); + +ALTER TABLE catalogdb.bhm_spiders_agn_superset + ADD CONSTRAINT ls_id_fk + FOREIGN KEY (ls_id) + REFERENCES catalogdb.legacy_survey_dr8 (ls_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); + +ALTER TABLE catalogdb.bhm_spiders_clusters_superset + ADD CONSTRAINT gaia_dr2_source_id_fk + FOREIGN KEY (gaia_dr2_source_id) + REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); + +ALTER TABLE catalogdb.bhm_spiders_clusters_superset + ADD CONSTRAINT ls_id_fk + FOREIGN KEY (ls_id) + REFERENCES catalogdb.legacy_survey_dr8 (ls_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql index 4c8e82e2..03800c04 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql @@ -11,17 +11,15 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -alter table catalogdb.gaia_dr2_source add primary key(source_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (l); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (b); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ecl_lon); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ecl_lat); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (phot_g_mean_flux); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (phot_g_mean_mag); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (solution_id); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ra); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (dec); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (l); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (b); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ecl_lon); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ecl_lat); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_flux); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_mag); +CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (solution_id); CREATE INDEX ON catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); diff --git a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql index 338d9aeb..bc673b82 100644 --- a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql +++ b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql @@ -48,6 +48,8 @@ CREATE TABLE catalogdb.gaia_qso ( \copy catalogdb.gaia_qso FROM PROGRAM 'cat $CATALOGDB_DIR/gaia_qso/Gaia_QSO_Liao_2019.csv' WITH CSV HEADER; ALTER TABLE catalogdb.gaia_qso ADD COLUMN pk BIGSERIAL; +ALTER TABLE catalogdb.gaia_qso ADD PRIMARY KEY (pk); + CREATE INDEX ON catalogdb.gaia_qso (q3c_ang2ipix(raj2000, dej2000)); CLUSTER gaia_qso_q3c_ang2ipix_idx ON catalogdb.gaia_qso; diff --git a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql index aad425d0..d4b34295 100644 --- a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql +++ b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql @@ -114,7 +114,7 @@ columns below are order in which they appearn in csvs */ CREATE TABLE catalogdb.guvcat( - OBJID bigint, + OBJID bigint PRIMARY KEY, PHOTOEXTRACTID bigint, MPSTYPE text, AVASPRA double precision, @@ -210,8 +210,3 @@ CREATE TABLE catalogdb.guvcat( INLARGEOBJ text, LARGEOBJSIZE real ); - - - - - diff --git a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql index 0e969642..2e3418bd 100644 --- a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql +++ b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql @@ -11,9 +11,7 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX CONCURRENTLY ON catalogdb.guvcat using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.guvcat using BTREE (dec); - -alter table catalogdb.guvcat add primary key(objid); - +CREATE INDEX ON catalogdb.guvcat (q3c_ang2ipix(ra, dec)); +CLUSTER guvcat_q3c_ang2ipix_idx ON catalogdb.guvcat; +ANALYZE catalogdb.guvcat; diff --git a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql index 3292079b..db8ad8f5 100644 --- a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql +++ b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql @@ -28,7 +28,7 @@ CREATE TABLE catalogdb.kepler_input_10 ( kic_hmag double precision, kic_kmag double precision, kic_kepmag double precision, - kic_kepler_id bigint, + kic_kepler_id bigint PRIMARY KEY, kic_tmid bigint, kic_scpid bigint, kic_altid bigint, diff --git a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql index 06e0fc80..fae7e2b7 100644 --- a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql +++ b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql @@ -11,21 +11,20 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -alter table catalogdb.kepler_input_10 add primary key(kic_kepler_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_ra); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_dec); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_umag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_gmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_rmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_imag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_zmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_jmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_hmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_kmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_kepmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_glon); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_glat); - - - +CREATE INDEX ON catalogdb.kepler_input_10 (q3c_ang2ipix(kic_ra, kic_dec)); +CLUSTER kepler_input_10_q3c_ang2ipix_idx ON catalogdb.kepler_input_10; +ANALYZE catalogdb.kepler_input_10; + +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_ra); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_dec); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_umag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_gmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_rmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_imag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_zmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_jmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_hmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_kmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_kepmag); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_glon); +CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_glat); diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index 3962c7ff..3a7177fd 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -4,9 +4,6 @@ UPDATE catalogdb.legacy_survey_dr8 SET lsid = objid + brickid * 2^16 + release * ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (lsid); -CREATE INDEX CONCURRENTLY legacy_survey_dr8_lsid - ON catalogdb.legacy_survey_dr8 using BTREE (lsid); - CREATE INDEX on catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey_dr8; ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql index 879bd578..05b721b0 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql @@ -58,7 +58,7 @@ htmID bigint 8 HTM ID */ CREATE TABLE catalogdb.sdss_dr14_apogeeStar( - apstar_id varchar(64), + apstar_id varchar(64) PRIMARY KEY, target_id varchar(64), reduction_id varchar(32), file varchar(128), @@ -112,11 +112,13 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStar( \copy catalogdb.sdss_dr14_apogeeStar FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeStar/dr14/src/sqlApogeeStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeStar add primary key(apstar_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (glon); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (glat); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_apogeeStar_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeStar; +ANALYZE catalogdb.sdss_dr14_apogeeStar; +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (ra); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (dec); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glon); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glat); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql index 3890b098..7936a106 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql @@ -20,8 +20,7 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStarVisit( \copy catalogdb.sdss_dr14_apogeeStarVisit FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeStarVisit/dr14/src/sqlApogeeStarVisit.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeStarVisit add primary key(visit_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (apstar_id); - +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit ADD COLUMN pk BIGSERIAL PRIMARY KEY; +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (apstar_id); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (visit_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql index e46999c7..f623875b 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql @@ -1,6 +1,6 @@ /* -schema for dr 14 apogeeStar table. +schema for dr 14 apogeeVisit table. Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx @@ -60,7 +60,7 @@ synthvhelio real 4 km/s Heliocentric radial velocity from cross-corr */ CREATE TABLE catalogdb.sdss_dr14_apogeeVisit( - visit_id varchar(64), + visit_id varchar(64) PRIMARY KEY, apred_version varchar(32), apogee_id varchar(64), target_id varchar(64), @@ -115,11 +115,11 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeVisit( \copy catalogdb.sdss_dr14_apogeeVisit FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeVisit/dr14/src/sqlApogeeVisit.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeVisit add primary key(visit_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (glon); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (glat); - +CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_apogeeVisit_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeVisit; +ANALYZE catalogdb.sdss_dr14_apogeeVisit; +CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (ra); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (dec); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glon); +CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glat); diff --git a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql index 1d82eda2..5816248c 100644 --- a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql +++ b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql @@ -171,7 +171,7 @@ felem_nd_h_err real 4 dex FERRE pipeline uncalibratied ratio original CREATE TABLE catalogdb.sdss_dr14_ascapStar( apstar_id varchar(64), target_id varchar(64), - aspcap_id varchar(64), + aspcap_id varchar(64) PRIMARY KEY, apogee_id varchar(32), aspcap_version varchar(32), results_version varchar(32), @@ -331,12 +331,7 @@ CREATE TABLE catalogdb.sdss_dr14_ascapStar( felem_nd_h_err real ); - \copy catalogdb.sdss_dr14_ascapStar FROM program 'bzcat $CATALOGDB_DIR/sdssAscapStar/dr14/src/sqlaspcapStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_ascapStar add primary key(apstar_id); - -CREATE INDEX CONCURRENTLY sdss_dr14_ascapStar_target_id_index ON catalogdb.sdss_dr14_ascapStar using BTREE (target_id); -CREATE INDEX CONCURRENTLY sdss_dr14_ascapStar_apogee_id_index ON catalogdb.sdss_dr14_ascapStar using BTREE (apogee_id); - - +CREATE INDEX ON catalogdb.sdss_dr14_ascapStar USING BTREE (target_id); +CREATE INDEX ON catalogdb.sdss_dr14_ascapStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql index e3719be5..64f0cc5c 100644 --- a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql +++ b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql @@ -89,7 +89,7 @@ r_chi_sq real 4 reduced chi^2 from Cannon analysis CREATE TABLE catalogdb.sdss_dr14_cannonStar( apogee_id varchar(32), - cannon_id varchar(64), + cannon_id varchar(64) PRIMARY KEY, filename varchar(128), location_id bigint, field varchar(128), @@ -169,11 +169,6 @@ CREATE TABLE catalogdb.sdss_dr14_cannonStar( r_chi_sq real ); - \copy catalogdb.sdss_dr14_cannonStar FROM program 'bzcat $CATALOGDB_DIR/sdssCannonStar/dr14/src/sqlcannonStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_cannonStar add primary key(cannon_id); - -CREATE INDEX CONCURRENTLY sdss_dr14_cannonStar_apogee_id_index ON catalogdb.sdss_dr14_cannonStar using BTREE (apogee_id); - - +CREATE INDEX ON catalogdb.sdss_dr14_cannonStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql index 45cb0177..a653a195 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql @@ -521,7 +521,7 @@ TAI_z */ CREATE TABLE catalogdb.sdss_dr13_photoobj( - objID bigint, + objID bigint PRIMARY KEY, run smallint, rerun smallint, camcol smallint, @@ -735,8 +735,3 @@ CREATE TABLE catalogdb.sdss_dr13_photoobj( b double precision, l double precision ); - - - - - diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql index 56439e1e..895873cd 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql @@ -3,19 +3,18 @@ indices for catalogdb tables, to be run after bulk uploads */ -alter table catalogdb.sdss_dr13_photoobj add primary key(objid); - -- Indices -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (l); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (b); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_u); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_g); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_r); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_i); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_z); - - +CREATE INDEX ON catalogdb.sdss_dr13_photoobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr13_photoobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr13_photoobj; +ANALYZE catalogdb.sdss_dr13_photoobj; +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (ra); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (dec); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (l); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (b); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_u); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_g); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_r); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_i); +CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_z); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql index aa68de15..cb36c189 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql @@ -205,7 +205,7 @@ img varbinary -1 IMAGE? Spectrum Image */ CREATE TABLE catalogdb.sdss_dr14_specobj( - specObjID numeric(20), + specObjID numeric(20) PRIMARY KEY, bestObjID bigint, fluxObjID bigint, targetObjID bigint, @@ -404,14 +404,13 @@ CREATE TABLE catalogdb.sdss_dr14_specobj( \copy catalogdb.sdss_dr14_specobj FROM program 'bzcat $CATALOGDB_DIR/sdssSpecObj/dr14/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_specobj add primary key(specObjID); - -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_bestObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (bestObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_fluxObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (fluxObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_targetObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (targetObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_ra_index ON catalogdb.sdss_dr14_specobj using BTREE (ra); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_dec_index ON catalogdb.sdss_dr14_specobj using BTREE (dec); - - - +CREATE INDEX ON catalogdb.sdss_dr14_specobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_specobj; +ANALYZE catalogdb.sdss_dr14_specobj; +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (bestObjID); +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (mjd); +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (plate); +CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fiberid); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index dd70583e..576ad0ed 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -205,7 +205,7 @@ img varbinary -1 IMAGE? Spectrum Image */ CREATE TABLE catalogdb.sdss_dr16_specobj( - specObjID numeric(20), + specObjID numeric(20) PRIMARY KEY, bestObjID bigint, fluxObjID bigint, targetObjID bigint, @@ -404,22 +404,13 @@ CREATE TABLE catalogdb.sdss_dr16_specobj( \copy catalogdb.sdss_dr16_specobj FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj/dr16/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr16_specobj add primary key(specObjID); - -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_bestObjID_index - ON catalogdb.sdss_dr16_specobj using BTREE (bestObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fluxObjID_index - ON catalogdb.sdss_dr16_specobj using BTREE (fluxObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_targetObjID_index - ON catalogdb.sdss_dr16_specobj using BTREE (targetObjID); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_mjd_index - ON catalogdb.sdss_dr16_specobj using BTREE (mjd); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_plate_index - ON catalogdb.sdss_dr16_specobj using BTREE (plate); -CREATE INDEX CONCURRENTLY sdss_dr16_sdss_dr16_specobj_fiberid_index - ON catalogdb.sdss_dr16_specobj using BTREE (fiberid); - - -CREATE INDEX on catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); -CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx on catalogdb.sdss_dr16_specobj; +CREATE INDEX ON catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr16_specobj; ANALYZE catalogdb.sdss_dr16_specobj; + +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (bestObjID); +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fluxObjID); +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (mjd); +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (plate); +CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fiberid); diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql index 84fa2478..5d918bbc 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql @@ -7,93 +7,129 @@ schema for tess: */ CREATE TABLE catalogdb.tic_v8 ( - ID bigint, - version text, - HIP integer, - TYC text, - UCAC text, - TWOMASS text, - SDSS bigint, - ALLWISE text, - GAIA text, - APASS text, - KIC integer, - objType text, - typeSrc text, - ra double precision, - dec double precision, - POSflag text, - pmRA real, - e_pmRA real, - pmDEC real, - e_pmDEC real, - PMflag text, - plx real, - e_plx real, - PARflag text, - gallong double precision, - gallat double precision, - eclong double precision, - eclat double precision, - Bmag real, - e_Bmag real, - Vmag real, - e_Vmag real, - umag real, - e_umag real, - gmag real, - e_gmag real, - rmag real, - e_rmag real, - imag real, - e_imag real, - zmag real, - e_zmag real, - Jmag real, - e_Jmag real, - Hmag real, - e_Hmag real, - Kmag real, - e_Kmag real, - TWOMflag text, - prox real, - w1mag real, - e_w1mag real, - w2mag real, - e_w2mag real, - w3mag real, - e_w3mag real, - w4mag real, - e_w4mag real, - GAIAmag real, - e_GAIAmag real, - Tmag real, - e_Tmag real, - TESSflag text, - SPFlag text, - Teff real, - e_Teff real, - logg real, - e_logg real, - MH real, - e_MH real, - rad real, - e_rad real, - mass real, - e_mass real, - rho real, - e_rho real, - lumclass text, - lum real, - e_lum real, - d real, - e_d real, - ebv real, - e_ebv real, - numcont integer, - contratio real, - disposition text, - duplicate_i bigint, - priority real, - objID bigint + id BIGINT PRIMARY KEY, + version VARCHAR(8), + hip INTEGER, + tyc VARCHAR(12), + ucac VARCHAR(10), + twomass VARCHAR(20), + sdss BIGINT, + allwise VARCHAR(20), + gaia BIGINT, + apass VARCHAR(30), + kic INTEGER, + objtype VARCHAR(16), + typesrc VARCHAR(16), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + posflag VARCHAR(12), + pmra REAL, + e_pmra REAL, + pmdec REAL, + e_pmdec REAL, + pmflag VARCHAR(12), + plx REAL, + e_plx REAL, + parflag VARCHAR(12), + gallong DOUBLE PRECISION, + gallat DOUBLE PRECISION, + eclong DOUBLE PRECISION, + eclat DOUBLE PRECISION, + bmag REAL, + e_bmag REAL, + vmag REAL, + e_vmag REAL, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + jmag REAL, + e_jmag REAL, + hmag REAL, + e_hmag REAL, + kmag REAL, + e_kmag REAL, + twomflag VARCHAR(20), + prox REAL, + w1mag REAL, + e_w1mag REAL, + w2mag REAL, + e_w2mag REAL, + w3mag REAL, + e_w3mag REAL, + w4mag REAL, + e_w4mag REAL, + gaiamag REAL, + e_gaiamag REAL, + tmag REAL, + e_tmag REAL, + tessflag VARCHAR(20), + spflag VARCHAR(20), + teff REAL, + e_teff REAL, + logg REAL, + e_logg REAL, + mh REAL, + e_mh REAL, + rad REAL, + e_rad REAL, + mass REAL, + e_mass REAL, + rho REAL, + e_rho REAL, + lumclass VARCHAR(10), + lum REAL, + e_lum REAL, + d REAL, + e_d REAL, + ebv REAL, + e_ebv REAL, + numcont INTEGER, + contratio REAL, + disposition VARCHAR(10), + duplicate_id BIGINT, + priority REAL, + eneg_ebv REAL, + epos_ebv REAL, + ebvflag VARCHAR(20), + eneg_mass REAL, + epos_mass REAL, + eneg_rad REAL, + epos_rad REAL, + eneg_rho REAL, + epos_rho REAL, + eneg_logg REAL, + epos_logg REAL, + eneg_lum REAL, + epos_lum REAL, + eneg_dist REAL, + epos_dist REAL, + distflag VARCHAR(20), + eneg_teff REAL, + epos_teff REAL, + tefflag VARCHAR(20), + gaiabp REAL, + e_gaiabp REAL, + gaiarp REAL, + e_gaiarp REAL, + gaiaqflag INTEGER, + starchareflag VARCHAR(20), + vmagflag VARCHAR(20), + bmagflag VARCHAR(20), + splits VARCHAR(20), + e_ra DOUBLE PRECISION, + e_dec DOUBLE PRECISION, + ra_orig DOUBLE PRECISION, + dec_orig DOUBLE PRECISION, + e_ra_orig DOUBLE PRECISION, + e_dec_orig DOUBLE PRECISION, + raddflag INTEGER, + wdflag INTEGER, + objid BIGINT, ) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index 9c1bb3ed..e60c2a54 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -1,22 +1,28 @@ --- PK -ALTER TABLE catalogdb.tic_v8 ADD PRIMARY KEY (id); +-- Convert SDSS and GAIA to bigint for more efficient FKs + +ALTER TABLE catalogdb.tic_v8 ADD COLUMN gaia_int BIGINT; +UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; + +ALTER TABLE catalogdb.tic_v8 ADD COLUMN sdss_int BIGINT; +UPDATE catalogdb.tic_v8 SET sdss_int = sdss::BIGINT; + -- Indices CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.tic_v8; +CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.tic_v8; ANALYZE catalogdb.tic_v8; -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gallong); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gallat); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Bmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Vmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (umag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (rmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (imag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (zmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Jmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Hmag); -CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Kmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gallong); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gallat); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Bmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Vmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (umag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (rmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (imag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (zmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Jmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Hmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Kmag); diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql index e4a10355..f7337278 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql @@ -37,7 +37,7 @@ CREATE TABLE catalogdb.twomass_psc ( pxcntr integer, gal_contam smallint, mp_flg smallint, - pts_key integer, + pts_key integer PRIMARY KEY, hemis character(1), date date, scan smallint, @@ -529,6 +529,3 @@ CREATE TABLE catalogdb.twomass_xsc ( k_seetrack real, ext_key integer ) WITHOUT OIDS; - - - diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql index c5bc5415..cb051987 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql @@ -8,24 +8,22 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; */ - -- Indices -alter table catalogdb.twomass_psc add primary key(pts_key); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (ra); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (decl); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (j_m); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (h_m); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (k_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (decl); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (j_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (h_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (k_m); -create index on catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); -CLUSTER twomass_psc_q3c_ang2ipix_idx on catalogdb.twomass_psc; -analyze catalogdb.twomass_psc; +CREATE INDEX ON catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); +CLUSTER twomass_psc_q3c_ang2ipix_idx ON catalogdb.twomass_psc; +ANALYZE catalogdb.twomass_psc; -ALTER TABLE catalogdb.twomass_psc ADD CONSTRAINT twomass_psc_desig_unique UNIQUE (designation); +ALTER TABLE catalogdb.twomass_psc ADD CONSTRAINT UNIQUE (designation); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (designation); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (ph_qual); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (cc_flg); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (gal_contam); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (rd_flg); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (designation); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (ph_qual); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (cc_flg); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (gal_contam); +CREATE INDEX ON catalogdb.twomass_psc USING BTREE (rd_flg); diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql index 858459d6..6bee10d5 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -333,7 +333,7 @@ References: CREATE TABLE catalogdb.tycho2 ( id_tycho BIGINT PRIMARY KEY, - name TEXT, + name TEXT PRIMARY KEY, pflag CHARACTER(1), ramdeg DOUBLE PRECISION, demdeg DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql index 465cba3c..b2224225 100644 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql @@ -1,4 +1,4 @@ -CREATE INDEX on catalogdb.unwise (q3c_ang2ipix(ra, dec)); -CLUSTER unwise_q3c_ang2ipix_idx on catalogdb.unwise; +CREATE INDEX ON catalogdb.unwise (q3c_ang2ipix(ra, dec)); +CLUSTER unwise_q3c_ang2ipix_idx ON catalogdb.unwise; ANALYZE catalogdb.unwise; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql index 0a7e4a9e..e8ca190c 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -1,8 +1,14 @@ -CREATE INDEX on catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); -CLUSTER gaia_dr2_wd_q3c_ang2ipix_idx on catalogdb.gaia_dr2_wd; +CREATE UNIQUE INDEX gaia_dr2_wd_wd_key ON catalogdb.gaia_dr2_wd(wd text_ops); + +CREATE INDEX ON catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd; ANALYZE catalogdb.gaia_dr2_wd; -CREATE INDEX on catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); -CLUSTER gaia_dr2_wd_sdss_q3c_ang2ipix_idx on catalogdb.gaia_dr2_wd_sdss; +CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_sdss_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd_sdss; ANALYZE catalogdb.gaia_dr2_wd_sdss; + +CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (plate); +CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (mjd); +CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (fiber); diff --git a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql index defc785f..54bcfdef 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql @@ -18,10 +18,9 @@ CREATE TABLE catalogdb.gaiadr2_tmass_best_neighbour( tmass_pts_key integer ); - \copy catalogdb.gaiadr2_tmass_best_neighbour FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' WITH CSV HEADER; -alter table catalogdb.gaiadr2_tmass_best_neighbour add primary key(source_id); +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD PRIMARY KEY (source_id); -CREATE INDEX CONCURRENTLY gaiadr2_tmass_best_neighbour_tmass_oid_index ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_oid); -CREATE INDEX CONCURRENTLY gaiadr2_tmass_best_neighbour_tmass_original_ext_source_id_index ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (original_ext_source_id); +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (tmass_oid); +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (original_ext_source_id); From 4179f9498fbcf3b62c1b6fdbe5fc0a32d7a54c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 23 Mar 2020 16:16:02 -0700 Subject: [PATCH 106/197] Files for CatWISE --- python/sdssdb/peewee/sdss5db/catalogdb.py | 14 + schema/sdss5db/catalogdb/CatWISE/CatWISE.sql | 603 ++++++++++++++++++ .../catalogdb/CatWISE/CatWISE_download | 4 + .../catalogdb/CatWISE/CatWISE_indexes.sql | 16 + schema/sdss5db/catalogdb/CatWISE/CatWISE_load | 13 + .../catalogdb/CatWISE/CatWISE_load_helper.py | 32 + 6 files changed, 682 insertions(+) create mode 100644 schema/sdss5db/catalogdb/CatWISE/CatWISE.sql create mode 100644 schema/sdss5db/catalogdb/CatWISE/CatWISE_download create mode 100644 schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql create mode 100755 schema/sdss5db/catalogdb/CatWISE/CatWISE_load create mode 100644 schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 51cb162e..da14a2f7 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -518,5 +518,19 @@ class Meta: schema = 'catalogdb' +class CatWISE(CatalogdbModel): + + class Meta: + table_name = 'tic_v8' + schema = 'catwise' + + +class CatWISEReject(CatalogdbModel): + + class Meta: + table_name = 'tic_v8' + schema = 'catwise_reject' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql new file mode 100644 index 00000000..53ad0e36 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql @@ -0,0 +1,603 @@ +/* + +CatWISE Catalog Output Table File Description (Rev. 1.4, May 11, 2019) +Based on WPHot Output Table File Description [pht12] Rev 1.5 Dec. 10, 2018 +and on output from stf 1.7 B90210 run on IRSA table format files with -L option. + +Interface Name: CatWISE catalog table file for Preliminary CatWISE Processing +Type of Interface: disk i/o + +Written By: J.Fowler, P.Eisenhardt + +Read By: CatWISE Software Team + +Description: + +Extension of Version 7 of WPHotpm for multiframe processing of two bands +(unWISE W1 and W2 coadds) with additional table-file columns derived from: +- merging the separate mdex files for ascending and descending scans of a tile +- adding cc_flags from the AllWISE catalog +- adding flags based on unWISE artifact bitmasks +- adding primary source in tile designation flag + +_______________________________________________________________________________________ + +Specification of output variables: + +Table Header information: + +-- # sources tabulated +-- unWISE epoch coadds engaged (ascending and descending for option 1) +-- WISE bands engaged (always bands 1 and 2 only for CatWISE) +-- zero point mag used +-- standard aperture radius in arcsec and pixel units (one line per band) +-- circular aperture radii in arcsec and pixel units (one line per band) +-- Modified Julian Date of the CatWISE coordinate epoch (MJD 56700.00 for Preliminary Catalog) +-- CatWISE processing history (6 lines) + +Tabular data: + +Note that the first column in the width is occupied by a "pipe" or "bar" delimiter ("|") + + Field cols Width Units Type Format Name Description + ---- ---- ----- ------- ---- ------- ------------ ------------------------------------ + 1 1-21 21 -- char a21 source_name source hexagesimal designation + 2 22-46 25 -- char a25 source_id tile name + processing code + wphot index + 3 47-58 12 deg R*8 f12.7 ra right ascension (J2000) + 4 59-70 12 deg R*8 f12.7 dec declination (J2000) + 5 71-79 9 asec R*4 f9.4 sigra uncertainty in ra (arcsec) + 6 80-88 9 asec R*4 f9.4 sigdec uncertainty in dec (arcsec) + 7 89-97 9 asec R*4 f9.4 sigradec uncertainty cross-term (arcsec) + +# the following positions reflect the source xy position in the unWISE full depth coadd + + 8 98-106 9 pix R*4 f9.3 wx x-pixel coordinate + 9 107-115 9 pix R*4 f9.3 wy y-pixel coordinate + +# The next set of columns are aperture/annulus measurements from the unWISE epoch coadds: + + 10 116-125 10 dn R*4 f10.3 w1sky frame sky background value, band-1 + 11 126-133 8 dn R*4 f8.3 w1sigsk frame sky background value uncertainty, band-1 + 12 134-141 8 dn R*4 f8.3 w1conf frame sky confusion based on the UNC images + 13 142-151 10 dn R*4 f10.3 w2sky frame sky background value, band-2 + 14 152-159 8 dn R*4 f8.3 w2sigsk frame sky background value uncertainty, band-2 + 15 160-167 8 dn R*4 f8.3 w2conf frame sky confusion based on the UNC images + +# WPRO epoch coadd measurements: + + 16 168-174 7 asec R*4 f7.2 w1fitr fitting radius, band-1 (clipped at 999.99) + 17 175-181 7 asec R*4 f7.2 w2fitr fitting radius, band-2 (clipped at 999.99) + 18 182-188 7 -- R*4 f7.1 w1snr S/N ratio, band-1 + 19 189-195 7 -- R*4 f7.1 w2snr S/N ratio, band-2 + 20 196-207 12 dn R*4 1pe12.4 w1flux WPRO raw flux, band-1 + 21 208-221 14 dn R*4 1pe14.4 w1sigflux WPRO raw flux uncertainty, band-1 + 22 222-233 12 dn R*4 1pe12.4 w2flux WPRO raw flux, band-2 + 23 234-247 14 dn R*4 1pe14.4 w2sigflux WPRO raw flux uncertainty, band-2 + 24 248-254 7 mag R*4 f7.3 w1mpro WPRO flux in mag units, band-1 + 25 255-264 10 mag R*4 f10.3 w1sigmpro WPRO flux uncertainty in mag units, band-1 + 26 265-275 11 -- R*4 1pe11.3 w1rchi2 WPRO reduced chi^2, band-1 + 27 276-282 7 mag R*4 f7.3 w2mpro WPRO flux in mag units, band-2 + 28 283-292 10 mag R*4 f10.3 w2sigmpro WPRO flux uncertainty in mag units, band-2 + 29 293-303 11 -- R*4 1pe11.3 w2rchi2 WPRO reduced chi^2, band-2 + 30 304-314 11 -- R*4 1pe11.3 rchi2 reduced chi squared, total + + 31 315-318 4 -- int i4 nb number of blend components used in each fit + 32 319-322 4 -- int i4 na number of actively deblended components + 33 323-330 8 -- R*4 f8.5 w1Sat fraction of pixels saturated, band-1 + 34 331-338 8 -- R*4 f8.5 w2Sat fraction of pixels saturated, band-2 + +# full depth coadd measurements: WAPPco, standard aperture w/ aperture correction; +# the standard (aperture corrected) aperture radius is 8.25 arcsec. + + 35 339-345 7 mag R*4 f7.3 w1mag standard aperture mag w/ correction applied + 36 346-352 7 mag R*4 f7.3 w1sigm standard aperture mag uncertainty, band-1 + 37 353-358 6 -- int i6 w1flg standard aperture flag, band-1 + 38 359-366 8 -- R*4 f8.2 w1Cov mean coverage depth, band-1 + 39 367-373 7 mag R*4 f7.3 w2mag standard aperture mag w/ correction applied + 40 374-380 7 mag R*4 f7.3 w2sigm standard aperture mag uncertainty, band-2 + 41 381-386 6 -- int i6 w2flg standard aperture flag, band-2 + 42 387-394 8 -- R*4 f8.2 w2Cov mean coverage depth, band-2 + +# full depth coadd measurements: WAPPco, circular apertures, no aperture correction is applied; +# radii: 5.50 8.25 11.00 13.75 16.50 19.25 22.00 24.75 arcsec + + 43 395-404 10 mag R*4 f10.3 w1mag_1 Aper 1, aperture mag, band-1 + 44 405-414 10 mag R*4 f10.3 w1sigm_1 Aper 1, aperture mag uncertainty, band-1 + 45 415-422 8 -- int i8 w1flg_1 Aper 1, aperture flag, band-1 + 46 423-432 10 mag R*4 f10.3 w2mag_1 Aper 1, aperture mag, band-2 + 47 433-442 10 mag R*4 f10.3 w2sigm_1 Aper 1, aperture mag uncertainty, band-2 + 48 443-450 8 -- int i8 w2flg_1 Aper 1, aperture flag, band-2 + + 49 451-460 10 mag R*4 f10.3 w1mag_2 Aper 2, aperture mag, band-1 + 50 461-470 10 mag R*4 f10.3 w1sigm_2 Aper 2, aperture mag uncertainty, band-1 + 51 471-478 8 -- int i8 w1flg_2 Aper 2, aperture flag, band-1 + 52 479-488 10 mag R*4 f10.3 w2mag_2 Aper 2, aperture mag, band-2 + 53 489-498 10 mag R*4 f10.3 w2sigm_2 Aper 2, aperture mag uncertainty, band-2 + 54 499-506 8 -- int i8 w2flg_2 Aper 2, aperture flag, band-2 + + 55 507-516 10 mag R*4 f10.3 w1mag_3 Aper 3, aperture mag, band-1 + 56 517-526 10 mag R*4 f10.3 w1sigm_3 Aper 3, aperture mag uncertainty, band-1 + 57 527-534 8 -- int i8 w1flg_3 Aper 3, aperture flag, band-1 + 58 535-544 10 mag R*4 f10.3 w2mag_3 Aper 3, aperture mag, band-2 + 59 545-554 10 mag R*4 f10.3 w2sigm_3 Aper 3, aperture mag uncertainty, band-2 + 60 555-562 8 -- int i8 w2flg_3 Aper 3, aperture flag, band-2 + + 61 563-572 10 mag R*4 f10.3 w1mag_4 Aper 4, aperture mag, band-1 + 62 573-582 10 mag R*4 f10.3 w1sigm_4 Aper 4, aperture mag uncertainty, band-1 + 63 583-590 8 -- int i8 w1flg_4 Aper 4, aperture flag, band-1 + 64 591-600 10 mag R*4 f10.3 w2mag_4 Aper 4, aperture mag, band-2 + 65 601-610 10 mag R*4 f10.3 w2sigm_4 Aper 4, aperture mag uncertainty, band-2 + 66 611-618 8 -- int i8 w2flg_4 Aper 4, aperture flag, band-2 + + 67 619-628 10 mag R*4 f10.3 w1mag_5 Aper 5, aperture mag, band-1 + 68 629-638 10 mag R*4 f10.3 w1sigm_5 Aper 5, aperture mag uncertainty, band-1 + 69 639-646 8 -- int i8 w1flg_5 Aper 5, aperture flag, band-1 + 70 647-656 10 mag R*4 f10.3 w2mag_5 Aper 5, aperture mag, band-2 + 71 657-666 10 mag R*4 f10.3 w2sigm_5 Aper 5, aperture mag uncertainty, band-2 + 72 667-674 8 -- int i8 w2flg_5 Aper 5, aperture flag, band-2 + + 73 675-684 10 mag R*4 f10.3 w1mag_6 Aper 6, aperture mag, band-1 + 74 685-694 10 mag R*4 f10.3 w1sigm_6 Aper 6, aperture mag uncertainty, band-1 + 75 695-702 8 -- int i8 w1flg_6 Aper 6, aperture flag, band-1 + 76 703-712 10 mag R*4 f10.3 w2mag_6 Aper 6, aperture mag, band-2 + 77 713-722 10 mag R*4 f10.3 w2sigm_6 Aper 6, aperture mag uncertainty, band-2 + 78 723-730 8 -- int i8 w2flg_6 Aper 6, aperture flag, band-2 + + 79 731-740 10 mag R*4 f10.3 w1mag_7 Aper 7, aperture mag, band-1 + 80 741-750 10 mag R*4 f10.3 w1sigm_7 Aper 7, aperture mag uncertainty, band-1 + 81 751-758 8 -- int i8 w1flg_7 Aper 7, aperture flag, band-1 + 82 759-768 10 mag R*4 f10.3 w2mag_7 Aper 7, aperture mag, band-2 + 83 769-778 10 mag R*4 f10.3 w2sigm_7 Aper 7, aperture mag uncertainty, band-2 + 84 779-786 8 -- int i8 w2flg_7 Aper 7, aperture flag, band-2 + + 85 787-796 10 mag R*4 f10.3 w1mag_8 Aper 8, aperture mag, band-1 + 86 797-806 10 mag R*4 f10.3 w1sigm_8 Aper 8, aperture mag uncertainty, band-1 + 87 807-814 8 -- int i8 w1flg_8 Aper 8, aperture flag, band-1 + 88 815-824 10 mag R*4 f10.3 w2mag_8 Aper 8, aperture mag, band-2 + 89 825-834 10 mag R*4 f10.3 w2sigm_8 Aper 8, aperture mag uncertainty, band-2 + 90 835-842 8 -- int i8 w2flg_8 Aper 8, aperture flag, band-2 + +# the following are "N of M" counters for WPRO measurements +# w?M - The number of individual epochs for band ? that are +# available to make a profile-fit measurement. +# w?NM - The number of individual epochs for band ? on which +# WPRO extracted a flux measurement that has snr>3. +# w?mLQ - variability indicator mLogQ for the flux array + + 91 843-849 7 -- int i7 w1NM WPRO, N (of M), band-1 + 92 850-855 6 -- int i6 w1M WPRO, M , band-1 + 93 856-863 8 mag R*4 f8.3 w1magP WPRO repeatability mag; band-1 + 94 864-871 8 mag R*4 f8.3 w1sigP1 WPRO mag population sigma; band-1 + 95 872-879 8 mag R*4 f8.3 w1sigP2 WPRO mag uncertainty of the mean; band-1 + 96 880-891 12 -- R*4 f12.5 w1k Stetson k index for variability; band-1 + 97 892-897 6 -- int i6 w1Ndf No. degrees of freedom in var chi-square; band-1 + 98 898-903 6 -- R*4 f6.2 w1mLQ -log(Q), Q = 1-P(chi-square); band-1 + 99 904-921 18 -- R*8 f18.8 w1mJDmin minimum mJD; band-1 + 100 922-939 18 -- R*8 f18.8 w1mJDmax maximum mJD; band-1 + 101 940-957 18 -- R*8 f18.8 w1mJDmean mean mJD; band-1 + + 102 958-964 7 -- int i7 w2NM WPRO, N (of M), band-2 + 103 965-970 6 -- int i6 w2M WPRO, M , band-2 + 104 971-978 8 mag R*4 f8.3 w2magP WPRO repeatability mag; band-2 + 105 979-986 8 mag R*4 f8.3 w2sigP1 WPRO mag population sigma; band-2 + 106 987-994 8 mag R*4 f8.3 w2sigP2 WPRO mag uncertainty of the mean; band-2 + 107 995-1006 12 -- R*4 f12.5 w2k Stetson k index for variability; band-2 + 108 1007-1012 6 -- int i6 w2Ndf No. degrees of freedom in var chi-square; band-2 + 109 1013-1018 6 -- R*4 f6.2 w2mLQ -log(Q), Q = 1-P(chi-square); band-2 + 110 1019-1036 18 -- R*8 f18.8 w2mJDmin minimum mJD; band-2 + 111 1037-1054 18 -- R*8 f18.8 w2mJDmax maximum mJD; band-2 + 112 1055-1072 18 -- R*8 f18.8 w2mJDmean mean mJD; band-2 + +# band-to-band sample correlation coefficients and their probabilities for adjacent bands + + 113 1073-1078 6 % int i4 rho12 W1W2 correlation coefficient + 114 1079-1084 6 -- int i4 q12 -log10(1-P(rho12)) given no real correlation + + 115 1085-1091 7 -- int i7 nIters number of chi-square-minimization iterations + 116 1092-1098 7 -- int i7 nSteps number of steps in all iterations + +# MDET-Related Parameters +# p1 and p2 are clipped to the range (-99.99999, +99.99999) if necessary + + 117 1099-1105 7 -- int i7 mdetID source ID in mdet list + 118 1106-1115 10 asec R*4 f10.5 p1 P vector component 1 + 119 1116-1125 10 asec R*4 f10.5 p2 P vector component 2 + +# Motion Fit Solution Parameters +# ra_pm and dec_pm are in the equinox J2000 frame at the CatWISE epoch +# (MJD 56700.00 for the Preliminary Catalog) based on PMRA and PMDec + + 120 1126-1138 13 MJD R*8 f13.6 MeanObsMJD mean observation epoch + 121 1139-1150 12 deg R*8 f12.7 ra_pm right ascension (J2000) + 122 1151-1162 12 deg R*8 f12.7 dec_pm declination (J2000) + 123 1163-1171 9 asec R*4 f9.4 sigra_pm uncertainty in ra_pm + 124 1172-1181 10 asec R*4 f10.4 sigdec_pm uncertainty in dec_pm + 125 1182-1193 12 asec R*4 f12.4 sigradec_pm uncertainty cross-term + 126 1194-1203 10 asec/yr R*4 f10.4 PMRA motion in ra + 127 1204-1213 10 asec/yr R*4 f10.4 PMDec proper motion in dec + 128 1214-1222 9 asec/yr R*4 f9.4 sigPMRA uncertainty in PMRA + 129 1223-1231 9 asec/yr R*4 f9.4 sigPMDec uncertainty in PMDec + + 130 1232-1240 9 -- R*4 f9.1 w1snr_pm flux S/N ratio, band-1 + 131 1241-1249 9 -- R*4 f9.1 w2snr_pm flux S/N ratio, band-2 + 132 1250-1261 12 dn R*4 1pe12.4 w1flux_pm WPRO raw flux, band-1 + 133 1262-1275 14 dn R*4 1pe14.4 w1sigflux_pm WPRO raw flux uncertainty, band-1 + 134 1276-1287 12 dn R*4 1pe12.4 w2flux_pm WPRO raw flux, band-2 + 135 1288-1301 14 dn R*4 1pe14.4 w2sigflux_pm fit WPRO raw flux uncertainty, band-2 + 136 1302-1311 10 mag R*4 f10.3 w1mpro_pm WPRO flux in mag units, band-1 + 137 1312-1324 13 mag R*4 f13.3 w1sigmpro_pm WPRO flux uncertainty in mag units, band-1 + 138 1325-1335 11 -- R*4 1pe11.3 w1rchi2_pm WPRO reduced chi^2, band-1 + 139 1336-1345 10 mag R*4 f10.3 w2mpro_pm WPRO flux in mag units, band-2 + 140 1346-1358 13 mag R*4 f13.3 w2sigmpro_pm WPRO flux uncertainty in mag units, band-2 + 141 1359-1369 11 -- R*4 1pe11.3 w2rchi2_pm WPRO reduced chi^2, band-2 + 142 1370-1380 11 -- R*4 1pe11.3 rchi2_pm reduced chi squared, total + +# pmcode provides information that may correlate with the quality of the PM solution. +# The format is ABCCC, where A is the number of components in the passive blend group +# (including the primary) before any are removed or added, +# B is "Y" or "N" to indicate "Yes" or "No" that a secondary blend component replaced the primary, +# CCC is the distance in hundredths of an arcsec between the +# PM position solution for the mean observation epoch and the stationary solution + + 143 1381-1387 7 -- char A5 pmcode quality of the PM solution + + 144 1388-1397 10 -- int i10 nIters_pm number of chi-square-minimization iterations + 145 1398-1407 10 -- int i10 nSteps_pm number of steps in all iterations + +# Parameters Derived From Ascending-Descending Scan Differences + + 146 1408-1416 9 asec R*4 f9.3 dist radial distance between apparitions + 147 1417-1423 7 mag R*4 f7.3 dw1mag w1mpro difference + 148 1424-1430 7 -- R*4 f7.3 rch2w1 chi-square for dw1mag (1 DF) + 149 1431-1437 7 mag R*4 f7.3 dw2mag w2mpro difference + 150 1438-1444 7 -- R*4 f7.3 rch2w2 chi-square for dw2mag (1 DF) + 151 1445-1455 11 deg R*8 f11.6 elon_avg averaged ecliptic longitude + 152 1456-1466 11 asec R*4 f11.3 elonSig one-sigma uncertainty in elon + 153 1467-1477 11 deg R*8 f11.6 elat_avg averaged ecliptic latitude + 154 1478-1487 10 asec R*4 f10.3 elatSig one-sigma uncertainty in elat + 155 1488-1498 11 asec R*4 f11.3 Delon desc-asce ecliptic longitude + 156 1499-1509 11 asec R*4 f11.3 DelonSig one-sigma uncertainty in Delon + 157 1510-1520 11 asec R*4 f11.3 Delat desc-asce ecliptic latitude + 158 1521-1530 10 asec R*4 f10.3 DelatSig one-sigma uncertainty in Delat + 159 1531-1541 11 -- R*4 f11.3 DelonSNR |Delon|/DelonSig + 160 1542-1552 11 -- R*4 f11.3 DelatSNR |Delat|/DelatSig + 161 1553-1562 10 -- R*4 1pE10.3 chi2pmra chi-square for PMRA difference (1 DF) + 162 1563-1572 10 -- R*4 1pE10.3 chi2pmdec chi-square for PMRA difference (1 DF) + 163 1573-1575 3 -- int i3 ka astrometry usage code + 164 1576-1578 3 -- int i3 k1 W1 photometry usage code + 165 1579-1581 3 -- int i3 k2 W2 photometry usage code + 166 1582-1584 3 -- int i3 km proper motion usage code + 167 1585-1595 11 asec R*4 f11.3 par_pm parallax from PM desc-asce elon + 168 1596-1606 11 asec R*4 f11.3 par_pmSig one-sigma uncertainty in par_pm + 169 1607-1617 11 asec R*4 f11.3 par_stat parallax estimate from stationary solution + 170 1618-1628 11 asec R*4 f11.3 par_sigma one-sigma uncertainty in par_stat + +# Parameters Derived from AllWISE Artifact Flags + + 171 1629-1641 13 asec R*4 f13.6 dist_x distance between CatWISE and AllWISE source + 172 1642-1657 16 -- char a16 cc_flags worst case 4 character cc_flag from AllWISE + 173 1658-1670 13 -- int i13 w1cc_map worst case w1cc_map from AllWISE + 174 1671-1690 20 -- char a20 w1cc_map_str worst case w1cc_map_str from AllWISE + 175 1691-1703 13 -- int i13 w2cc_map worst case w2cc_map from AllWISE + 176 1704-1723 20 -- char a20 w2cc_map_str worst case w2cc_map_str from AllWISE + 177 1724-1728 5 -- int i5 n_aw number of AllWISE matches within 2.75 asec + +# Parameters Derived from unWISE Artifact Bitmasks + + 178 1729-1737 9 -- char a9 ab_flags Two character (W1 W2) artifact flag + 179 1738-1746 9 -- int i9 w1ab_map w1 artifact code value + 180 1747-1759 13 -- char a13 w1ab_map_str w1 artifact string + 181 1760-1768 9 -- int i9 w2ab_map w2 artifact code value + 182 1769-1781 13 -- char a13 w2ab_map_str w2 artifact string + +# Indexed Coordinates Derived from Stationary ra and dec + + 183 1782-1793 12 deg R*8 f12.6 glon galactic longitude + 184 1794-1805 12 deg R*8 f12.6 glat galactic latitude + 185 1806-1817 12 deg R*8 f12.7 elon ecliptic longitude + 186 1818-1829 12 deg R*8 f12.7 elat ecliptic latitude + +# The reject table also includes + + 187 1830-1835 6 -- int i6 P Flag to indicate if source is primary in tile + + +NOTES +1.) Delon may have a bias correction for a systematic error induced by residual + PSF errors that manifest themselves in ascending-descending differences; see + the "-pb" command line option of the mrgad program. +2.) Delon is defined as descending position minus ascending position in order to + have the proper sign for parallax (~Delon/2). +3.) the usage codes ka, k1, k2, and km have values of 0 - 3 meaning + 0: neither the ascending nor the descending scan provided a solution + 1: only the ascending scan provided a solution + 2: only the descending scan provided a solution + 3: both scans provided solutions which were combined in the relevant way +4.) Delon is the straightforward difference in ecliptic longitude, ignoring the + different effective observation epochs of ascending and descending scans; + par_pm is computed from the motion-solution positions, which are translated + by WPHotpmc to the standard epoch (MJD0), so except for estimation errors, + par_pm is the parallax; the par_stat column is computed by using the + motion estimate to move the ascending stationary-solution position from the + ascending effective observation epoch to that of the descending solution, then + dividing the ecliptic longitude difference by 2. +5.) Delon will be null unless ka = 3; par_pm will be null unless km = 3; + par_stat will be null unless ka = 3 AND km > 0 AND all W?mJDmin/max/mean values + are non-null in both ascending and descending mdex files. + +____________________________________________________________________________________ + +N of M Statistics + +M == number of flux measurements for a given source +N == number of M sources that have SNR >= 3 + +____________________________________________________________________________________ + +Photometry Flags: + +Standard aperture measurement quality flag. This flag +indicates if one or more image pixels in the measurement aperture for this +band is confused with nearby objects, is contaminated by saturated +or otherwise ususable pixels, or is an upper limit. The flag value +is the integer sum of any of following values which correspond to +different conditions. + +value Condition +----- ------------------------------------------------------ + 0 nominal -- no contamination + 1 source confusion -- another source falls within the measurement aperture + 2 bad or fatal pixels: presence of bad pixels in the measurement aperture + (bit 2 or 18 set) + 4 non-zero bit flag tripped (other than 2 or 18) + 8 corruption -- all pixels are flagged as unusable, or the aperture flux is + negative; in the former case, the aperture magnitude is NULL; in the + latter case, the aperture magnitude is a 95% confidence upper limit + 16 saturation -- here are one or more saturated pixels in the measurement + aperture + 32 upper limit -- the magnitude is a 95% confidence upper limit + +combinations: + + 3 source confusion + bad pixels + 5 source confusion + non-zero bit flag + 6 bad pixels + non-zero bit flag + 7 source confusion + bad pixels + non-zero bit flag + 9 source confusion + corruption + 10 bad pixels + corruption + 11 source confusion + bad pixels + corruption + 12 non-zero bit flag + corruption + 13 source confusion + non-zero bit flag + corruption + 14 bad pixels + non-zero bit flag + corruption + 15 source confusion + bad pixels + non-zero bit flag + corruption + 17 source confusion + saturation + 18 bad pixels + saturation + 19 source confusion + bad pixels + saturation + 20 non-zero bit flag + saturation + 21 source confusion + non-zero bit flag + saturation + 22 bad pixels + non-zero bit flag + saturation + 23 source confusion + bad pixels + non-zero bit flag + saturation + 24 corruption + saturation + 25 source confusion + corruption + saturation + 26 bad pixels + corruption + saturation + 27 source confusion + bad pixels + corruption + saturation + 28 non-zero bit flag + corruption + saturation + 29 source confusion + non-zero bit flag + corruption + saturation + 30 bad pixels + non-zero bit flag + corruption + saturation + 31 source confusion + bad pixels + non-zero bit flag + corruption + saturation + +______________________________________________________________________________________________ + +Note about upper limits: + +threshold = 2 * RMS +if ( f < threshold) then + if (f < 0) then + upper_limit = threshold + else + upper_limit = f + threshold + +where f is the flux of the source, RMS is the source flux uncertainty + +The upper limit is reported in the WPRO and standard aperture MAG columns; +the reported uncertainty is set to 9.99 mag. + +______________________________________________________________________________________________ + +Note: Null values are now indicated by the character string "null" + +*/ + + +CREATE TABLE catalogdb.catwise ( + source_name VARCHAR(21), + source_id VARCHAR(25), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + sigra REAL, + sigdec REAL, + sigradec REAL, + wx REAL, + wy REAL, + w1sky REAL, + w1sigsk REAL, + w1conf REAL, + w2sky REAL, + w2sigsk REAL, + w2conf REAL, + w1fitr REAL, + w2fitr REAL, + w1snr REAL, + w2snr REAL, + w1flux DOUBLE PRECISION, + w1sigflux DOUBLE PRECISION, + w2flux DOUBLE PRECISION, + w2sigflux DOUBLE PRECISION, + w1mpro REAL, + w1sigmpro DOUBLE PRECISION, + w1rchi2 DOUBLE PRECISION, + w2mpro REAL, + w2sigmpro DOUBLE PRECISION, + w2rchi2 DOUBLE PRECISION, + rchi2 DOUBLE PRECISION, + nb SMALLINT, + na SMALLINT, + w1Sat DOUBLE PRECISION, + w2Sat DOUBLE PRECISION, + w1mag REAL, + w1sigm REAL, + w1flg INTEGER, + w1Cov REAL, + w2mag REAL, + w2sigm REAL, + w2flg INTEGER, + w2Cov REAL, + w1mag_1 REAL, + w1sigm_1 REAL, + w1flg_1 BIGINT, + w2mag_1 REAL, + w2sigm_1 REAL, + w2flg_1 BIGINT, + w1mag_2 REAL, + w1sigm_2 REAL, + w1flg_2 BIGINT, + w2mag_2 REAL, + w2sigm_2 REAL, + w2flg_2 BIGINT, + w1mag_3 REAL, + w1sigm_3 REAL, + w1flg_3 BIGINT, + w2mag_3 REAL, + w2sigm_3 REAL, + w2flg_3 BIGINT, + w1mag_4 REAL, + w1sigm_4 REAL, + w1flg_4 BIGINT, + w2mag_4 REAL, + w2sigm_4 REAL, + w2flg_4 BIGINT, + w1mag_5 REAL, + w1sigm_5 REAL, + w1flg_5 BIGINT, + w2mag_5 REAL, + w2sigm_5 REAL, + w2flg_5 BIGINT, + w1mag_6 REAL, + w1sigm_6 REAL, + w1flg_6 BIGINT, + w2mag_6 REAL, + w2sigm_6 REAL, + w2flg_6 BIGINT, + w1mag_7 REAL, + w1sigm_7 REAL, + w1flg_7 BIGINT, + w2mag_7 REAL, + w2sigm_7 REAL, + w2flg_7 BIGINT, + w1mag_8 REAL, + w1sigm_8 REAL, + w1flg_8 BIGINT, + w2mag_8 REAL, + w2sigm_8 REAL, + w2flg_8 BIGINT, + w1NM BIGINT, + w1M INTEGER, + w1magP REAL, + w1sigP1 REAL, + w1sigP2 REAL, + w1k REAL, + w1Ndf INTEGER, + w1mLQ REAL, + w1mJDmin DOUBLE PRECISION, + w1mJDmax DOUBLE PRECISION, + w1mJDmean DOUBLE PRECISION, + w2NM BIGINT, + w2M INTEGER, + w2magP REAL, + w2sigP1 REAL, + w2sigP2 REAL, + w2k REAL, + w2Ndf INTEGER, + w2mLQ REAL, + w2mJDmin DOUBLE PRECISION, + w2mJDmax DOUBLE PRECISION, + w2mJDmean DOUBLE PRECISION, + rho12 SMALLINT, + q12 SMALLINT, + nIters BIGINT, + nSteps BIGINT, + mdetID BIGINT, + p1 DOUBLE PRECISION, + p2 DOUBLE PRECISION, + MeanObsMJD DOUBLE PRECISION, + ra_pm DOUBLE PRECISION, + dec_pm DOUBLE PRECISION, + sigra_pm REAL, + sigdec_pm REAL, + sigradec_pm REAL, + PMRA REAL, + PMDec REAL, + sigPMRA REAL, + sigPMDec REAL, + w1snr_pm REAL, + w2snr_pm REAL, + w1flux_pm DOUBLE PRECISION, + w1sigflux_pm DOUBLE PRECISION, + w2flux_pm DOUBLE PRECISION, + w2sigflux_pm DOUBLE PRECISION, + w1mpro_pm REAL, + w1sigmpro_pm REAL, + w1rchi2_pm DOUBLE PRECISION, + w2mpro_pm REAL, + w2sigmpro_pm REAL, + w2rchi2_pm DOUBLE PRECISION, + rchi2_pm DOUBLE PRECISION, + pmcode VARCHAR(5), + nIters_pm BIGINT, + nSteps_pm BIGINT, + dist REAL, + dw1mag REAL, + rch2w1 REAL, + dw2mag REAL, + rch2w2 REAL, + elon_avg DOUBLE PRECISION, + elonSig REAL, + elat_avg DOUBLE PRECISION, + elatSig REAL, + Delon REAL, + DelonSig REAL, + Delat REAL, + DelatSig REAL, + DelonSNR REAL, + DelatSNR REAL, + chi2pmra DOUBLE PRECISION, + chi2pmdec DOUBLE PRECISION, + ka INTEGER, + k1 INTEGER, + k2 INTEGER, + km INTEGER, + par_pm REAL, + par_pmSig REAL, + par_stat REAL, + par_sigma REAL, + dist_x DOUBLE PRECISION, + cc_flags VARCHAR(16), + w1cc_map BIGINT, + w1cc_map_str VARCHAR(20), + w2cc_map BIGINT, + w2cc_map_str VARCHAR(20), + n_aw INTEGER, + ab_flags VARCHAR(9), + w1ab_map BIGINT, + w1ab_map_str VARCHAR(13), + w2ab_map BIGINT, + w2ab_map_str VARCHAR(13), + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + elon DOUBLE PRECISION, + elat DOUBLE PRECISION +); + + +CREATE TABLE catalogdb.catwise_reject AS (SELECT * FROM catalogdb.catwise) WITH NO DATA; +ALTER TABLE catalogdb.catwise_reject ADD COLUMN p BOOLEAN; diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_download b/schema/sdss5db/catalogdb/CatWISE/CatWISE_download new file mode 100644 index 00000000..7c66d07b --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_download @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +wget -r -nH --no-parent --reject="index.html*" -P ./ https://portal.nersc.gov/project/cosmo/data/CatWISE/ diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql new file mode 100644 index 00000000..c5576cf2 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql @@ -0,0 +1,16 @@ + +-- PKs + +ALTER TABLE catalogdb.catwise ADD PRIMARY KEY (source_name); +ALTER TABLE catalogdb.catwise_reject ADD PRIMARY KEY (source_name); + + +-- Indexes + +CREATE INDEX ON catalogdb.catwise (q3c_ang2ipix(ra, dec)); +CLUSTER catwise_q3c_ang2ipix_idx ON catalogdb.catwise; +ANALYZE catalogdb.catwise; + +CREATE INDEX ON catalogdb.catwise_reject (q3c_ang2ipix(ra, dec)); +CLUSTER catwise_reject_q3c_ang2ipix_idx ON catalogdb.catwise_reject; +ANALYZE catalogdb.catwise_reject; diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load new file mode 100755 index 00000000..269f2a6b --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load @@ -0,0 +1,13 @@ +#! /usr/bin/env bash +# encoding: utf-8 + + +path=/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/CatWISE + +for dir in $path/prelim/*/ ; do + echo $dir + # gzcat $dir/*.tbl.gz | sed -e '/^[\\|]/ d' | psql -U sdss sdss5db -c "\COPY catalogdb.catwise FROM STDIN WITH CSV DELIMITER ' ' NULL 'null';" + ls $dir/*.tbl.gz | parallel -j4 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py {}" + ls $dir/*_cat_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c \"\COPY catalogdb.catwise FROM '{}' WITH CSV HEADER NULL '\N';\"" + ls $dir/*_rej_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c \"\COPY catalogdb.catwise_reject FROM '{}' WITH CSV HEADER NULL '\N';\"" +done diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py new file mode 100644 index 00000000..bd3756f7 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-23 +# @Filename: CatWISE_load_helper.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os +import sys + +import astropy.table +from astropy.io import ascii + + +def main(): + + file_ = sys.argv[1] + dest = file_ + '.csv' + + if os.path.exists(dest): + return + + table = astropy.table.Table.read(file_, format='ascii.ipac') + table.meta = {} + table.write(dest, format='csv', fill_values=[(ascii.masked, '\\N')]) + + return + + +if __name__ == '__main__': + main() From 343981df3a08bb81b8347a2cdc7914a932bd44af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 23 Mar 2020 17:42:49 -0700 Subject: [PATCH 107/197] Fix meta for CatWISE --- python/sdssdb/peewee/sdss5db/catalogdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index da14a2f7..1bf33d3c 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -521,15 +521,15 @@ class Meta: class CatWISE(CatalogdbModel): class Meta: - table_name = 'tic_v8' - schema = 'catwise' + table_name = 'catwise' + schema = 'catalogdb' class CatWISEReject(CatalogdbModel): class Meta: - table_name = 'tic_v8' - schema = 'catwise_reject' + table_name = 'catwise_reject' + schema = 'catalogdb' _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) From cc06ba60b87df7db0db1796bfab034c68358f254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 23 Mar 2020 18:46:58 -0600 Subject: [PATCH 108/197] Only create CSV for CatWISE for now --- schema/sdss5db/catalogdb/CatWISE/CatWISE_load | 6 +++--- schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load index 269f2a6b..13f7c4f7 100755 --- a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load @@ -7,7 +7,7 @@ path=/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/CatWISE for dir in $path/prelim/*/ ; do echo $dir # gzcat $dir/*.tbl.gz | sed -e '/^[\\|]/ d' | psql -U sdss sdss5db -c "\COPY catalogdb.catwise FROM STDIN WITH CSV DELIMITER ' ' NULL 'null';" - ls $dir/*.tbl.gz | parallel -j4 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py {}" - ls $dir/*_cat_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c \"\COPY catalogdb.catwise FROM '{}' WITH CSV HEADER NULL '\N';\"" - ls $dir/*_rej_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c \"\COPY catalogdb.catwise_reject FROM '{}' WITH CSV HEADER NULL '\N';\"" + ls $dir/*.tbl.gz | parallel -j15 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py {}" + # ls $dir/*_cat_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c '\copy catalogdb.catwise FROM {} WITH CSV HEADER NULL \'\N\';'" + # ls $dir/*_rej_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c '\copy catalogdb.catwise_reject FROM {} WITH CSV HEADER NULL \'\N\';'" done diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py index bd3756f7..d173e496 100644 --- a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py @@ -6,7 +6,6 @@ # @Filename: CatWISE_load_helper.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) -import os import sys import astropy.table @@ -18,12 +17,10 @@ def main(): file_ = sys.argv[1] dest = file_ + '.csv' - if os.path.exists(dest): - return - table = astropy.table.Table.read(file_, format='ascii.ipac') table.meta = {} - table.write(dest, format='csv', fill_values=[(ascii.masked, '\\N')]) + table.write(dest, format='csv', fill_values=[(ascii.masked, '\\N')], + overwrite=True) return From 428c8113be1c85603a0d1030d3592b48216a1568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 23 Mar 2020 22:47:01 -0600 Subject: [PATCH 109/197] No need to cast sdss in TIC --- schema/sdss5db/catalogdb/foreignKeys.sql | 4 ++-- schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index aba32cd6..3fa71772 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -118,12 +118,12 @@ ALTER TABLE catalogdb.tic_v8 CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT sdss_int_fk + ADD CONSTRAINT sdss_fk FOREIGN KEY (sdss_int) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss_int); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT gaia_int_fk diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index e60c2a54..8c638215 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -4,9 +4,6 @@ ALTER TABLE catalogdb.tic_v8 ADD COLUMN gaia_int BIGINT; UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; -ALTER TABLE catalogdb.tic_v8 ADD COLUMN sdss_int BIGINT; -UPDATE catalogdb.tic_v8 SET sdss_int = sdss::BIGINT; - -- Indices From 42a82fd3c51b268aa8b0fa38ee23e5c2594dfe1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 23 Mar 2020 22:48:35 -0600 Subject: [PATCH 110/197] Create indexes on fk first, then fk --- schema/sdss5db/catalogdb/foreignKeys.sql | 56 ++++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 3fa71772..56b3391c 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -20,13 +20,15 @@ ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 -- gaiadr2_tmass_best_neighbour +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (source_id ASC); + ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (source_id ASC); +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT tmass_pts_key_fk @@ -34,29 +36,27 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour REFERENCES catalogdb.twomass_psc (pts_key) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); - -- sdss_dr14_specobj +CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); + ALTER TABLE catalogdb.sdss_dr14_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); - -- sdss_dr16_specobj +CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); + ALTER TABLE catalogdb.sdss_dr16_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); - -- gaia_unwise_agn @@ -66,14 +66,14 @@ ALTER TABLE catalogdb.gaia_unwise_agn REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; +CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); + ALTER TABLE catalogdb.gaia_unwise_agn ADD CONSTRAINT unwise_objid_fk FOREIGN KEY (unwise_objid) REFERENCES catalogdb.unwise (unwise_objid) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); - -- sdss_dr14_apogeeStarVisit @@ -101,29 +101,31 @@ ALTER TABLE catalogdb.sdss_dr14_ascapStar -- tic_v8 +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); + ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT tyc_fk - FOREIGN KEY (tyc) - REFERENCES catalogdb.tycho2 (name) + ADD CONSTRAINT sdss_fk + FOREIGN KEY (sdss_int) + REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT twomass_fk - FOREIGN KEY (twomass) - REFERENCES catalogdb.twomass_psc (designation) + ADD CONSTRAINT tyc_fk + FOREIGN KEY (tyc) + REFERENCES catalogdb.tycho2 (name) ON UPDATE CASCADE ON DELETE CASCADE; CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT sdss_fk - FOREIGN KEY (sdss_int) - REFERENCES catalogdb.sdss_dr13_photoobj (objid) + ADD CONSTRAINT twomass_fk + FOREIGN KEY (twomass) + REFERENCES catalogdb.twomass_psc (designation) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT gaia_int_fk @@ -131,7 +133,7 @@ ALTER TABLE catalogdb.tic_v8 REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT allwise_fk @@ -139,7 +141,7 @@ ALTER TABLE catalogdb.tic_v8 REFERENCES catalogdb.allwise (designation) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT kic_fk @@ -147,18 +149,18 @@ ALTER TABLE catalogdb.tic_v8 REFERENCES catalogdb.kepler_input_10 (kic_kepler_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); - -- bhm_spiders_agn_superset, bhm_spiders_clusters_superset +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); + ALTER TABLE catalogdb.bhm_spiders_agn_superset ADD CONSTRAINT gaia_dr2_source_id_fk FOREIGN KEY (gaia_dr2_source_id) REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); ALTER TABLE catalogdb.bhm_spiders_agn_superset ADD CONSTRAINT ls_id_fk @@ -166,7 +168,7 @@ ALTER TABLE catalogdb.bhm_spiders_agn_superset REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); ALTER TABLE catalogdb.bhm_spiders_clusters_superset ADD CONSTRAINT gaia_dr2_source_id_fk @@ -174,12 +176,10 @@ ALTER TABLE catalogdb.bhm_spiders_clusters_superset REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); ALTER TABLE catalogdb.bhm_spiders_clusters_superset ADD CONSTRAINT ls_id_fk FOREIGN KEY (ls_id) REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; - -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); From c366158f5832bf0ae2f17e6f12ba72ff0eaceb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 00:29:47 -0600 Subject: [PATCH 111/197] Add WATCHDOG --- python/sdssdb/peewee/sdss5db/catalogdb.py | 14 ++++++++ .../legacy_survey/dr8/legacy_survey_index.sql | 4 +-- .../catalogdb/mwm_small/watchdog/watchdog.py | 32 +++++++++++++++++++ .../catalogdb/mwm_small/watchdog/watchdog.sql | 30 +++++++++++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100755 schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 1bf33d3c..90df9ba2 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -532,5 +532,19 @@ class Meta: schema = 'catalogdb' +class Watchdog(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'watchdog' + schema = 'catalogdb' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index 3a7177fd..a3b56db3 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -4,6 +4,6 @@ UPDATE catalogdb.legacy_survey_dr8 SET lsid = objid + brickid * 2^16 + release * ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (lsid); -CREATE INDEX on catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx on catalogdb.legacy_survey_dr8; +CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py new file mode 100755 index 00000000..501f3057 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: watchdog.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/watchdog_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'watchdog', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql new file mode 100644 index 00000000..7b665e1e --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql @@ -0,0 +1,30 @@ +/* + +WATCHDOG Galactic Black Hole Binaries (B. E. Tetarenko et al 2016 ApJS 222 15) + +*/ + +CREATE TABLE catalogdb.watchdog ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.watchdog (q3c_ang2ipix(radeg, decdeg)); +CLUSTER watchdog_q3c_ang2ipix_idx ON catalogdb.watchdog; +ANALYZE catalogdb.watchdog; + +ALTER TABLE catalogdb.watchdog + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; From 7c4a6e4cab38d826538669262c8170c67e36054d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 00:39:25 -0600 Subject: [PATCH 112/197] Add BlackCAT --- python/sdssdb/peewee/sdss5db/catalogdb.py | 14 ++++++++ .../catalogdb/mwm_small/blackcat/backcat.py | 32 +++++++++++++++++++ .../catalogdb/mwm_small/blackcat/blackcat.sql | 30 +++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 90df9ba2..7ddfb860 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -546,5 +546,19 @@ class Meta: schema = 'catalogdb' +class BlackCAT(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'blackcat' + schema = 'catalogdb' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py b/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py new file mode 100755 index 00000000..fc169b68 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: blackcat.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/blackcat_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'blackcat', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql new file mode 100644 index 00000000..bc931a15 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql @@ -0,0 +1,30 @@ +/* + +BlackCAT Stellar-Mass Black Holes in X-ray Binaries http://www.astro.puc.cl/BlackCAT/ + +*/ + +CREATE TABLE catalogdb.blackcat ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.blackcat (q3c_ang2ipix(radeg, decdeg)); +CLUSTER blackcat_q3c_ang2ipix_idx ON catalogdb.blackcat; +ANALYZE catalogdb.blackcat; + +ALTER TABLE catalogdb.blackcat + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; From 7ef8fd5b347cb92569525ec9e199eeecb453d843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 00:48:19 -0600 Subject: [PATCH 113/197] Add XRay Pulsars --- python/sdssdb/peewee/sdss5db/catalogdb.py | 14 ++++++++ .../mwm_small/xray_pulsars/xray_pulsars.py | 32 +++++++++++++++++++ .../mwm_small/xray_pulsars/xray_pulsars.sql | 30 +++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 7ddfb860..d72e2968 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -560,5 +560,19 @@ class Meta: schema = 'catalogdb' +class XRay_Pulsars(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'xray_pulsars' + schema = 'catalogdb' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py new file mode 100755 index 00000000..7af1a39e --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: xray_pulsars.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/xraypsr_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'xray_pulsars', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql new file mode 100644 index 00000000..59c69fdf --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql @@ -0,0 +1,30 @@ +/* + +Accreting X-ray Binary Pulsars http://www.iasfbo.inaf.it/~mauro/pulsar_list.html + +*/ + +CREATE TABLE catalogdb.xray_pulsars ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.xray_pulsars (q3c_ang2ipix(radeg, decdeg)); +CLUSTER xray_pulsars_q3c_ang2ipix_idx ON catalogdb.xray_pulsars; +ANALYZE catalogdb.xray_pulsars; + +ALTER TABLE catalogdb.xray_pulsars + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; From d1e028c88cc25eb370f3ebf33d5a4e61c0167e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 01:15:45 -0600 Subject: [PATCH 114/197] Add several more small catalogues --- python/sdssdb/peewee/sdss5db/catalogdb.py | 42 +++++++++++++++++++ .../sdss5db/catalogdb/mwm_small/atnf/atnf.py | 32 ++++++++++++++ .../sdss5db/catalogdb/mwm_small/atnf/atnf.sql | 32 ++++++++++++++ .../galactic_millisecond_pulsars.py | 32 ++++++++++++++ .../galactic_millisecond_pulsars.sql | 30 +++++++++++++ .../sdss5db/catalogdb/mwm_small/lmxb/lmxb.py | 32 ++++++++++++++ .../sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql | 30 +++++++++++++ 7 files changed, 230 insertions(+) create mode 100755 schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql create mode 100755 schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql create mode 100755 schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py create mode 100644 schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index d72e2968..4bcd36f3 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -574,5 +574,47 @@ class Meta: schema = 'catalogdb' +class LowMassXRayBinaries(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'lmxb' + schema = 'catalogdb' + + +class GalacticMillisecondPulsars(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'galactic_millisecond_pulsars' + schema = 'catalogdb' + + +class ATNF(CatalogdbModel): + + name = TextField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'atnf' + schema = 'catalogdb' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py new file mode 100755 index 00000000..07c1ba64 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: atnf.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/atnfpsr_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'atnf', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql new file mode 100644 index 00000000..51990c2a --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql @@ -0,0 +1,32 @@ +/* + +ATNF Pulsar Catalog https://bit.ly/3aajZF7 + +*/ + +CREATE TABLE catalogdb.atnf ( + name TEXT PRIMARY KEY, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT +); + +CREATE INDEX ON catalogdb.atnf (q3c_ang2ipix(radeg, decdeg)); +CLUSTER atnf_q3c_ang2ipix_idx ON catalogdb.atnf; +ANALYZE catalogdb.atnf; + +CREATE INDEX ON catalogdb.atnf USING BTREE (gaia_source_id); + +ALTER TABLE catalogdb.atnf + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py new file mode 100755 index 00000000..b8a65884 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: galactic_millisecond_pulsars.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/msp_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'galactic_millisecond_pulsars', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql new file mode 100644 index 00000000..406eaa95 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql @@ -0,0 +1,30 @@ +/* + +Galactic Millisecond Pulsar Catalog https://bit.ly/33NXK5C + +*/ + +CREATE TABLE catalogdb.galactic_millisecond_pulsars ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.galactic_millisecond_pulsars (q3c_ang2ipix(radeg, decdeg)); +CLUSTER galactic_millisecond_pulsars_q3c_ang2ipix_idx ON catalogdb.galactic_millisecond_pulsars; +ANALYZE catalogdb.galactic_millisecond_pulsars; + +ALTER TABLE catalogdb.galactic_millisecond_pulsars + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py new file mode 100755 index 00000000..117ed4c6 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: lmxb.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/lmxb_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'lmxb', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql new file mode 100644 index 00000000..c1e4f819 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql @@ -0,0 +1,30 @@ +/* + +Ritter & Kolb Catalog of Low Mass X-ray Binaries https://bit.ly/3agEcZT + +*/ + +CREATE TABLE catalogdb.lmxb ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.lmxb (q3c_ang2ipix(radeg, decdeg)); +CLUSTER lmxb_q3c_ang2ipix_idx ON catalogdb.lmxb; +ANALYZE catalogdb.lmxb; + +ALTER TABLE catalogdb.lmxb + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; From b9cbf9aabb25c0bb28d9ba4e98cc4500851ed4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 02:06:28 -0600 Subject: [PATCH 115/197] Prepare to load SkyMapper --- python/sdssdb/peewee/sdss5db/catalogdb.py | 40 ++++++ schema/sdss5db/catalogdb/foreignKeys.sql | 25 ++++ .../catalogdb/skymapper/dr1.1/skymapper.sql | 117 ++++++++++++++++++ .../skymapper/dr1.1/skymapper_indexes.sql | 4 + .../catalogdb/skymapper/dr1.1/skymapper_load | 4 + 5 files changed, 190 insertions(+) create mode 100644 schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql create mode 100644 schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql create mode 100755 schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 4bcd36f3..f6d79a17 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -616,5 +616,45 @@ class Meta: schema = 'catalogdb' +class SkyMapper_DR1_1(CatalogdbModel): + + allwise = ForeignKeyField(AllWise, field='cntr', + column_name='allwise_cntr', + object_id_name='allwise_cntr', + backref='skymapper') + + gaia1 = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_dr2_id1', + object_id_name='gaia_dr2_id1', + backref='skymapper1') + + gaia2 = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_dr2_id2', + object_id_name='gaia_dr2_id2', + backref='skymapper2') + + @property + def twomass1(self): + """Returns the closest 2MASS PSC source, if defined.""" + + if self.twomass_cat1 != 'PSC': + return None + + return TwoMassPSC.get(pts_key=self.twomass_key1) + + @property + def twomass2(self): + """Returns the second closest 2MASS PSC source, if defined.""" + + if self.twomass_cat2 != 'PSC': + return None + + return TwoMassPSC.get(pts_key=self.twomass_key2) + + class Meta: + table_name = 'skymapper_dr1_1' + schema = 'catalogdb' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 56b3391c..e90021aa 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -183,3 +183,28 @@ ALTER TABLE catalogdb.bhm_spiders_clusters_superset FOREIGN KEY (ls_id) REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; + + +-- skymapper_dr1_1 + +CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (allwise_cntr); +CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id1); +CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id2); + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT allwise_cntr_fk + FOREIGN KEY (allwise_cntr) + REFERENCES catalogdb.allwise (cntr) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT gaia_dr2_id1_fk + FOREIGN KEY (gaia_dr2_id1) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT gaia_dr2_id2_fk + FOREIGN KEY (gaia_dr2_id2) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql new file mode 100644 index 00000000..a9bd806c --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql @@ -0,0 +1,117 @@ +/* + +SkyMapper Southern Sky Survey DR1.1 - http://skymapper.anu.edu.au + +To download the file see: http://skymapper.anu.edu.au/_data/DR1.1/ +Table columns: http://skymapper.anu.edu.au/table-browser/ (different order as the CSVs!) + +At Utah: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/skymapper/dr1.1 + +*/ + +CREATE TABLE skymapper_dr1_1 ( + object_id BIGINT PRIMARY KEY, + raj2000 DOUBLE PRECISION, + dej2000 DOUBLE PRECISION, + e_raj2000 INTEGER, + e_dej2000 INTEGER, + smss_j VARCHAR(18), + mean_epoch DOUBLE PRECISION, + rms_epoch REAL, + glon REAL, + glat REAL, + flags SMALLINT, + nimaflags INTEGER, + ngood SMALLINT, + ngood_min SMALLINT, + nch_max SMALLINT, + u_flags SMALLINT, + u_nimaflags INTEGER, + u_ngood SMALLINT, + u_nch SMALLINT, + u_nvisit SMALLINT, + v_flags SMALLINT, + v_nimaflags INTEGER, + v_ngood SMALLINT, + v_nch SMALLINT, + v_nvisit SMALLINT, + g_flags SMALLINT, + g_nimaflags INTEGER, + g_ngood SMALLINT, + g_nch SMALLINT, + g_nvisit SMALLINT, + r_flags SMALLINT, + r_nimaflags INTEGER, + r_ngood SMALLINT, + r_nch SMALLINT, + r_nvisit SMALLINT, + i_flags SMALLINT, + i_nimaflags INTEGER, + i_ngood SMALLINT, + i_nch SMALLINT, + i_nvisit SMALLINT, + z_flags SMALLINT, + z_nimaflags INTEGER, + z_ngood SMALLINT, + z_nch SMALLINT, + z_nvisit SMALLINT, + class_star REAL, + radius_petro REAL, + a REAL, + e_a REAL, + b REAL, + e_b REAL, + pa REAL, + e_pa REAL, + u_psf REAL, + e_u_psf REAL, + u_petro REAL, + e_u_petro REAL, + v_psf REAL, + e_v_psf REAL, + v_petro REAL, + e_v_petro REAL, + g_psf REAL, + e_g_psf REAL, + g_petro REAL, + e_g_petro REAL, + r_psf REAL, + e_r_psf REAL, + r_petro REAL, + e_r_petro REAL, + i_psf REAL, + e_i_psf REAL, + i_petro REAL, + e_i_petro REAL, + z_psf REAL, + e_z_psf REAL, + z_petro REAL, + e_z_petro REAL, + ebmv_sfd REAL, + prox REAL, + prox_id BIGINT, + edr_id BIGINT, + edr_dist REAL, + twomass_key1 BIGINT, + twomass_dist1 REAL, + twomass_cat1 VARCHAR(3), + twomass_key2 BIGINT, + twomass_dist2 REAL, + twomass_cat2 VARCHAR(3), + allwise_cntr BIGINT, + allwise_dist REAL, + ucac4_mpos BIGINT, + ucac4_dist REAL, + apass_recno BIGINT, + apass_dist REAL, + gaia_dr1_id BIGINT, + gaia_dr1_dist REAL, + ps1_dr1_id BIGINT, + ps1_dr1_dist REAL, + galex_bcs_id BIGINT, + galex_bcs_dist REAL, + gaia_dr2_id1 BIGINT, + gaia_dr2_dist1 REAL, + gaia_dr2_id2 BIGINT, + gaia_dr2_dist2 REAL +); diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql new file mode 100644 index 00000000..eed4f41e --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql @@ -0,0 +1,4 @@ + +CREATE INDEX ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, decj2000)); +CLUSTER skymapper_dr1_1_q3c_ang2ipix_idx ON catalogdb.skymapper_dr1_1; +ANALYZE catalogdb.skymapper_dr1_1; diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load new file mode 100755 index 00000000..64bc620d --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +ls $CATALOGDB_DIR/skymapper/dr1.1/*.csv.gz | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.skymapper_dr1_1 FROM program 'zcat {}' WITH CSV HEADER;\"" From 98442e8a9476b8b17f254699992960a0849d8537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 01:46:06 -0700 Subject: [PATCH 116/197] Prepare to load apogeeAllStarMerge --- schema/sdss5db/catalogdb/foreignKeys.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index e90021aa..d5d8c232 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -184,6 +184,7 @@ ALTER TABLE catalogdb.bhm_spiders_clusters_superset REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; +<<<<<<< Updated upstream -- skymapper_dr1_1 @@ -207,4 +208,15 @@ ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT gaia_dr2_id2_fk FOREIGN KEY (gaia_dr2_id2) REFERENCES catalogdb.gaia_dr2_source (source_id) +======= +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); + + +-- sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx + +ALTER TABLE catalogdb.bhm_spiders_clusters_superset + ADD CONSTRAINT ls_id_fk + FOREIGN KEY (ls_id) + REFERENCES catalogdb.legacy_survey_dr8 (ls_id) +>>>>>>> Stashed changes ON UPDATE CASCADE ON DELETE CASCADE; From e2810c0aa724dd8c5a262dcef9ca139451a6bd14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 01:49:25 -0700 Subject: [PATCH 117/197] Actually add changes for apogeeAllStarMerge --- python/sdssdb/peewee/sdss5db/catalogdb.py | 13 +++ schema/sdss5db/catalogdb/foreignKeys.sql | 13 ++- .../r13/AllStarMerge.sql | 84 +++++++++++++++++++ .../r13/AllStarMerge_index.sql | 8 ++ .../r13/AllStarMerge_load.py | 39 +++++++++ 5 files changed, 149 insertions(+), 8 deletions(-) create mode 100644 schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql create mode 100644 schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql create mode 100755 schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index f6d79a17..4e3c892a 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -184,6 +184,19 @@ class Meta: schema = 'catalogdb' +class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel): + + apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, + field='apogee_id', + column_name='apogee_id', + object_id_name='apogee_id', + backref='allstar') + + class Meta: + table_name = 'sdss_apogeeAllStarMerge_r13' + schema = 'catalogdb' + + class SDSS_DR14_SpecObj(CatalogdbModel): photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index d5d8c232..7011dda8 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -184,7 +184,6 @@ ALTER TABLE catalogdb.bhm_spiders_clusters_superset REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; -<<<<<<< Updated upstream -- skymapper_dr1_1 @@ -208,15 +207,13 @@ ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT gaia_dr2_id2_fk FOREIGN KEY (gaia_dr2_id2) REFERENCES catalogdb.gaia_dr2_source (source_id) -======= CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); --- sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx +-- sdss_apogeeAllStarMerge_r13 -ALTER TABLE catalogdb.bhm_spiders_clusters_superset - ADD CONSTRAINT ls_id_fk - FOREIGN KEY (ls_id) - REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ->>>>>>> Stashed changes +ALTER TABLE catalogdb.sdss_apogeeAllStarMerge_r13 + ADD CONSTRAINT apogee_id_fk + FOREIGN KEY (apogee_id) + REFERENCES catalogdb.sdss_dr14_apogeeStar (apogee_id) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql new file mode 100644 index 00000000..e70827a6 --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql @@ -0,0 +1,84 @@ +/* + +SDSS AllStarMerge r13 + +Provided by Nicholas Troup - https://bit.ly/39emJ31 + +At Utah: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/sdssApogeeAllStarMerge/r13/ +Catalogue generated via IDL script: aporbitprep_mergestar.pro + +*/ + +CREATE TABLE catalogdb.sdss_apogeeAllStarMerge_r13 ( + apogee_id VARCHAR(18) PRIMARY KEY, + nvisits SMALLINT, + nentries INTEGER, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pm_src VARCHAR(4), + j REAL, + j_err REAL, + h REAL, + h_err REAL, + k REAL, + k_err REAL, + ak REAL, + vhelio_avg REAL, + vhelio_err REAL, + vscatter REAL, + sig_rvvar REAL, + baseline REAL, + mean_fiber REAL, + sig_fiber REAL, + apstar_ids TEXT, + nvisit_apstar SMALLINT[20], + weights REAL[20], + visits TEXT, + fields TEXT, + surveys TEXT, + telescopes TEXT, + location_id SMALLINT[20], + commiss SMALLINT[20], + targflags TEXT, + starflags TEXT, + aspcapflags TEXT, + teff REAL, + teff_err REAL, + logg REAL, + logg_err REAL, + feh REAL, + feh_err REAL, + param REAL[9], + param_cov REAL[9], + fparam REAL[9], + fparam_cov REAL[9], + param_symbol VARCHAR(15)[9], + x_h REAL[26], + x_h_err REAL[26], + felem REAL[26], + felem_err REAL[26], + elem_symbol VARCHAR(4)[26], + felemtoh SMALLINT[31], + startype VARCHAR(1), + vjitter REAL, + dist REAL, + dist_err REAL, + dist_src VARCHAR(1), + dist_srclist VARCHAR(1), + mstar REAL, + mstar_err REAL, + rstar REAL, + rstar_err REAL, + mstar_src VARCHAR(1), + jd DOUBLE PRECISION[60], + rv DOUBLE PRECISION[60], + rv_err DOUBLE PRECISION[60], + rv_src VARCHAR(6)[60], + snr REAL[60], + fiberid SMALLINT[60], + visit_pk INTEGER[60] +); diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql new file mode 100644 index 00000000..3447f91c --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql @@ -0,0 +1,8 @@ + +CREATE INDEX ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; +ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; + +CREATE INDEX sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(glon, glat)); +CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; +ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py new file mode 100755 index 00000000..6ae9df07 --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-13 +# @Filename: AllStarMerge_load.py.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + file_ = os.environ['CATALOGDB_DIR'] + 'sdssApogeeAllStarMerge/r13/allStarMerge-r13-l33-58814.fits' # noqa + + data = astropy.table.Table.read(file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, overwrite=True) + del data + + cursor = database.cursor() + fileobj = open(file_ + '.csv') + fileobj.readline() # Read header + cursor.copy_from(fileobj, 'catalogdb.sdss_apogeeAllStarMerge_r13', sep=',') + database.commit() + + +if __name__ == '__main__': + + main() From 6c55ad198edfc6e653b22924c9a9d073edaff111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 13:24:28 -0700 Subject: [PATCH 118/197] Use string substitution in to_csv (same as copy_data) --- python/sdssdb/utils/ingest.py | 84 +++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index 8afaa48d..8006ce65 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -9,7 +9,9 @@ # @Last modified by: José Sánchez-Gallego (gallegoj@uw.edu) # @Last modified time: 2019-09-23 11:47:00 +import functools import io +import multiprocessing import os import re import warnings @@ -37,7 +39,7 @@ } -def to_csv(table, path, header=True, convert_arrays=True, **kwargs): +def to_csv(table, path, header=True, delimiter='\t', use_multiprocessing=False, workers=4): """Creates a PostgreSQL-valid CSV file from a table, handling arrays. Parameters @@ -48,32 +50,31 @@ def to_csv(table, path, header=True, convert_arrays=True, **kwargs): The path to which to write the CSV file. header : bool Whether to add a header with the column names. - convert_arrays : bool - If `True`, the arrays in the table are converted into a - PostgreSQL-valid string. - kwargs : dict - Other arguments to pass to `~astropy.table.Table.write`. + delimiter : str + The delimiter between columns in the CSV files. + use_multiprocessing : bool + Whether to use multiple cores. The rows of the resulting file will not + have the same ordering as the original table. + workers : int + How many workers to use with multiprocessing. """ - if convert_arrays: - columns = [col for col in table.colnames if table[col].ndim > 1] - for col in columns: - index = table.index_column(col) - col_str = ['{' + ','.join(map(str, xx)) + '}' for xx in table[col]] - table.remove_column(col) - table.add_column(astropy.table.Column(col_str, col), index) - - if header: - write_kwargs = {'format': 'csv', 'fast_writer': True} + if use_multiprocessing: + pool = multiprocessing.Pool(workers) + tmp_list = pool.map(functools.partial(convert_row_to_psql, + delimiter=delimiter), + table, chunksize=1000) else: - write_kwargs = {'format': 'ascii.no_header', - 'delimiter': ',', - 'fast_writer': True} + tmp_list = [convert_row_to_psql(row, delimiter=delimiter) for row in table] + + csv_str = '\n'.join(tmp_list) - write_kwargs.update(kwargs) + if header: + csv_str = delimiter.join(table.colnames) + '\n' + csv_str - table.write(path, **write_kwargs) + unit = open(path, 'w') + unit.write(csv_str) def table_exists(table_name, connection, schema=None): @@ -204,6 +205,28 @@ class Meta: return type(str(table_name), (BaseModel,), attrs) +def convert_row_to_psql(row, delimiter='\t'): + """Concerts an astropy table row to a Postgresql-valid CSV string.""" + + row_data = [] + + for col_value in row: + if numpy.isscalar(col_value): + row_data.append(str(col_value)) + elif numpy.ma.is_masked(col_value): + row_data.append('') + else: + if col_value.dtype.base.str[1] == 'S': + col_value = col_value.astype('U') + row_data.append( + str(col_value.tolist()) + .replace('\n', '') + .replace('\'', '\"') + .replace('[', '{').replace(']', '}')) + + return delimiter.join(row_data) + + def copy_data(data, connection, table_name, schema=None, chunk_size=10000, show_progress=False): """Loads data into a DB table using ``COPY``. @@ -252,24 +275,7 @@ def copy_data(data, connection, table_name, schema=None, chunk_size=10000, for ii in iterable: row = data[ii] - - row_data = [] - - for col_value in row: - if numpy.isscalar(col_value): - row_data.append(str(col_value)) - elif numpy.ma.is_masked(col_value): - row_data.append('') - else: - if col_value.dtype.base.str[1] == 'S': - col_value = col_value.astype('U') - row_data.append( - str(col_value.tolist()) - .replace('\n', '') - .replace('\'', '\"') - .replace('[', '{').replace(']', '}')) - - tmp_list.append('\t'.join(row_data)) + tmp_list.append(convert_row_to_psql(row)) chunk += 1 # If we have reached a chunk commit point, or this is the last item, From a6d92882b7a276be9f030153f5b559e39cee0c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 13:24:43 -0700 Subject: [PATCH 119/197] Fix issues in loading of sdssApogeeAllStarMerge --- .../catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql | 4 ++-- .../catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql index e70827a6..d491494b 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql @@ -53,9 +53,9 @@ CREATE TABLE catalogdb.sdss_apogeeAllStarMerge_r13 ( feh REAL, feh_err REAL, param REAL[9], - param_cov REAL[9], + param_cov REAL[9][9], fparam REAL[9], - fparam_cov REAL[9], + fparam_cov REAL[9][9], param_symbol VARCHAR(15)[9], x_h REAL[26], x_h_err REAL[26], diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py index 6ae9df07..a260ad69 100755 --- a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py @@ -24,13 +24,13 @@ def main(): data = astropy.table.Table.read(file_) data.meta = {} data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) - to_csv(data, file_ + '.csv', header=True, overwrite=True) + to_csv(data, file_ + '.csv', header=True) del data cursor = database.cursor() fileobj = open(file_ + '.csv') fileobj.readline() # Read header - cursor.copy_from(fileobj, 'catalogdb.sdss_apogeeAllStarMerge_r13', sep=',') + cursor.copy_from(fileobj, 'catalogdb.sdss_apogeeAllStarMerge_r13', sep='\t') database.commit() From 31706d263ca7a18a00b98ea853fb9679e1708f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 14:25:27 -0600 Subject: [PATCH 120/197] Create indixes first when creating TIC fks --- schema/sdss5db/catalogdb/foreignKeys.sql | 27 ++++++++++-------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 7011dda8..1ee45a23 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -102,6 +102,11 @@ ALTER TABLE catalogdb.sdss_dr14_ascapStar -- tic_v8 CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT sdss_fk @@ -109,46 +114,36 @@ ALTER TABLE catalogdb.tic_v8 REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); - ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT tyc_fk FOREIGN KEY (tyc) REFERENCES catalogdb.tycho2 (name) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); - ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT twomass_fk FOREIGN KEY (twomass) REFERENCES catalogdb.twomass_psc (designation) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); - -ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT gaia_int_fk - FOREIGN KEY (gaia_int) - REFERENCES catalogdb.gaia_dr2_source (source_id) - ON UPDATE CASCADE ON DELETE CASCADE; - -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); - ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT allwise_fk FOREIGN KEY (allwise) REFERENCES catalogdb.allwise (designation) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); - ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT kic_fk FOREIGN KEY (kic) REFERENCES catalogdb.kepler_input_10 (kic_kepler_id) ON UPDATE CASCADE ON DELETE CASCADE; +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT gaia_int_fk + FOREIGN KEY (gaia_int) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + -- bhm_spiders_agn_superset, bhm_spiders_clusters_superset From b29215174d2237c09100114783500bf70416b93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 14:52:51 -0600 Subject: [PATCH 121/197] Fix apogeeAllStarMerge FKs --- python/sdssdb/peewee/sdss5db/catalogdb.py | 12 +++++++----- schema/sdss5db/catalogdb/foreignKeys.sql | 9 --------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 4e3c892a..89bec3bb 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -186,11 +186,13 @@ class Meta: class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel): - apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, - field='apogee_id', - column_name='apogee_id', - object_id_name='apogee_id', - backref='allstar') + @property + def apstar(self): + """Returns the stars on `.SDSS_DR14_APOGEE_Star` with matching ``apogee_id``.""" + + return (SDSS_DR14_APOGEE_Star + .select() + .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id)) class Meta: table_name = 'sdss_apogeeAllStarMerge_r13' diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 1ee45a23..d8a0fe40 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -203,12 +203,3 @@ ALTER TABLE catalogdb.skymapper_dr1_1 FOREIGN KEY (gaia_dr2_id2) REFERENCES catalogdb.gaia_dr2_source (source_id) CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); - - --- sdss_apogeeAllStarMerge_r13 - -ALTER TABLE catalogdb.sdss_apogeeAllStarMerge_r13 - ADD CONSTRAINT apogee_id_fk - FOREIGN KEY (apogee_id) - REFERENCES catalogdb.sdss_dr14_apogeeStar (apogee_id) - ON UPDATE CASCADE ON DELETE CASCADE; From 6c4d7fc735cea1dc9970ef0c8843e8895376147a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 15:00:55 -0600 Subject: [PATCH 122/197] Add index on apogee_id --- .../catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql index 3447f91c..b396ac62 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql @@ -6,3 +6,5 @@ ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; CREATE INDEX sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(glon, glat)); CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; + +CREATE INDEX ON catalogdb.sdss_apogeeAllStarMerge_r13 USING BTREE (apogee_id); From cac70220b26c308f1414cc42c92c06b81ccd30b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 24 Mar 2020 15:31:25 -0600 Subject: [PATCH 123/197] Tweak CatWISE loading --- schema/sdss5db/catalogdb/CatWISE/CatWISE_load | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load index 13f7c4f7..d4373a30 100755 --- a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load @@ -6,8 +6,7 @@ path=/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/CatWISE for dir in $path/prelim/*/ ; do echo $dir - # gzcat $dir/*.tbl.gz | sed -e '/^[\\|]/ d' | psql -U sdss sdss5db -c "\COPY catalogdb.catwise FROM STDIN WITH CSV DELIMITER ' ' NULL 'null';" ls $dir/*.tbl.gz | parallel -j15 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py {}" - # ls $dir/*_cat_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c '\copy catalogdb.catwise FROM {} WITH CSV HEADER NULL \'\N\';'" - # ls $dir/*_rej_*.tbl.gz.csv | parallel -j4 "psql -U sdss sdss5db -c '\copy catalogdb.catwise_reject FROM {} WITH CSV HEADER NULL \'\N\';'" + ls $dir/*_cat_*.tbl.gz.csv | parallel -j16 "psql -U sdss sdss5db -c \"\copy catalogdb.catwise FROM '{}' WITH CSV HEADER NULL '\N';\"" + ls $dir/*_rej_*.tbl.gz.csv | parallel -j16 "psql -U sdss sdss5db -c \"\copy catalogdb.catwise_reject FROM '{}' WITH CSV HEADER NULL '\N';\"" done From ac7f3ab88d35acb4c410e7f984793cafc2c279d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 25 Mar 2020 10:27:05 -0600 Subject: [PATCH 124/197] Reorganised indexes and fks --- schema/sdss5db/catalogdb/foreignKeys.sql | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index d8a0fe40..e28dbcf5 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -21,6 +21,7 @@ ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 -- gaiadr2_tmass_best_neighbour CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (source_id ASC); +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk @@ -28,8 +29,6 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); - ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT tmass_pts_key_fk FOREIGN KEY (tmass_pts_key) @@ -60,14 +59,14 @@ ALTER TABLE catalogdb.sdss_dr16_specobj -- gaia_unwise_agn +CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); + ALTER TABLE catalogdb.gaia_unwise_agn ADD CONSTRAINT source_id_fk FOREIGN KEY (gaia_sourceid) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); - ALTER TABLE catalogdb.gaia_unwise_agn ADD CONSTRAINT unwise_objid_fk FOREIGN KEY (unwise_objid) @@ -148,15 +147,15 @@ ALTER TABLE catalogdb.tic_v8 -- bhm_spiders_agn_superset, bhm_spiders_clusters_superset CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); +UPDATE catalogdb.bhm_spiders_agn_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; +CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); ALTER TABLE catalogdb.bhm_spiders_agn_superset ADD CONSTRAINT gaia_dr2_source_id_fk FOREIGN KEY (gaia_dr2_source_id) - REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); - ALTER TABLE catalogdb.bhm_spiders_agn_superset ADD CONSTRAINT ls_id_fk FOREIGN KEY (ls_id) @@ -164,15 +163,15 @@ ALTER TABLE catalogdb.bhm_spiders_agn_superset ON UPDATE CASCADE ON DELETE CASCADE; CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); +UPDATE catalogdb.bhm_spiders_clusters_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; ALTER TABLE catalogdb.bhm_spiders_clusters_superset ADD CONSTRAINT gaia_dr2_source_id_fk FOREIGN KEY (gaia_dr2_source_id) - REFERENCES catalogdb.gaia_dr2_source (gaia_dr2_source_id_kepler_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); - ALTER TABLE catalogdb.bhm_spiders_clusters_superset ADD CONSTRAINT ls_id_fk FOREIGN KEY (ls_id) @@ -202,4 +201,3 @@ ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT gaia_dr2_id2_fk FOREIGN KEY (gaia_dr2_id2) REFERENCES catalogdb.gaia_dr2_source (source_id) -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); From 04fa31f59938e20fca379f2ab74a2c19000f086c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 25 Mar 2020 10:27:22 -0600 Subject: [PATCH 125/197] Fix table names and use lower case --- python/sdssdb/peewee/sdss5db/catalogdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 89bec3bb..e6a20c3e 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -139,7 +139,7 @@ class SDSS_DR14_APOGEE_Star(CatalogdbModel): backref='stars') class Meta: - table_name = 'sdss_dr14_APOGEE_Star' + table_name = 'sdss_dr14_apogeestar' schema = 'catalogdb' @@ -154,7 +154,7 @@ class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel): lazy_load=False) class Meta: - table_name = 'sdss_dr14_APOGEE_Starvisit' + table_name = 'sdss_dr14_apogeestarvisit' schema = 'catalogdb' @@ -187,7 +187,7 @@ class Meta: class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel): @property - def apstar(self): + def apstars(self): """Returns the stars on `.SDSS_DR14_APOGEE_Star` with matching ``apogee_id``.""" return (SDSS_DR14_APOGEE_Star @@ -195,7 +195,7 @@ def apstar(self): .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id)) class Meta: - table_name = 'sdss_apogeeAllStarMerge_r13' + table_name = 'sdss_apogeeallstarmerge_r13' schema = 'catalogdb' From ae99d84f49a078eddb346dbcce362459900706e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 25 Mar 2020 10:28:12 -0600 Subject: [PATCH 126/197] Add ls_id as generated column --- .../catalogdb/legacy_survey/dr8/legacy_survey.sql | 3 ++- .../legacy_survey/dr8/legacy_survey_index.sql | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql index 2d80d925..01fbc9e4 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql @@ -126,5 +126,6 @@ CREATE TABLE catalogdb.legacy_survey_dr8 ( pmra_ivar REAL, pmdec REAL, pmdec_ivar REAL, - maskbits INTEGER + maskbits INTEGER, + ls_id BIGINT GENERATED ALWAYS AS (objid + brickid * 2^16 + release * 2^40) STORED ) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index a3b56db3..7ece7593 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -1,8 +1,10 @@ --- Add the lsid primary key. -ALTER TABLE catalogdb.legacy_survey_dr8 ADD COLUMN lsid BIGINT; -UPDATE catalogdb.legacy_survey_dr8 SET lsid = objid + brickid * 2^16 + release * 2^40; -ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (lsid); +CREATE UNIQUE INDEX CONCURRENTLY ls_id_pk + ON catalogdb.legacy_survey_dr8 (ls_id); + +ALTER TABLE catalogdb.legacy_survey_dr8 + ADD CONSTRAINT ls_id_pk PRIMARY KEY + USING INDEX ls_id_pk; CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; From c5ed637a23cc9e96135c33c71fa75a01741889d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 25 Mar 2020 10:29:15 -0600 Subject: [PATCH 127/197] Change how to load data to LS --- schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load index edfdf279..8291c4fd 100755 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load @@ -1,4 +1,5 @@ #!/usr/bin/env bash # encoding: utf-8 -ls ./*.fits.csv | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM '{}' WITH HEADER DELIMITER ',' NULL '\N' CSV;;\"" +ls $CATALOGDB_DIR/legacysurvey/dr8/north/sweep/*.fits.csv | xargs -I {} sh -c "echo {} && psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM {} WITH HEADER DELIMITER ',' NULL '\N' CSV;\"" +ls $CATALOGDB_DIR/legacysurvey/dr8/south/sweep/*.fits.csv | xargs -I {} sh -c "echo {} && psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM {} WITH HEADER DELIMITER ',' NULL '\N' CSV;\"" From 45a8c442cb02816af7df86a91a4a98666965c973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 25 Mar 2020 10:29:36 -0600 Subject: [PATCH 128/197] Create PK for SkyMapper in a non-blocking way --- schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql | 4 ++-- .../catalogdb/skymapper/dr1.1/skymapper_indexes.sql | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql index a9bd806c..2dfe1d2a 100644 --- a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql @@ -9,8 +9,8 @@ At Utah: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/skymapp */ -CREATE TABLE skymapper_dr1_1 ( - object_id BIGINT PRIMARY KEY, +CREATE TABLE catalogdb.skymapper_dr1_1 ( + object_id BIGINT, raj2000 DOUBLE PRECISION, dej2000 DOUBLE PRECISION, e_raj2000 INTEGER, diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql index eed4f41e..cb9cddca 100644 --- a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql @@ -1,4 +1,14 @@ +-- Doing it this way prevent blocks: https://bit.ly/2UDbpbz + +CREATE UNIQUE INDEX CONCURRENTLY object_id_pk + ON catalogdb.skymapper_dr1_1 (object_id); -- takes a long time, but doesn’t block queries + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT object_id_pk PRIMARY KEY + USING INDEX object_id_pk; -- blocks queries, but only very briefly + + CREATE INDEX ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, decj2000)); CLUSTER skymapper_dr1_1_q3c_ang2ipix_idx ON catalogdb.skymapper_dr1_1; ANALYZE catalogdb.skymapper_dr1_1; From 4e6dc694480e87a942a4106199fd08f0bccfc90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 26 Mar 2020 13:41:54 -0600 Subject: [PATCH 129/197] Rename SDSS QSO to sdss_dr14_qso --- python/sdssdb/peewee/sdss5db/catalogdb.py | 12 ++++++++++-- .../catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql | 6 ------ .../dr14/sdss_dr14_qso.sql} | 2 +- .../catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql | 10 ++++++++++ 4 files changed, 21 insertions(+), 9 deletions(-) delete mode 100644 schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql rename schema/sdss5db/catalogdb/{dr14q_v4_4/dr14q_v4_4.sql => sdss_qso/dr14/sdss_dr14_qso.sql} (97%) create mode 100644 schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index e6a20c3e..2261322a 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -259,10 +259,18 @@ class Meta: schema = 'catalogdb' -class DR14_Q_V4_4(CatalogdbModel): +class SDSS_DR14_QSO(CatalogdbModel): + + @property + def specobj(self): + """Returns the matching record in `.SDSS_DR16_SpecObj`.""" + + return SDSS_DR16_SpecObj.get(SDSS_DR16_SpecObj.plate == self.plate, + SDSS_DR16_SpecObj.mjd == self.mjd, + SDSS_DR16_SpecObj.fiberid == self.fiber) class Meta: - table_name = 'dr14q_v4_4' + table_name = 'sdss_dr14_qso' schema = 'catalogdb' diff --git a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql b/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql deleted file mode 100644 index 111a26d1..00000000 --- a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4_index.sql +++ /dev/null @@ -1,6 +0,0 @@ - -ALTER TABLE catalogdb.dr14q_v4_4 ADD PRIMARY KEY (pk); - -CREATE INDEX ON catalogdb.dr14q_v4_4 (q3c_ang2ipix(ra, dec)); -CLUSTER dr14q_v4_4_q3c_ang2ipix_idx ON catalogdb.dr14q_v4_4; -ANALYZE catalogdb.dr14q_v4_4; diff --git a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql similarity index 97% rename from schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql rename to schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql index 81bc23f9..6261dd92 100644 --- a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS "catalogdb"."dr14q_v4_4" ( +CREATE UNLOGGED TABLE IF NOT EXISTS "catalogdb"."sdss_dr14_qso" ( "sdss_name" VARCHAR(18), "ra" DOUBLE PRECISION, "dec" DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql new file mode 100644 index 00000000..4ac3e9ae --- /dev/null +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql @@ -0,0 +1,10 @@ + +ALTER TABLE catalogdb.sdss_dr14_qso ADD PRIMARY KEY (pk); + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (plate); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (mjd); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); + +CREATE INDEX ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_qso_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_qso; +ANALYZE catalogdb.sdss_dr14_qso; From 7cae877edd28bc6528708bcb329b955aa8c19f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 26 Mar 2020 15:50:15 -0600 Subject: [PATCH 130/197] Fix bug in redefinition of fields when subclassing a model --- python/sdssdb/peewee/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 96c306cc..224096c6 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -112,11 +112,6 @@ def reflect(self): if not hasattr(self._meta, 'use_reflection') or not self._meta.use_reflection: return - # First clear all previously reflected fields and indexes. - for field in self._meta.fields: - if hasattr(field, 'reflected') and field.reflected: - self._meta.remove_field(field) - for index in self._meta.indexes: if hasattr(index, 'reflected') and index.reflected: self._meta.indexes.remove(index) @@ -138,20 +133,25 @@ def reflect(self): for field_name, field in ReflectedModel._meta.fields.items(): if field_name in self._meta.fields: - continue + meta_field = self._meta.fields[field_name] + if not hasattr(meta_field, 'reflected') or not meta_field.reflected: + continue if (isinstance(field, peewee.ForeignKeyField) and hasattr(self._meta, 'reflection_options') and self._meta.reflection_options.get('skip_foreign_keys', False)): continue - field.reflected = True + if field_name == 'specobjid': + print(field, field.primary_key) if field.primary_key: self._meta.set_primary_key(field_name, field) else: self._meta.add_field(field_name, field) + self._meta.fields[field_name].reflected = True + for index in ReflectedModel._meta.indexes: # TODO: this probably can never work. Find a better way of determining @@ -159,6 +159,7 @@ def reflect(self): if index in self._meta.indexes: continue + index.reflected = True self._meta.indexes.append(index) From 76723f5a2515ab7ada45221afd96475ef1a90c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 27 Mar 2020 17:49:03 -0600 Subject: [PATCH 131/197] Add PS1_g18 --- python/sdssdb/peewee/__init__.py | 4 -- python/sdssdb/peewee/sdss5db/catalogdb.py | 59 +++++-------------- python/sdssdb/utils/ingest.py | 4 +- schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql | 23 ++++++++ .../catalogdb/PS1/g18/PS1_g18_index.sql | 14 +++++ .../sdss5db/catalogdb/PS1/g18/PS1_g18_load.py | 43 ++++++++++++++ 6 files changed, 96 insertions(+), 51 deletions(-) create mode 100644 schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql create mode 100644 schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql create mode 100755 schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 224096c6..ddb54cf8 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -142,9 +142,6 @@ def reflect(self): self._meta.reflection_options.get('skip_foreign_keys', False)): continue - if field_name == 'specobjid': - print(field, field.primary_key) - if field.primary_key: self._meta.set_primary_key(field_name, field) else: @@ -159,7 +156,6 @@ def reflect(self): if index in self._meta.indexes: continue - index.reflected = True self._meta.indexes.append(index) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 2261322a..f9fc8640 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -20,6 +20,7 @@ class CatalogdbModel(BaseModel): class Meta: database = database + schema = 'catalogdb' use_reflection = True reflection_options = {'skip_foreign_keys': True} primary_key = False @@ -36,7 +37,6 @@ class AllWise(CatalogdbModel): class Meta: table_name = 'allwise' - schema = 'catalogdb' class TwoMassPSC(CatalogdbModel): @@ -46,7 +46,6 @@ class TwoMassPSC(CatalogdbModel): class Meta: table_name = 'twomass_psc' - schema = 'catalogdb' class Gaia_DR2(CatalogdbModel): @@ -59,47 +58,42 @@ class Gaia_DR2(CatalogdbModel): class Meta: table_name = 'gaia_dr2_source' - schema = 'catalogdb' class Gaia_DR2_Clean(CatalogdbModel): - source_id = ForeignKeyField(Gaia_DR2, - field='source_id', - backref='gaia_clean', - lazy_load=True) + source = ForeignKeyField(Gaia_DR2, + field='source_id', + column_name='source_id', + object_id_name='source_id', + backref='+') class Meta: table_name = 'gaia_dr2_clean' - schema = 'catalogdb' class Gaia_DR2_SDSS_DR9_Best_Neighbour(CatalogdbModel): class Meta: table_name = 'gaiadr2_sdssdr9_best_neighbour' - schema = 'catalogdb' class GalacticGenesis(CatalogdbModel): class Meta: table_name = 'galactic_genesis' - schema = 'catalogdb' class GalacticGenesisBig(CatalogdbModel): class Meta: table_name = 'galactic_genesis_big' - schema = 'catalogdb' class GUVCat(CatalogdbModel): class Meta: table_name = 'guvcat' - schema = 'catalogdb' class KeplerInput_DR10(CatalogdbModel): @@ -108,7 +102,6 @@ class KeplerInput_DR10(CatalogdbModel): class Meta: table_name = 'kepler_input_10' - schema = 'catalogdb' class SDSS_DR13_PhotoObj(CatalogdbModel): @@ -117,7 +110,6 @@ class SDSS_DR13_PhotoObj(CatalogdbModel): class Meta: table_name = 'sdss_dr13_photoobj' - schema = 'catalogdb' class SDSS_DR14_APOGEE_Visit(CatalogdbModel): @@ -126,7 +118,6 @@ class SDSS_DR14_APOGEE_Visit(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeevisit' - schema = 'catalogdb' class SDSS_DR14_APOGEE_Star(CatalogdbModel): @@ -140,7 +131,6 @@ class SDSS_DR14_APOGEE_Star(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeestar' - schema = 'catalogdb' class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel): @@ -155,7 +145,6 @@ class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel): class Meta: table_name = 'sdss_dr14_apogeestarvisit' - schema = 'catalogdb' class SDSS_DR14_ASCAP_Star(CatalogdbModel): @@ -166,7 +155,6 @@ class SDSS_DR14_ASCAP_Star(CatalogdbModel): class Meta: table_name = 'sdss_dr14_ascapstar' - schema = 'catalogdb' class SDSS_DR14_Cannon_Star(CatalogdbModel): @@ -181,7 +169,6 @@ def apstars(self): class Meta: table_name = 'sdss_dr14_cannonstar' - schema = 'catalogdb' class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel): @@ -196,7 +183,6 @@ def apstars(self): class Meta: table_name = 'sdss_apogeeallstarmerge_r13' - schema = 'catalogdb' class SDSS_DR14_SpecObj(CatalogdbModel): @@ -208,7 +194,6 @@ class SDSS_DR14_SpecObj(CatalogdbModel): class Meta: table_name = 'sdss_dr14_specobj' - schema = 'catalogdb' class SDSS_DR16_SpecObj(SDSS_DR14_SpecObj): @@ -230,14 +215,12 @@ class TwoMass_Clean(CatalogdbModel): class Meta: table_name = 'twomass_clean' - schema = 'catalogdb' class TwoMass_Clean_No_Neighbor(CatalogdbModel): class Meta: table_name = 'twomass_clean_noneighbor' - schema = 'catalogdb' class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): @@ -256,7 +239,6 @@ class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): class Meta: table_name = 'gaiadr2_tmass_best_neighbour' - schema = 'catalogdb' class SDSS_DR14_QSO(CatalogdbModel): @@ -271,14 +253,14 @@ def specobj(self): class Meta: table_name = 'sdss_dr14_qso' - schema = 'catalogdb' class unWISE(CatalogdbModel): + unwise_objid = TextField(primary_key=True) + class Meta: table_name = 'unwise' - schema = 'catalogdb' class Legacy_Survey_DR8(CatalogdbModel): @@ -297,7 +279,6 @@ def gaia(self): class Meta: table_name = 'legacy_survey_dr8' - schema = 'catalogdb' class Gaia_unWISE_AGN(CatalogdbModel): @@ -316,14 +297,12 @@ class Gaia_unWISE_AGN(CatalogdbModel): class Meta: table_name = 'gaia_unwise_agn' - schema = 'catalogdb' class eBOSS_Taarget_v5(CatalogdbModel): class Meta: table_name = 'ebosstarget_v5' - schema = 'catalogdb' # The following model (BhmSpidersGenericSuperset) does not need to be represented @@ -441,7 +420,6 @@ class BHM_Spiders_Generic_Superset(CatalogdbModel): class Meta: table_name = 'bhm_spiders_generic_superset' - schema = 'catalogdb' use_reflection = False @@ -463,7 +441,6 @@ class BHM_CSC(CatalogdbModel): class Meta: table_name = 'bhm_csc' - schema = 'catalogdb' class Gaia_DR2_WD_SDSS(CatalogdbModel): @@ -478,7 +455,6 @@ def specobj(self): class Meta: table_name = 'gaia_dr2_wd_sdss' - schema = 'catalogdb' class Gaia_DR2_WD(CatalogdbModel): @@ -491,7 +467,6 @@ def sdss(self): class Meta: table_name = 'gaia_dr2_wd' - schema = 'catalogdb' class Tycho2(CatalogdbModel): @@ -500,14 +475,12 @@ class Tycho2(CatalogdbModel): class Meta: table_name = 'tycho2' - schema = 'catalogdb' class GaiaQSO(CatalogdbModel): class Meta: table_name = 'gaia_qso' - schema = 'catalogdb' class TIC_v8(CatalogdbModel): @@ -538,21 +511,18 @@ class TIC_v8(CatalogdbModel): class Meta: table_name = 'tic_v8' - schema = 'catalogdb' class CatWISE(CatalogdbModel): class Meta: table_name = 'catwise' - schema = 'catalogdb' class CatWISEReject(CatalogdbModel): class Meta: table_name = 'catwise_reject' - schema = 'catalogdb' class Watchdog(CatalogdbModel): @@ -566,7 +536,6 @@ class Watchdog(CatalogdbModel): class Meta: table_name = 'watchdog' - schema = 'catalogdb' class BlackCAT(CatalogdbModel): @@ -580,7 +549,6 @@ class BlackCAT(CatalogdbModel): class Meta: table_name = 'blackcat' - schema = 'catalogdb' class XRay_Pulsars(CatalogdbModel): @@ -594,7 +562,6 @@ class XRay_Pulsars(CatalogdbModel): class Meta: table_name = 'xray_pulsars' - schema = 'catalogdb' class LowMassXRayBinaries(CatalogdbModel): @@ -608,7 +575,6 @@ class LowMassXRayBinaries(CatalogdbModel): class Meta: table_name = 'lmxb' - schema = 'catalogdb' class GalacticMillisecondPulsars(CatalogdbModel): @@ -622,7 +588,6 @@ class GalacticMillisecondPulsars(CatalogdbModel): class Meta: table_name = 'galactic_millisecond_pulsars' - schema = 'catalogdb' class ATNF(CatalogdbModel): @@ -636,7 +601,6 @@ class ATNF(CatalogdbModel): class Meta: table_name = 'atnf' - schema = 'catalogdb' class SkyMapper_DR1_1(CatalogdbModel): @@ -676,7 +640,12 @@ def twomass2(self): class Meta: table_name = 'skymapper_dr1_1' - schema = 'catalogdb' + + +class PS1_g18(CatalogdbModel): + + class Meta: + table_name = 'ps1_g18' _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index 8006ce65..d88064ed 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -205,7 +205,7 @@ class Meta: return type(str(table_name), (BaseModel,), attrs) -def convert_row_to_psql(row, delimiter='\t'): +def convert_row_to_psql(row, delimiter='\t', null='\\N'): """Concerts an astropy table row to a Postgresql-valid CSV string.""" row_data = [] @@ -214,7 +214,7 @@ def convert_row_to_psql(row, delimiter='\t'): if numpy.isscalar(col_value): row_data.append(str(col_value)) elif numpy.ma.is_masked(col_value): - row_data.append('') + row_data.append(null) else: if col_value.dtype.base.str[1] == 'S': col_value = col_value.astype('U') diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql new file mode 100644 index 00000000..b4a09578 --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql @@ -0,0 +1,23 @@ +/* + +PS1 gMeanPSFMag < 18. + +*/ + + +CREATE UNLOGGED TABLE catalogdb.ps1_g18 ( + objid BIGINT, + ndetections INTEGER, + ramean DOUBLE PRECISION, + decmean DOUBLE PRECISION, + qualityflag INTEGER, + gmeanpsfmag DOUBLE PRECISION, + gmeanpsfmagerr DOUBLE PRECISION, + gflags BIGINT, + rmeanpsfmag DOUBLE PRECISION, + rmeanpsfmagerr DOUBLE PRECISION, + rflags BIGINT, + imeanpsfmag DOUBLE PRECISION, + imeanpsfmagerr DOUBLE PRECISION, + iflags BIGINT +); diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql new file mode 100644 index 00000000..3750bb7e --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql @@ -0,0 +1,14 @@ + +-- PK + +ALTER TABLE catalogdb.ps1_g18 ADD PRIMARY KEY (objid); + +-- Indexes + +CREATE INDEX ON catalogdb.ps1_g18 (q3c_ang2ipix(raMean, decMean)); +CLUSTER ps1_g18_q3c_ang2ipix_idx ON catalogdb.ps1_g18; +ANALYZE catalogdb.ps1_g18; + +CREATE INDEX ON catalogdb.ps1_g18 (gmeanpsfmag); +CREATE INDEX ON catalogdb.ps1_g18 (rmeanpsfmag); +CREATE INDEX ON catalogdb.ps1_g18 (imeanpsfmag); diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py new file mode 100755 index 00000000..3508c0c5 --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-27 +# @Filename: PS1_g18_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import io +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database + + +def main(): + + assert database.connected + + files = glob.glob(os.environ['CATALOGDB_DIR'] + '/PS1/g18/*.fit') + + for file_ in files: + + print(file_) + + data = astropy.table.Table.read(file_) + data.meta = {} + + stream = io.StringIO() + data.write(stream, format='csv', fast_writer=True) + + cursor = database.cursor() + stream.seek(0) + stream.readline() # Read header + cursor.copy_from(stream, 'catalogdb.ps1_g18', sep=',') + database.commit() + + +if __name__ == '__main__': + + main() From ccff9ff9bcb4f35f146a971ea295b75b1eb6db0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 10:04:24 -0600 Subject: [PATCH 132/197] Add GLIMPSE --- python/sdssdb/peewee/sdss5db/catalogdb.py | 11 ++ schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql | 183 ++++++++++++++++++ .../catalogdb/GLIMPSE/GLIMPSE_index.sql | 11 ++ .../sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py | 61 ++++++ schema/sdss5db/catalogdb/foreignKeys.sql | 11 ++ 5 files changed, 277 insertions(+) create mode 100644 schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql create mode 100644 schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql create mode 100755 schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index f9fc8640..1de52794 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -648,5 +648,16 @@ class Meta: table_name = 'ps1_g18' +class GLIMPSE(CatalogdbModel): + + twomass = ForeignKeyField(TwoMassPSC, field='pts_key', + column_name='tmass_cntr', + backref='glimpse_targets') + + class Meta: + table_name = 'glimpse' + print_fields = ['designation'] + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql new file mode 100644 index 00000000..98206866 --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql @@ -0,0 +1,183 @@ +/* + +GLIMPSE I+II+3D (Archive) + +https://irsa.ipac.caltech.edu/data/SPITZER/GLIMPSE/overview.html + +\fixlen = T +\ Default Data Dictionary +\ +\ +\alias = none +\longitude = ra +\latitude = dec +\primary = cntr +\spt_ind = spt_ind +\x = x +\y = y +\z = z +\ +\ +| cntr| name| original_name| description| units| intype| format| dbtype|nulls|tableflg| groupid| irsadef| sel|indx| notes| +| integer| char| char| char| char| char| char| char| char| integer| integer| char |char|char| char| + 1 designation designation GLIMPSEI source designation name char character(26) n 2 1 y y n + 2 tmass_designation tmass_designation Source name (designation) from 2MASS All-Sky point source catalog (PSC) char character(16) y 2 2 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#designation + 3 tmass_cntr tmass_cntr Unique id number (counter) for the 2MASS source from the 2MASS All-Sky PSC int integer y 2 3 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#pts_key + 4 l l Galactic longitude deg double float n 2 4 y y n + 5 b b Galactic latitude deg double float n 2 5 y y n + 6 dl dl error in Galactic longitude arcsec double float n 2 6 y y n + 7 db db error in Galactic latitude arcsec double float n 2 7 y y n + 8 ra ra Right ascension J2000 deg double float n 2 8 y y n + 9 dec dec Declination J2000 deg double float n 2 9 y y n + 10 dra dra error in Right ascension arcsec double float n 2 10 y y n + 11 ddec ddec error in Declination arcsec double float n 2 11 y y n + 12 csf csf close source flag int integer n 2 12 y y n + 13 mag_J mag_J 2MASS All-Sky PSC J Band magnitude mag real smallfloat y 2 13 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_m + 14 dJ_m dJ_m 2MASS All-Sky PSC J Band 1 sigma error mag real smallfloat y 2 13 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_msigcom + 15 mag_H mag_H 2MASS All-Sky PSC H Band magnitude mag real smallfloat y 2 15 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_m + 16 dH_m dH_m 2MASS All-Sky PSC H Band 1 sigma error mag real smallfloat y 2 15 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_msigcom + 17 mag_Ks mag_Ks 2MASS All-Sky PSC Ks Band magnitude mag real smallfloat y 2 17 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_m + 18 dKs_m dKs_m 2MASS All-Sky PSC Ks Band 1 sigma error mag real smallfloat y 2 17 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_msigcom + 19 mag3_6 mag3_6 3.6um IRAC (Band 1) magnitude mag real smallfloat y 2 19 y y n + 20 d3_6m d3_6m 3.6um IRAC (Band 1) 1 sigma error mag real smallfloat y 2 19 y y n + 21 mag4_5 mag4_5 4.5um IRAC (Band 2) magnitude mag real smallfloat y 2 21 y y n + 22 d4_5m d4_5m 4.5um IRAC (Band 2) 1 sigma error mag real smallfloat y 2 21 y y n + 23 mag5_8 mag5_8 5.8um IRAC (Band 3) magnitude mag real smallfloat y 2 23 y y n + 24 d5_8m d5_8m 5.8um IRAC (Band 3) 1 sigma error mag real smallfloat y 2 23 y y n + 25 mag8_0 mag8_0 8.0um IRAC (Band 4) magnitude mag real smallfloat y 2 25 y y n + 26 d8_0m d8_0m 8.0um IRAC (Band 4) 1 sigma error mag real smallfloat y 2 25 y y n + 27 f_J f_J 2MASS All-Sky PSC J Band flux mJy real smallfloat y 2 27 y y n + 28 df_J df_J 2MASS All-Sky PSC J Band 1 sigma error mJy real smallfloat y 2 27 y y n + 29 f_H f_H 2MASS All-Sky PSC H Band flux mJy real smallfloat y 2 29 y y n + 30 df_H df_H 2MASS All-Sky PSC H Band 1 sigma error mJy real smallfloat y 2 29 y y n + 31 f_Ks f_Ks 2MASS All-Sky PSC Ks Band flux mJy real smallfloat y 2 31 y y n + 32 df_Ks df_Ks 2MASS All-Sky PSC Ks Band 1 sigma error mJy real smallfloat y 2 31 y y n + 33 f3_6 f3_6 3.6um IRAC (Band 1) flux mJy real smallfloat y 2 33 y y n + 34 df3_6 df3_6 3.6um IRAC (Band 1) 1 sigma error mJy real smallfloat y 2 33 y y n + 35 f4_5 f4_5 4.5um IRAC (Band 2) flux mJy real smallfloat y 2 35 y y n + 36 df4_5 df4_5 4.5um IRAC (Band 2) 1 sigma error mJy real smallfloat y 2 35 y y n + 37 f5_8 f5_8 5.8um IRAC (Band 3) flux mJy real smallfloat y 2 37 y y n + 38 df5_8 df5_8 5.8um IRAC (Band 3) 1 sigma error mJy real smallfloat y 2 37 y y n + 39 f8_0 f8_0 8.0um IRAC (Band 4) flux mJy real smallfloat y 2 39 y y n + 40 df8_0 df8_0 8.0um IRAC (Band 4) 1 sigma error mJy real smallfloat y 2 39 y y n + 41 rms_f3_6 rms_f3_6 rms deviation of the individual detections from the final flux for 3.6um IRAC mJy real smallfloat y 2 41 y y n + 42 rms_f4_5 rms_f4_5 rms deviation of the individual detections from the final flux for 4.5um IRAC mJy real smallfloat y 2 42 y y n + 43 rms_f5_8 rms_f5_8 rms deviation of the individual detections from the final flux for 5.8um IRAC mJy real smallfloat y 2 43 y y n + 44 rms_f8_0 rms_f8_0 rms deviation of the individual detections from the final flux for 8.0um IRAC mJy real smallfloat y 2 44 y y n + 45 sky_3_6 sky_3_6 Local sky background for 3.6um IRAC (Band 1) MJy/sr real smallfloat y 2 45 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 46 sky_4_5 sky_4_5 Local sky background for 4.5um IRAC (Band 2) MJy/sr real smallfloat y 2 46 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 47 sky_5_8 sky_5_8 Local sky background for 5.8um IRAC (Band 3) MJy/sr real smallfloat y 2 47 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 48 sky_8_0 sky_8_0 Local sky background for 8.0um IRAC (Band 4) MJy/sr real smallfloat y 2 48 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 49 sn_J sn_J 2MASS All-Sky PSC J Band Signal/Noise real smallfloat y 2 49 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_snr + 50 sn_H sn_H 2MASS All-Sky PSC H Band Signal/Noise real smallfloat y 2 50 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_snr + 51 sn_Ks sn_Ks 2MASS All-Sky PSC Ks Band Signal/Noise real smallfloat y 2 51 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_snr + 52 sn_3_6 sn_3_6 3.6um IRAC (Band 1) Signal/Noise real smallfloat y 2 52 y y n + 53 sn_4_5 sn_4_5 4.5um IRAC (Band 2) Signal/Noise real smallfloat y 2 53 y y n + 54 sn_5_8 sn_5_8 5.8um IRAC (Band 3) Signal/Noise real smallfloat y 2 54 y y n + 55 sn_8_0 sn_8_0 8.0um IRAC (Band 4) Signal/Noise real smallfloat y 2 55 y y n + 56 dens_3_6 dens_3_6 Local source density for 3.6um IRAC (Band 1) #/sqamin real smallfloat y 2 56 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 57 dens_4_5 dens_4_5 Local source density for 4.5um IRAC (Band 2) #/sqamin real smallfloat y 2 57 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 58 dens_5_8 dens_5_8 Local source density for 5.8um IRAC (Band 3) #/sqamin real smallfloat y 2 58 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 59 dens_8_0 dens_8_0 Local source density for 8.0um IRAC (Band 4) #/sqamin real smallfloat y 2 59 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 60 m3_6 m3_6 Number of detections for 3.6um IRAC (Band 1) int integer n 2 60 y y n + 61 m4_5 m4_5 Number of detections for 4.5um IRAC (Band 2) int integer n 2 61 y y n + 62 m5_8 m5_8 Number of detections for 5.8um IRAC (Band 3) int integer n 2 62 y y n + 63 m8_0 m8_0 Number of detections for 8.0um IRAC (Band 4) int integer n 2 63 y y n + 64 n3_6 n3_6 Number of possible detections for 3.6um IRAC (Band 1) int integer n 2 64 y y n + 65 n4_5 n4_5 Number of possible detections for 4.5um IRAC (Band 2) int integer n 2 65 y y n + 66 n5_8 n5_8 Number of possible detections for 5.8um IRAC (Band 3) int integer n 2 66 y y n + 67 n8_0 n8_0 Number of possible detections for 8.0um IRAC (Band 4) int integer n 2 67 y y n + 68 sqf_J sqf_J 2MASS All-Sky PSC J Band Source Quality Flag int integer y 2 68 y y n + 69 sqf_H sqf_H 2MASS All-Sky PSC H Band Source Quality Flag int integer y 2 69 y y n + 70 sqf_Ks sqf_Ks 2MASS All-Sky PSC Ks Band Source Quality Flag int integer y 2 70 y y n + 71 sqf_3_6 sqf_3_6 Source Quality Flag for 3.6um IRAC (Band 1) int integer y 2 71 y y n + 72 sqf_4_5 sqf_4_5 Source Quality Flag for 4.5um IRAC (Band 2) int integer y 2 72 y y n + 73 sqf_5_8 sqf_5_8 Source Quality Flag for 5.8um IRAC (Band 3) int integer y 2 73 y y n + 74 sqf_8_0 sqf_8_0 Source Quality Flag for 8.0um IRAC (Band 4) int integer y 2 74 y y n + 75 mf3_6 mf3_6 Flux calculation method flag 3.6um IRAC (Band 1) int integer y 2 75 y y n + 76 mf4_5 mf4_5 Flux calculation method flag 4.5um IRAC (Band 2) int integer y 2 76 y y n + 77 mf5_8 mf5_8 Flux calculation method flag 5.8um IRAC (Band 3) int integer y 2 77 y y n + 78 mf8_0 mf8_0 Flux calculation method flag 8.0um IRAC (Band 4) int integer y 2 78 y y n + +*/ + +CREATE UNLOGGED TABLE catalogdb.glimpse ( + designation TEXT, + tmass_designation VARCHAR(18), + tmass_cntr INTEGER, + l DOUBLE PRECISION, + b DOUBLE PRECISION, + dl DOUBLE PRECISION, + db DOUBLE PRECISION, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + dra DOUBLE PRECISION, + ddec DOUBLE PRECISION, + csf INTEGER, + mag_J REAL, + dJ_m REAL, + mag_H REAL, + dH_m REAL, + mag_Ks REAL, + dKs_m REAL, + mag3_6 REAL, + d3_6m REAL, + mag4_5 REAL, + d4_5m REAL, + mag5_8 REAL, + d5_8m REAL, + mag8_0 REAL, + d8_0m REAL, + f_J REAL, + df_J REAL, + f_H REAL, + df_H REAL, + f_Ks REAL, + df_Ks REAL, + f3_6 REAL, + df3_6 REAL, + f4_5 REAL, + df4_5 REAL, + f5_8 REAL, + df5_8 REAL, + f8_0 REAL, + df8_0 REAL, + rms_f3_6 REAL, + rms_f4_5 REAL, + rms_f5_8 REAL, + rms_f8_0 REAL, + sky_3_6 REAL, + sky_4_5 REAL, + sky_5_8 REAL, + sky_8_0 REAL, + sn_J REAL, + sn_H REAL, + sn_Ks REAL, + sn_3_6 REAL, + sn_4_5 REAL, + sn_5_8 REAL, + sn_8_0 REAL, + dens_3_6 REAL, + dens_4_5 REAL, + dens_5_8 REAL, + dens_8_0 REAL, + m3_6 INTEGER, + m4_5 INTEGER, + m5_8 INTEGER, + m8_0 INTEGER, + n3_6 INTEGER, + n4_5 INTEGER, + n5_8 INTEGER, + n8_0 INTEGER, + sqf_J INTEGER, + sqf_H INTEGER, + sqf_Ks INTEGER, + sqf_3_6 INTEGER, + sqf_4_5 INTEGER, + sqf_5_8 INTEGER, + sqf_8_0 INTEGER, + mf3_6 INTEGER, + mf4_5 INTEGER, + mf5_8 INTEGER, + mf8_0 INTEGER +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql new file mode 100644 index 00000000..2decd9ed --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql @@ -0,0 +1,11 @@ + +-- PK + +ALTER TABLE catalogdb.glimpse ADD COLUMN pk BIGSERIAL PRIMARY KEY; + + +-- Indexes + +CREATE INDEX ON catalogdb.glimpse (q3c_ang2ipix(ra, dec)); +CLUSTER glimpse_q3c_ang2ipix_idx ON catalogdb.glimpse; +ANALYZE catalogdb.glimpse; diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py new file mode 100755 index 00000000..6c6c1552 --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-27 +# @Filename: GLIMPSE_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import io +import os + +import pandas + +from sdssdb.peewee.sdss5db import database + + +def main(): + + assert database.connected + + skiprows = {'GLMIA.tbl.gz': 9, + 'GLMIIA.tbl.gz': 13, + 'GLM3D_jan2009_Archive.tbl.gz': 13, + 'GLM3DA_l330+02.tbl.gz': 14, + 'GLM3DA_l330-02.tbl.gz': 14, + 'GLM3DA_l335-02.tbl.gz': 14} + + files = glob.glob(os.environ['CATALOGDB_DIR'] + '/GLIMPSE/*.tbl.gz') + + for file_ in files: + + print(file_) + + basename = os.path.basename(file_) + nrows = skiprows[basename] + + tables = pandas.read_csv(file_, skipinitialspace=True, delimiter=' ', + chunksize=1000000, header=None, skiprows=nrows) + + cursor = database.cursor() + + for ii, table in enumerate(tables): + + print(f'Chunk {ii+1} ...') + + table[0] = table[0] + table[1] + table = table.drop(columns=1) + table[3] = table[3].replace({0: pandas.NA}) + + stream = io.StringIO() + table.to_csv(stream, header=False, index=False, na_rep='\\0') + + stream.seek(0) + cursor.copy_from(stream, 'catalogdb.glimpse', sep=',', null='\\0') + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index e28dbcf5..48dd3926 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -201,3 +201,14 @@ ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT gaia_dr2_id2_fk FOREIGN KEY (gaia_dr2_id2) REFERENCES catalogdb.gaia_dr2_source (source_id) + + +-- glimpse + +CREATE INDEX ON catalogdb.glimpse USING BTREE (tmass_cntr); + +ALTER TABLE catalogdb.glimpse + ADD CONSTRAINT tmass_cntr_fk + FOREIGN KEY (tmass_cntr) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; From 5359a12a9a1322714ae8e0e27ed2539f482cc10e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 10:22:01 -0600 Subject: [PATCH 133/197] Move print_fields to Meta --- python/sdssdb/peewee/__init__.py | 14 ++++++-------- python/sdssdb/peewee/operationsdb/platedb.py | 3 +-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index ddb54cf8..9d0f6e12 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -163,17 +163,15 @@ class BaseModel(Model, metaclass=ReflectMeta): """A custom peewee `.Model` with enhanced representation and methods. By default it always prints ``pk``, ``name``, and ``label``, if found. - Models can define they own `.print_fields` as a list of field to be output - in the representation. + Models can define they own ``print_fields`` in ``Meta`` as a list of field + names to be output in the representation. """ - #: A list of fields (as strings) to be included in the ``__repr__`` - print_fields = [] - class Meta: primary_key = False use_reflection = False + print_fields = [] def __str__(self): """A custom str for the model repr.""" @@ -186,10 +184,10 @@ def __str__(self): fields = [] for extra_field in ['label', 'name']: - if extra_field not in self.print_fields: - self.print_fields.append(extra_field) + if extra_field not in self._meta.print_fields: + self._meta.print_fields.append(extra_field) - for ff in self.print_fields: + for ff in self._meta.print_fields: if ff == pk_field: continue if hasattr(self, ff): diff --git a/python/sdssdb/peewee/operationsdb/platedb.py b/python/sdssdb/peewee/operationsdb/platedb.py index 24f65dc5..be47cbc9 100644 --- a/python/sdssdb/peewee/operationsdb/platedb.py +++ b/python/sdssdb/peewee/operationsdb/platedb.py @@ -141,8 +141,6 @@ class Meta: class Plate(OperationsDBModel): - print_fields = ['plate_id'] - chunk = TextField(null=True) comment = TextField(null=True) current_survey_mode = ForeignKeyField(column_name='current_survey_mode_pk', @@ -198,6 +196,7 @@ def mangadb_plate(self): class Meta: db_table = 'plate' schema = 'platedb' + print_fields = ['plate_id'] class PluggingStatus(OperationsDBModel): From 90fbd4c851ce04c8fe4d1985140c952eeacd9835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 10:48:31 -0600 Subject: [PATCH 134/197] Add indexes for (plate, mjd, fiberid) --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql | 2 ++ schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql | 1 + schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 1de52794..cc3db786 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -249,7 +249,7 @@ def specobj(self): return SDSS_DR16_SpecObj.get(SDSS_DR16_SpecObj.plate == self.plate, SDSS_DR16_SpecObj.mjd == self.mjd, - SDSS_DR16_SpecObj.fiberid == self.fiber) + SDSS_DR16_SpecObj.fiberid == self.fiberid) class Meta: table_name = 'sdss_dr14_qso' diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index 576ad0ed..5be41e3d 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -414,3 +414,5 @@ CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (mjd); CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (plate); CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fiberid); + +CREATE INDEX ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid); diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql index 4ac3e9ae..fb586870 100644 --- a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql @@ -4,6 +4,7 @@ ALTER TABLE catalogdb.sdss_dr14_qso ADD PRIMARY KEY (pk); CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (plate); CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (mjd); CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); CREATE INDEX ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_qso_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_qso; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql index e8ca190c..42a4e865 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -12,3 +12,4 @@ ANALYZE catalogdb.gaia_dr2_wd_sdss; CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (plate); CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (mjd); CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (fiber); +CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss (mjd, plate, fiber); From 3ff6ea0426dcd3026c1cc2e06d267f2a986ef125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 12:14:14 -0600 Subject: [PATCH 135/197] Add indexes on mjd, plate, fiberid --- .../sdssSpecObj/dr14/sdss_dr14_specObj.sql | 4 +--- .../sdss_qso/dr14/sdss_dr14_qso_index.sql | 15 +++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql index cb36c189..e4ba64ee 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql @@ -411,6 +411,4 @@ ANALYZE catalogdb.sdss_dr14_specobj; CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (bestObjID); CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (mjd); -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (plate); -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fiberid); +CREATE INDEX ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid); diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql index fb586870..e83fe6d2 100644 --- a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql @@ -1,10 +1,17 @@ ALTER TABLE catalogdb.sdss_dr14_qso ADD PRIMARY KEY (pk); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (plate); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (mjd); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); +-- ALTER TABLE catalogdb.sdss_dr14_qso ADD COLUMN specobjid BIGINT; +-- UPDATE catalogdb.sdss_dr14_qso +-- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + +-- ((mjd-50000)::bigint<<24) + (26::bigint<<10); + +CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (plate); +CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (mjd); +CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); +CREATE INDEX ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); + +-- CREATE INDEX ON catalogdb.sdss_dr14_qso (specobjid); CREATE INDEX ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_qso_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_qso; From f571ef0c51439a662698e64d174ecb81f986a38a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 14:17:56 -0600 Subject: [PATCH 136/197] Support for composite keys in ReflectMeta --- python/sdssdb/peewee/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 9d0f6e12..44fd1028 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -149,6 +149,12 @@ def reflect(self): self._meta.fields[field_name].reflected = True + # Composite keys are not a normal column so if the pk has not been + # set already, check if it exists in the reflected model. + if not self._meta.primary_key and ReflectedModel._meta.primary_key: + pk = ReflectedModel._meta.primary_key + self._meta.set_primary_key(pk.name, pk) + for index in ReflectedModel._meta.indexes: # TODO: this probably can never work. Find a better way of determining @@ -177,7 +183,10 @@ def __str__(self): """A custom str for the model repr.""" if self._meta.primary_key: - pk_field = self._meta.primary_key.name + if self._meta.composite_key: + pk_field = '(' + ', '.join(self._meta.primary_key.field_names) + ')' + else: + pk_field = self._meta.primary_key.name fields = ['{0}={1!r}'.format(pk_field, self.get_id())] else: pk_field = None From a0c765263709f85e7e58c778487a57fdfd153ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 14:19:05 -0600 Subject: [PATCH 137/197] Add bhm_efeds_veto --- python/sdssdb/peewee/sdss5db/catalogdb.py | 6 + .../v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 110 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index cc3db786..0eb137b7 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -659,5 +659,11 @@ class Meta: print_fields = ['designation'] +class BHM_eFEDS_Veto(CatalogdbModel): + + class Meta: + table_name = 'bhm_efeds_veto' + + _Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql new file mode 100644 index 00000000..8b401299 --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -0,0 +1,110 @@ +/* + +BHM-SPIDERS eFEDS veto catalogue + +* A minimalist catalogue of 6300 science targets in the SPIDERS eFEDS field +that received SDSS-IV spectroscopy during the March2020 SPIDERS observing run +(and hence are not in SDSS-SpecObj-DR16) +* Many of these objects have v low SNR and so we will want to observe them again, +but we will take account of this in the Carton code +* Available now on Utah systems under +* /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/ +* Datamodel: + * This is a subset (in both rows and columns) of the spAll-v5_13_1.fits file + * orig file from: https://data.sdss.org/sas/ebosswork/eboss/spectro/redux/v5_13_1/ + * orig data model at: https://data.sdss.org/datamodel/files/BOSS_SPECTRO_REDUX/RUN2D/spAll.html + + No. Type EXTNAME BITPIX Dimensions(columns) PCOUNT GCOUNT + + 0 PRIMARY 8 0 0 1 + 1 BINTABLE 8 174(26) 6300 0 1 + + Column Name Format Dims Units TLMIN TLMAX + 1 PROGRAMNAME 5A + 2 CHUNK 7A + 3 PLATESN2 E + 4 PLATE J + 5 TILE J + 6 MJD J + 7 FIBERID J + 8 RUN2D 7A + 9 RUN1D 7A + 10 PLUG_RA D + 11 PLUG_DEC D + 12 Z_ERR E + 13 RCHI2 E + 14 DOF J + 15 RCHI2DIFF E + 16 WAVEMIN E + 17 WAVEMAX E + 18 WCOVERAGE E + 19 ZWARNING J + 20 SN_MEDIAN 5E + 21 SN_MEDIAN_ALL E + 22 SPECTROFLUX 5E + 23 SPECTROFLUX_IVAR 5E + 24 ANYANDMASK J + 25 ANYORMASK J + 26 SPECOBJID K + +Note: the SPECOBJID column overflows the data type so it was removed when +converting the FITS table to CSV. + +*/ + +CREATE UNLOGGED TABLE catalogdb.bhm_efeds_veto ( + programname VARCHAR(5), + chunk VARCHAR(7), + platesn2 REAL, + plate INTEGER, + tile INTEGER, + mjd INTEGER, + fiberid INTEGER, + run2d VARCHAR(7), + run1d VARCHAR(7), + plug_ra DOUBLE PRECISION, + plug_dec DOUBLE PRECISION, + z_err REAL, + rchi2 REAL, + dof INTEGER, + rchi2diff REAL, + wavemin REAL, + wavemax REAL, + wcoverage REAL, + zwarning INTEGER, + sn_median REAL[5], + sn_median_all REAL, + spectroflux REAL[5], + spectroflux_ivar REAL[5], + anyandmask INTEGER, + anyormask INTEGER +); + +\COPY catalogdb.bhm_efeds_vetoexit + FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; + +-- Query to create a new column with the correct specobjid. 1301 is the result of +-- (N-5)*10000+M*100+P for RUN2D v5_13_1 (see https://www.sdss.org/dr16/help/glossary/#specobj). +-- These specobjid are uint64 and larger than what can be stored as a BIGINT, so we're not +-- creating them for now. + +-- ALTER TABLE catalogdb.bhm_efeds_vetoexit + ADD COLUMN specobjid BIGINT; +-- UPDATE catalogdb.bhm_efeds_vetoexit + +-- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + +-- ((mjd-50000)::bigint<<24) + (1301::bigint<<10); + +-- CREATE INDEX ON catalogdb.bhm_efeds_vetoexit + (specobjid); + +CREATE INDEX ON catalogdb.bhm_efeds_vetoexit + (mjd, plate, fiberid, run2d); + +CREATE INDEX ON catalogdb.bhm_efeds_vetoexit + (q3c_ang2ipix(plug_ra, plug_dec)); +CLUSTER bhm_efeds_vetoexit +_q3c_ang2ipix_idx ON catalogdb.bhm_efeds_vetoexit +; +ANALYZE catalogdb.bhm_efeds_vetoexit +; From 30236affdfa4cf8a536c4686c1d8dd17daa8bf03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 14:59:27 -0600 Subject: [PATCH 138/197] Use catwise.source_id as pk --- schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql index c5576cf2..17846aca 100644 --- a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql @@ -1,8 +1,8 @@ -- PKs -ALTER TABLE catalogdb.catwise ADD PRIMARY KEY (source_name); -ALTER TABLE catalogdb.catwise_reject ADD PRIMARY KEY (source_name); +ALTER TABLE catalogdb.catwise ADD PRIMARY KEY (source_id); +ALTER TABLE catalogdb.catwise_reject ADD PRIMARY KEY (source_id); -- Indexes From 46783bf52e205ad2749de9898de640470115f2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 15:00:05 -0600 Subject: [PATCH 139/197] Make ls_id the pk --- .../catalogdb/legacy_survey/dr8/legacy_survey_index.sql | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index 7ece7593..b7b75d83 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -1,10 +1,5 @@ -CREATE UNIQUE INDEX CONCURRENTLY ls_id_pk - ON catalogdb.legacy_survey_dr8 (ls_id); - -ALTER TABLE catalogdb.legacy_survey_dr8 - ADD CONSTRAINT ls_id_pk PRIMARY KEY - USING INDEX ls_id_pk; +ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (ls_id); CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; From f99ef8cf5cbe5f7d38e0f0329adffe7838db43ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 15:00:22 -0600 Subject: [PATCH 140/197] FK between gaia clean and source --- schema/sdss5db/catalogdb/foreignKeys.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 48dd3926..e9d46f85 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -9,6 +9,16 @@ https://www.postgresql.org/docs/8.2/static/sql-altertable.html */ + +-- gaia_dr2_clean + +ALTER TABLE catalogdb.gaia_dr2_clean + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + -- gaia_dr2_wd_candidates_v1 ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 From 32595f86534c07e225c373865f1a4e381484ce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 19:42:50 -0600 Subject: [PATCH 141/197] Remove some indexes from gaia --- schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql index 03800c04..56bc42c5 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql @@ -11,12 +11,6 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ra); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (dec); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (l); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (b); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ecl_lon); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (ecl_lat); CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_flux); CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_mag); CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (solution_id); From 79f9f2e457aa81b23ddb46532b8dfc94325fcb1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 28 Mar 2020 21:18:10 -0600 Subject: [PATCH 142/197] Tool to run vacuum on a schema --- python/sdssdb/utils/maintenance.py | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 python/sdssdb/utils/maintenance.py diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/maintenance.py new file mode 100644 index 00000000..7645d8da --- /dev/null +++ b/python/sdssdb/utils/maintenance.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-28 +# @Filename: maintenance.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import sys +import time + + +def vacuum_all(database, analyze=True, schema=None): + """Vacuums all the tables in a database or schema. + + Parameters + ---------- + database : .PeeweeDatabaseConnection + A PeeWee connection to the database to vacuum. + analyze : bool + Whether to run ``ANALYZE`` when vacuumming. + schema : str + The schema to vacuum. If `None`, vacuums the entire database. + + """ + + def execute_sql(statement): + + sys.stdout.write(statement + ' ... ') + sys.stdout.flush() + + tstart = time.time() + + # Change isolation level to allow executing commands such as VACUUM. + connection = database.connection() + original_isolation_level = connection.isolation_level + connection.set_isolation_level(0) + + database.execute_sql(statement) + + tend = time.time() + telapsed = tend - tstart + print(f'{telapsed:.1f} s') + + connection.set_isolation_level(original_isolation_level) + + assert database.is_connection_usable(), 'connection is not usable.' + + if schema is None: + + statement = 'VACUUM' + (' ANALYZE' if analyze else '') + execute_sql(statement) + + return + + tables = database.get_tables(schema=schema) + + for table in sorted(tables): + + table_name = table if schema is None else schema + '.' + table + statement = 'VACUUM' + (' ANALYZE' if analyze else '') + ' ' + table_name + + execute_sql(statement) From 71c4e130b783fd45b121c440a7797442b5e11469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 31 Mar 2020 15:03:33 -0600 Subject: [PATCH 143/197] Supplemental changes to catalogdb --- python/sdssdb/peewee/sdss5db/catalogdb.py | 25 +++---- python/sdssdb/utils/maintenance.py | 24 +++++-- schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql | 2 +- schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql | 2 +- .../v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 2 +- .../eRosita/{v0.1.0 => v0.1.1}/eRosita.sql | 8 +-- .../{v0.1.0 => v0.1.1}/eRosita_index.sql | 0 .../{v0.1.0 => v0.1.1}/eRosita_load.py | 4 +- schema/sdss5db/catalogdb/foreignKeys.sql | 65 +++++++++++++++---- .../dr13/sdss_dr13_photoobj_index.sql | 6 +- .../sdssSpecObj/dr14/sdss_dr14_specObj.sql | 2 +- .../sdssSpecObj/dr16/sdss_dr16_specObj.sql | 5 +- .../catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql | 2 +- .../skymapper/dr1.1/skymapper_indexes.sql | 4 +- .../sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 2 - .../twomass/psc/twomass_psc_index.sql | 2 - .../gaiaDR2/gaiadr2_tmass_best_neighbour.sql | 4 +- .../gaiadr2_sdssdr9_best_neighbour.sql | 8 +-- 18 files changed, 106 insertions(+), 61 deletions(-) rename schema/sdss5db/catalogdb/eRosita/{v0.1.0 => v0.1.1}/eRosita.sql (94%) rename schema/sdss5db/catalogdb/eRosita/{v0.1.0 => v0.1.1}/eRosita_index.sql (100%) rename schema/sdss5db/catalogdb/eRosita/{v0.1.0 => v0.1.1}/eRosita_load.py (92%) rename schema/sdss5db/catalogdb/{unused => xmatch}/gaiadr2_sdssdr9_best_neighbour.sql (54%) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 0eb137b7..6a18e33e 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -26,7 +26,7 @@ class Meta: primary_key = False -_Gaia_DR2_TwoMass_Best_Neighbour_Deferred = DeferredThroughModel() +_Gaia_DR2_TwoMass_Deferred = DeferredThroughModel() _APOGEE_Star_Visit_Deferred = DeferredThroughModel() @@ -52,9 +52,9 @@ class Gaia_DR2(CatalogdbModel): source_id = BigIntegerField(primary_key=True) - tmass_best_sources = ManyToManyField(TwoMassPSC, - through_model=_Gaia_DR2_TwoMass_Best_Neighbour_Deferred, - backref='gaia_best_sources') + tmass_best = ManyToManyField(TwoMassPSC, + through_model=_Gaia_DR2_TwoMass_Deferred, + backref='gaia_best') class Meta: table_name = 'gaia_dr2_source' @@ -72,12 +72,6 @@ class Meta: table_name = 'gaia_dr2_clean' -class Gaia_DR2_SDSS_DR9_Best_Neighbour(CatalogdbModel): - - class Meta: - table_name = 'gaiadr2_sdssdr9_best_neighbour' - - class GalacticGenesis(CatalogdbModel): class Meta: @@ -418,6 +412,9 @@ class BHM_Spiders_Generic_Superset(CatalogdbModel): opt_modelflux_z = FloatField(null=True) opt_modelflux_ivar_z = FloatField(null=True) + ls = ForeignKeyField(Legacy_Survey_DR8, backref='+') + gaia = ForeignKeyField(Gaia_DR2, object_id_name='gaia_dr2_source_id', backref='+') + class Meta: table_name = 'bhm_spiders_generic_superset' use_reflection = False @@ -459,6 +456,12 @@ class Meta: class Gaia_DR2_WD(CatalogdbModel): + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='wd', + unique=True) + @property def sdss(self): """Returns records from `.Gaia_DR2_WD_SDSS` with matching ``wd``.""" @@ -665,5 +668,5 @@ class Meta: table_name = 'bhm_efeds_veto' -_Gaia_DR2_TwoMass_Best_Neighbour_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) +_Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/maintenance.py index 7645d8da..5d453cf9 100644 --- a/python/sdssdb/utils/maintenance.py +++ b/python/sdssdb/utils/maintenance.py @@ -10,7 +10,7 @@ import time -def vacuum_all(database, analyze=True, schema=None): +def vacuum_all(database, analyze=True, verbose=False, schema=None): """Vacuums all the tables in a database or schema. Parameters @@ -19,6 +19,8 @@ def vacuum_all(database, analyze=True, schema=None): A PeeWee connection to the database to vacuum. analyze : bool Whether to run ``ANALYZE`` when vacuumming. + verbose : bool + Whether to run in verbose mode. schema : str The schema to vacuum. If `None`, vacuums the entire database. @@ -26,8 +28,11 @@ def vacuum_all(database, analyze=True, schema=None): def execute_sql(statement): - sys.stdout.write(statement + ' ... ') - sys.stdout.flush() + if 'VEBOSE' in statement: + print(statement + ' ... ') + else: + sys.stdout.write(statement + ' ... ') + sys.stdout.flush() tstart = time.time() @@ -40,7 +45,11 @@ def execute_sql(statement): tend = time.time() telapsed = tend - tstart - print(f'{telapsed:.1f} s') + + if 'VEBOSE' in statement: + print('Elapsed {telap sed:.1f} s') + else: + print(f'{telapsed:.1f} s') connection.set_isolation_level(original_isolation_level) @@ -48,7 +57,7 @@ def execute_sql(statement): if schema is None: - statement = 'VACUUM' + (' ANALYZE' if analyze else '') + statement = 'VACUUM' + (' VEBOSE' if verbose else '') + (' ANALYZE' if analyze else '') execute_sql(statement) return @@ -58,6 +67,9 @@ def execute_sql(statement): for table in sorted(tables): table_name = table if schema is None else schema + '.' + table - statement = 'VACUUM' + (' ANALYZE' if analyze else '') + ' ' + table_name + statement = ('VACUUM' + + (' VEBOSE' if verbose else '') + + (' ANALYZE' if analyze else '') + + ' ' + table_name) execute_sql(statement) diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql index 98206866..c2eb2915 100644 --- a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql @@ -101,7 +101,7 @@ https://irsa.ipac.caltech.edu/data/SPITZER/GLIMPSE/overview.html */ -CREATE UNLOGGED TABLE catalogdb.glimpse ( +CREATE TABLE catalogdb.glimpse ( designation TEXT, tmass_designation VARCHAR(18), tmass_cntr INTEGER, diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql index b4a09578..71b20640 100644 --- a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql @@ -5,7 +5,7 @@ PS1 gMeanPSFMag < 18. */ -CREATE UNLOGGED TABLE catalogdb.ps1_g18 ( +CREATE TABLE catalogdb.ps1_g18 ( objid BIGINT, ndetections INTEGER, ramean DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql index 8b401299..7d2b0c06 100644 --- a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -52,7 +52,7 @@ converting the FITS table to CSV. */ -CREATE UNLOGGED TABLE catalogdb.bhm_efeds_veto ( +CREATE TABLE catalogdb.bhm_efeds_veto ( programname VARCHAR(5), chunk VARCHAR(7), platesn2 REAL, diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita.sql similarity index 94% rename from schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql rename to schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita.sql index a9674a7e..130036d9 100644 --- a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita.sql +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita.sql @@ -9,8 +9,8 @@ https://github.com/sdss/sdssdb/blob/add_bhm_peewee_classes1/python/sdssdb/peewee see classes 'BhmSpidersAgnSuperset' and 'BhmSpidersClustersSuperset' ##Filename rows -# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.0.fits 23994 -# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.0.fits 19636 +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits 23994 +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.1.fits 19636 Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/eRosita/eFEDS_for_catalogdb_v0 @@ -19,7 +19,7 @@ Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/eRosita/eFED CREATE TABLE catalogdb.bhm_spiders_clusters_superset ( pk BIGSERIAL PRIMARY KEY, ero_version TEXT, - ero_souuid TEXT, + ero_detuid TEXT, ero_flux REAL, ero_flux_err REAL, ero_ext REAL, @@ -63,7 +63,7 @@ CREATE TABLE catalogdb.bhm_spiders_clusters_superset ( CREATE TABLE catalogdb.bhm_spiders_agn_superset ( pk BIGSERIAL PRIMARY KEY, ero_version TEXT, - ero_souuid TEXT, + ero_detuid TEXT, ero_flux REAL, ero_flux_err REAL, ero_ext REAL, diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_index.sql similarity index 100% rename from schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_index.sql rename to schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_index.sql diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py b/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py similarity index 92% rename from schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py rename to schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py index fd62c5b6..fd97ca3f 100755 --- a/schema/sdss5db/catalogdb/eRosita/v0.1.0/eRosita_load.py +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py @@ -20,9 +20,9 @@ def main(): path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' 'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/') - for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.0.fits', + for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.1.fits', 'bhm_spiders_clusters_superset'), - ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.0.fits', + ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits', 'bhm_spiders_agn_superset')): data = astropy.table.Table.read(path_ + file_) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index e9d46f85..b11b2ea5 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -19,18 +19,45 @@ ALTER TABLE catalogdb.gaia_dr2_clean ON UPDATE CASCADE ON DELETE CASCADE; --- gaia_dr2_wd_candidates_v1 +-- gaiadr2_sdssdr9_best_neighbour -ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 +-- Some objectids are not present in dr13_photooj. It should not matter +-- because the TIC has x-matching with SDSS. + +-- CREATE INDEX ON catalogdb.gaiadr2_sdssdr9_best_neighbour (sdssdr9_oid); + +-- ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour +-- ADD CONSTRAINT source_id_fk +-- FOREIGN KEY (source_id) +-- REFERENCES catalogdb.gaia_dr2_source (source_id) +-- ON UPDATE CASCADE ON DELETE CASCADE; + +-- ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour +-- ADD CONSTRAINT sdssdr9_oid_fk +-- FOREIGN KEY (sdssdr9_oid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaiadr2_tmass_best_neighbour + +CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT tmass_pts_key_fk + FOREIGN KEY (tmass_pts_key) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; + -- gaiadr2_tmass_best_neighbour -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (source_id ASC); CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour @@ -48,23 +75,27 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour -- sdss_dr14_specobj +UPDATE catalogdb.sdss_dr14_specobj SET bestobjid = NULL WHERE bestobjid = 0; CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); -ALTER TABLE catalogdb.sdss_dr14_specobj - ADD CONSTRAINT bestobjid_fk - FOREIGN KEY (bestobjid) - REFERENCES catalogdb.sdss_dr13_photoobj (objid) - ON UPDATE CASCADE ON DELETE CASCADE; +-- Cannot be created because some bestobjids are not present in dr13_photoobj. +-- ALTER TABLE catalogdb.sdss_dr14_specobj +-- ADD CONSTRAINT bestobjid_fk +-- FOREIGN KEY (bestobjid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; -- sdss_dr16_specobj +UPDATE catalogdb.sdss_dr16_specobj SET bestobjid = NULL WHERE bestobjid = 0; CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); -ALTER TABLE catalogdb.sdss_dr16_specobj - ADD CONSTRAINT bestobjid_fk - FOREIGN KEY (bestobjid) - REFERENCES catalogdb.sdss_dr13_photoobj (objid) - ON UPDATE CASCADE ON DELETE CASCADE; +-- Cannot be created because some bestobjids are not present in dr13_photoobj. +-- ALTER TABLE catalogdb.sdss_dr16_specobj +-- ADD CONSTRAINT bestobjid_fk +-- FOREIGN KEY (bestobjid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; -- gaia_unwise_agn @@ -222,3 +253,11 @@ ALTER TABLE catalogdb.glimpse FOREIGN KEY (tmass_cntr) REFERENCES catalogdb.twomass_psc (pts_key) ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaia_dr2_wd + +ALTER TABLE catalogdb.gaia_dr2_wd + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql index 895873cd..0c9af804 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql @@ -7,12 +7,8 @@ indices for catalogdb tables, to be run after bulk uploads CREATE INDEX ON catalogdb.sdss_dr13_photoobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr13_photoobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr13_photoobj; -ANALYZE catalogdb.sdss_dr13_photoobj; +VACUUM ANALYZE catalogdb.sdss_dr13_photoobj; -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (ra); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (dec); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (l); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (b); CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_u); CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_g); CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_r); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql index e4ba64ee..5beadcf4 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql @@ -408,7 +408,7 @@ CREATE INDEX ON catalogdb.sdss_dr14_specobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_specobj; ANALYZE catalogdb.sdss_dr14_specobj; -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (bestObjID); CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); CREATE INDEX ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid); +CREATE INDEX ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index 5be41e3d..111562d4 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -408,11 +408,8 @@ CREATE INDEX ON catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr16_specobj; ANALYZE catalogdb.sdss_dr16_specobj; -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (bestObjID); CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fluxObjID); CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (mjd); -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (plate); -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fiberid); CREATE INDEX ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid); +CREATE INDEX ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql index 6261dd92..8096ef9c 100644 --- a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql @@ -1,4 +1,4 @@ -CREATE UNLOGGED TABLE IF NOT EXISTS "catalogdb"."sdss_dr14_qso" ( +CREATE TABLE IF NOT EXISTS "catalogdb"."sdss_dr14_qso" ( "sdss_name" VARCHAR(18), "ra" DOUBLE PRECISION, "dec" DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql index cb9cddca..88b43505 100644 --- a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql @@ -6,9 +6,9 @@ CREATE UNIQUE INDEX CONCURRENTLY object_id_pk ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT object_id_pk PRIMARY KEY - USING INDEX object_id_pk; -- blocks queries, but only very briefly + USING INDEX object_id_pk; -- blocks queries, but only very briefly -CREATE INDEX ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, decj2000)); +CREATE INDEX ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, dej2000)); CLUSTER skymapper_dr1_1_q3c_ang2ipix_idx ON catalogdb.skymapper_dr1_1; ANALYZE catalogdb.skymapper_dr1_1; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index 8c638215..c7fdbaf1 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -11,8 +11,6 @@ CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.tic_v8; ANALYZE catalogdb.tic_v8; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gallong); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gallat); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Bmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Vmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (umag); diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql index cb051987..d5a45acc 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql @@ -10,8 +10,6 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (ra); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (decl); CREATE INDEX ON catalogdb.twomass_psc USING BTREE (j_m); CREATE INDEX ON catalogdb.twomass_psc USING BTREE (h_m); CREATE INDEX ON catalogdb.twomass_psc USING BTREE (k_m); diff --git a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql index 54bcfdef..e27f3295 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql @@ -18,7 +18,9 @@ CREATE TABLE catalogdb.gaiadr2_tmass_best_neighbour( tmass_pts_key integer ); -\copy catalogdb.gaiadr2_tmass_best_neighbour FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' WITH CSV HEADER; +\copy catalogdb.gaiadr2_tmass_best_neighbour + FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' + WITH CSV HEADER; ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD PRIMARY KEY (source_id); diff --git a/schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql similarity index 54% rename from schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql rename to schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql index b7c17e6d..399ca862 100644 --- a/schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql @@ -18,8 +18,8 @@ CREATE TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour( ); -\copy catalogdb.gaiadr2_sdssdr9_best_neighbour FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' WITH CSV HEADER; +\COPY catalogdb.gaiadr2_sdssdr9_best_neighbour + FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' + WITH CSV HEADER; -alter table catalogdb.gaiadr2_sdssdr9_best_neighbour add primary key(source_id); - -CREATE INDEX CONCURRENTLY gaiadr2_sdssdr9_best_neighbour_sdssdr9_oid_index ON catalogdb.gaiadr2_sdssdr9_best_neighbour using BTREE (sdssdr9_oid); \ No newline at end of file +ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour ADD PRIMARY KEY (source_id); From b708abe692769821a243037e0ece1fe15d93d6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 31 Mar 2020 20:00:37 -0700 Subject: [PATCH 144/197] Fix comment --- .../v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql index 7d2b0c06..a1b203b4 100644 --- a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -80,7 +80,7 @@ CREATE TABLE catalogdb.bhm_efeds_veto ( anyormask INTEGER ); -\COPY catalogdb.bhm_efeds_vetoexit +\COPY catalogdb.bhm_efeds_veto FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; -- Query to create a new column with the correct specobjid. 1301 is the result of @@ -88,23 +88,17 @@ CREATE TABLE catalogdb.bhm_efeds_veto ( -- These specobjid are uint64 and larger than what can be stored as a BIGINT, so we're not -- creating them for now. --- ALTER TABLE catalogdb.bhm_efeds_vetoexit +-- ALTER TABLE catalogdb.bhm_efeds_veto ADD COLUMN specobjid BIGINT; --- UPDATE catalogdb.bhm_efeds_vetoexit +-- UPDATE catalogdb.bhm_efeds_veto -- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + -- ((mjd-50000)::bigint<<24) + (1301::bigint<<10); --- CREATE INDEX ON catalogdb.bhm_efeds_vetoexit - (specobjid); +-- CREATE INDEX ON catalogdb.bhm_efeds_veto (specobjid); -CREATE INDEX ON catalogdb.bhm_efeds_vetoexit - (mjd, plate, fiberid, run2d); +CREATE INDEX ON catalogdb.bhm_efeds_veto (mjd, plate, fiberid, run2d); -CREATE INDEX ON catalogdb.bhm_efeds_vetoexit - (q3c_ang2ipix(plug_ra, plug_dec)); -CLUSTER bhm_efeds_vetoexit -_q3c_ang2ipix_idx ON catalogdb.bhm_efeds_vetoexit -; -ANALYZE catalogdb.bhm_efeds_vetoexit -; +CREATE INDEX ON catalogdb.bhm_efeds_veto (q3c_ang2ipix(plug_ra, plug_dec)); +CLUSTER bhm_efeds_veto_q3c_ang2ipix_idx ON catalogdb.bhm_efeds_veto; +ANALYZE catalogdb.bhm_efeds_veto; From 9e657f18a09e0f6b1e561a3c3b1e8feeb4022615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:26:20 -0700 Subject: [PATCH 145/197] schemadisplay: better handing of indexes and q3c --- python/sdssdb/utils/schemadisplay.py | 30 ++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index e4803c20..736f7a24 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -112,20 +112,38 @@ def format_field_str(field): # Add indexes and unique constraints if show_indices: - indexes = [index for index in model._meta.fields_to_index() - if not isinstance(index._expressions[0], ForeignKeyField)] + if model._meta.database.connected: + indexes = model._meta.database.get_indexes(model._meta.table_name, + schema=model._meta.schema) + else: + indexes = [index._expressions[0] for index in model._meta.fields_to_index() + if not isinstance(index._expressions[0], ForeignKeyField)] if len(indexes) > 0: html += '' for index in indexes: - column_name = index._expressions[0].column_name + column_names = index.columns + ilabel = 'INDEX' + + if len(column_names) == 1: + column_name = column_names[0] + if column_name == '': + match = re.match(r'.+q3c_ang2ipix\("*(\w+)"*, "*(\w+)"*\).+', + index.sql) + if match: + column_name = '(' + ', '.join(match.groups()) + ')' + ilabel = 'Q3C' + else: + continue + else: + column_name = '(' + ', '.join(column_names) + ')' - if index._unique: + if index.unique: + if column_name == pk.column_name: + continue ilabel = 'UNIQUE' - else: - ilabel = 'INDEX' html += f'{ilabel} {column_name}' From 583d8bba7a36cbf9bad702e823f05d92c5e63ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:26:48 -0700 Subject: [PATCH 146/197] schemadisplay: better field formatting --- python/sdssdb/utils/schemadisplay.py | 42 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index 736f7a24..923fbf10 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -66,10 +66,12 @@ def format_field_str(field): """Add in (PK) OR (FK) suffixes to column names.""" column_name = field.column_name + if model._meta.composite_key: + column_name = '(' + ', '.join(pk.field_names) + ')' suffixes = [] - if column_name in pk_col_names: + if column_name in pk_col_names or column_name == '__composite_key__': suffixes.append('PK') if column_name in fk_col_names: suffixes.append('FK') @@ -85,27 +87,41 @@ def format_field_str(field): return f'- {column_name}{suffix}' html = (f'<' - f'' - f'') + f'') added_col_name = [] fields_html = [] + pk = model._meta.primary_key + if show_pks and pk: + if model._meta.composite_key: + column_name = '(' + ', '.join(pk.field_names) + ')' + else: + column_name = pk.column_name + fields_html.append( + ''.format( + column_name, format_field_str(pk))) + # Add a row for each column in the table. - for field in fields.values(): + if show_columns: + for field in fields.values(): - column_name = field.column_name + if field.primary_key: + continue - # Avoids repeating columns. This can happen if there are multiple - # FKs pointing to the same column. - if column_name in added_col_name: - continue + column_name = field.column_name - fields_html.append( - ''.format( - column_name, format_field_str(field))) + # Avoids repeating columns. This can happen if there are multiple + # FKs pointing to the same column. + if column_name in added_col_name: + continue + + fields_html.append( + ''.format( + column_name, format_field_str(field))) - added_col_name.append(column_name) + added_col_name.append(column_name) html += ''.join(fields_html) From 428da150ec984667c50110361ac2072ad458935f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:27:31 -0700 Subject: [PATCH 147/197] schemadisplay: allow to show/hide columns and pks --- python/sdssdb/utils/schemadisplay.py | 63 +++++++++++++++++++++------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index 923fbf10..072f2b10 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -12,6 +12,8 @@ # The following functions are adapted from the sqlalchemy_schemadisplay by # Florian Schulze (https://github.com/fschulze/sqlalchemy_schemadisplay). +import re + import pydot from peewee import ForeignKeyField @@ -39,13 +41,18 @@ 'VARCHAR': 'VARCHAR'} -def _render_table_html(model, show_indices=True, show_datatypes=True): +def _render_table_html(model, show_columns=True, show_pks=True, + show_indices=True, show_datatypes=True): """Creates the HTML tags for a table, including PKs, FKs, and indices. Parameters ---------- model : `peewee.Model` The Peewee model for which to create the table. + show_columns : bool + Whether to show the column names. + show_pks : bool + Whether to show the primary key. Supersedes ``show_columns``. show_indices : `bool` Whether to show the indices from the table as separate rows. show_datatypes : `bool` @@ -168,8 +175,9 @@ def format_field_str(field): return html -def create_schema_graph(models=None, base=None, schema=None, show_indices=True, - show_datatypes=True, font='Bitstream-Vera Sans', +def create_schema_graph(models=None, base=None, schema=None, show_columns=True, + show_pks=True, show_indices=True, show_datatypes=True, + skip_tables=[], font='Bitstream-Vera Sans', concentrate=True, relation_options={}, rankdir='TB'): """Creates a graph visualisation from a series of Peewee models. @@ -179,27 +187,33 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, Parameters ---------- - models : `list` + models : list A list of Peewee `models ` to be graphed. - base : `peewee:Model` + base : peewee:Model A base model class. If passed, all the model classes that were created by subclassing from the base model will be used. - schema : `str` + schema : str A schema name. If passed, will be used to limit the list of models or ``base`` subclasses to only the models that match the schema name. - show_indices : `bool` + show_columns : bool + Whether to show the column names. + show_pks : bool + Whether to show the primary key. Supersedes ``show_columns``. + show_indices : bool Whether to show the indices from the table as separate rows. - show_datatypes : `bool` + show_datatypes : bool Whether to show the data type of each column. - font : `str` + skip_tables : list + List of table names to skip. + font : str The name of the font to use. - relation_options : `dict` + relation_options : dict Additional parameters to be passed to ``pydot.Edge`` when creating the relationships. Returns ------- - graph : pydot.Dot + graph : `pydot.Dot` A ``pydot.Dot`` object with the graph representation of the schema. Example @@ -217,7 +231,13 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, relation_kwargs.update(relation_options) if base and not models: - models = base.__subclasses__() + models = set(base.__subclasses__()) + while True: + old_models = models.copy() + for model in old_models: + models |= set(model.__subclasses__()) + if models == old_models: + break if schema: models = [model for model in models if model._meta.schema == schema] @@ -231,10 +251,18 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, for model in models: + if model._meta.table_name in skip_tables: + continue + + if model._meta.database.connected and not model.table_exists(): + continue + graph.add_node( pydot.Node(str(model._meta.table_name), shape='plaintext', label=_render_table_html(model, + show_columns=show_columns, + show_pks=show_pks, show_indices=show_indices, show_datatypes=show_datatypes), fontname=font, @@ -247,8 +275,13 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, field.rel_model not in models): continue - from_col_name = field.column_name + from_col_name = '+ ' + field.column_name + to_col_name = field.rel_field.column_name + if field.rel_field.primary_key: + to_col_name = '' + else: + to_col_name = '+ ' + to_col_name edge = [model._meta.table_name, field.rel_model._meta.table_name] @@ -260,8 +293,8 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, graph_edge = pydot.Edge( dir='both', - headlabel=f'+ {to_col_name}', - taillabel=f'+ {from_col_name}', + headlabel=to_col_name, + taillabel=from_col_name, arrowhead='none', arrowtail='none', # arrowhead=is_inheritance and 'none' or 'odot', From f99d46b295458bf6bbeacc990404f0985488b4f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:27:54 -0700 Subject: [PATCH 148/197] Typos in veto catalogue SQL --- .../SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql index a1b203b4..b6e9c167 100644 --- a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -81,22 +81,22 @@ CREATE TABLE catalogdb.bhm_efeds_veto ( ); \COPY catalogdb.bhm_efeds_veto - FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; + FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv + WITH CSV HEADER DELIMITER E'\t' NULL '\N'; -- Query to create a new column with the correct specobjid. 1301 is the result of -- (N-5)*10000+M*100+P for RUN2D v5_13_1 (see https://www.sdss.org/dr16/help/glossary/#specobj). -- These specobjid are uint64 and larger than what can be stored as a BIGINT, so we're not -- creating them for now. --- ALTER TABLE catalogdb.bhm_efeds_veto - ADD COLUMN specobjid BIGINT; +-- ALTER TABLE catalogdb.bhm_efeds_veto ADD COLUMN specobjid BIGINT; -- UPDATE catalogdb.bhm_efeds_veto - -- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + -- ((mjd-50000)::bigint<<24) + (1301::bigint<<10); - -- CREATE INDEX ON catalogdb.bhm_efeds_veto (specobjid); +ALTER TABLE catalogdb.bhm_efeds_veto ADD PRIMARY KEY (plate, mjd, fiberid, run2d); + CREATE INDEX ON catalogdb.bhm_efeds_veto (mjd, plate, fiberid, run2d); CREATE INDEX ON catalogdb.bhm_efeds_veto (q3c_ang2ipix(plug_ra, plug_dec)); From 2aa490f4800ed0de4889353dd501403d3963871d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:28:19 -0700 Subject: [PATCH 149/197] Fix fk TIC->photoobj --- schema/sdss5db/catalogdb/foreignKeys.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index b11b2ea5..33ab7d46 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -150,7 +150,7 @@ CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT sdss_fk - FOREIGN KEY (sdss_int) + FOREIGN KEY (sdss) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; From af4bbf514c8d2846e77f4844427a965c88fa3a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:29:56 -0700 Subject: [PATCH 150/197] Better logging of exceptions in reflection --- python/sdssdb/peewee/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 44fd1028..fc183001 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -125,9 +125,9 @@ def reflect(self): try: ReflectedModel = generate_models(database, schema=schema, - table_names=table_name)[table_name] - except KeyError: - log.debug('reflection failed for {}'.format(table_name)) + table_names=[table_name])[table_name] + except Exception as ee: + log.debug('reflection failed for {}: {}'.format(table_name, ee)) return for field_name, field in ReflectedModel._meta.fields.items(): From ca3ba27351eaf9a74cf5e76813ed02cc556b9b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:30:06 -0700 Subject: [PATCH 151/197] Do not add indexes in reflection --- python/sdssdb/peewee/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index fc183001..50f1e605 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -155,15 +155,6 @@ def reflect(self): pk = ReflectedModel._meta.primary_key self._meta.set_primary_key(pk.name, pk) - for index in ReflectedModel._meta.indexes: - - # TODO: this probably can never work. Find a better way of determining - # if the index has been added manually. - if index in self._meta.indexes: - continue - - self._meta.indexes.append(index) - class BaseModel(Model, metaclass=ReflectMeta): """A custom peewee `.Model` with enhanced representation and methods. From a10f2cd0a1edb3d35f6973b09cdb8892e28af6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:33:13 -0700 Subject: [PATCH 152/197] Do not reflect models with non-existing tables --- python/sdssdb/peewee/__init__.py | 3 ++- python/sdssdb/peewee/sdss5db/catalogdb.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 50f1e605..9f907cbb 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -101,7 +101,8 @@ def __new__(cls, name, bases, attrs): database.models.append(Model) if Model._meta.use_reflection and database and database.connected: - cls.reflect(Model) + if Model.table_exists(): + cls.reflect(Model) return Model diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 6a18e33e..0b2379e3 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -416,7 +416,6 @@ class BHM_Spiders_Generic_Superset(CatalogdbModel): gaia = ForeignKeyField(Gaia_DR2, object_id_name='gaia_dr2_source_id', backref='+') class Meta: - table_name = 'bhm_spiders_generic_superset' use_reflection = False From e70168b1dda1887eee9512e6ba857d780c13d9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:33:35 -0700 Subject: [PATCH 153/197] Fix FK TIC-photoobj in ORM --- python/sdssdb/peewee/sdss5db/catalogdb.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 0b2379e3..23d36174 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -201,22 +201,6 @@ class Meta: table_name = 'sdss_dr16_specobj' -class TwoMass_Clean(CatalogdbModel): - - pts_key = ForeignKeyField(TwoMassPSC, - backref='tmass_clean', - lazy_load=True) - - class Meta: - table_name = 'twomass_clean' - - -class TwoMass_Clean_No_Neighbor(CatalogdbModel): - - class Meta: - table_name = 'twomass_clean_noneighbor' - - class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): source_id = ForeignKeyField(Gaia_DR2, @@ -496,7 +480,7 @@ class TIC_v8(CatalogdbModel): backref='tic') photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, field='objid', - column_name='sdss_int', object_id_name='sdss_int', + column_name='sdss', object_id_name='sdss', backref='tic') gaia = ForeignKeyField(Gaia_DR2, field='source_id', From aa4962cb906fc14dac5f2abfd9eac905bd172e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 08:39:01 -0700 Subject: [PATCH 154/197] schemadisplay: aesthetics --- python/sdssdb/utils/schemadisplay.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index 072f2b10..11b5b99c 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -143,7 +143,7 @@ def format_field_str(field): if not isinstance(index._expressions[0], ForeignKeyField)] if len(indexes) > 0: - html += '' + first = True for index in indexes: @@ -168,6 +168,10 @@ def format_field_str(field): continue ilabel = 'UNIQUE' + if first: + html += '' + first = False + html += f'' html += '
{table_name}
' + f'{table_name}
{}
{}
{}
{ilabel} {column_name}
>' From f052869aadd6393ce29cb3dc0b938eec26f10628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 09:30:29 -0700 Subject: [PATCH 155/197] schemadisplay: check if pk exists --- python/sdssdb/utils/schemadisplay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index 11b5b99c..b769e431 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -164,7 +164,7 @@ def format_field_str(field): column_name = '(' + ', '.join(column_names) + ')' if index.unique: - if column_name == pk.column_name: + if pk and column_name == pk.column_name: continue ilabel = 'UNIQUE' From 419809ee95324ab0a283ac13a3002c234917564f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 09:32:57 -0700 Subject: [PATCH 156/197] Deal with pks that are also fks in reflection --- python/sdssdb/peewee/__init__.py | 24 ++++++++++++++--------- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 ++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 9f907cbb..05a8285b 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -84,9 +84,10 @@ class Meta: it's not possible to set the through model based on the reflected information. - - When the primary key of a model is also a foreign key, it needs to be - defined explicitely. Otherwise the field will be added but not marked as - primary key. + - When the primary key of a model is also a foreign key and + ``reflection_options = {'skip_foreign_keys': True}``, both the primary key + and the foreign key need to be defined explicitely. Otherwise neither + will be added. """ @@ -121,6 +122,9 @@ def reflect(self): if not database.is_connection_usable(): raise peewee.DatabaseError('database not connected.') + skip_fks = (hasattr(self._meta, 'reflection_options') and + self._meta.reflection_options.get('skip_foreign_keys', False)) + table_name = self._meta.table_name schema = self._meta.schema @@ -138,9 +142,7 @@ def reflect(self): if not hasattr(meta_field, 'reflected') or not meta_field.reflected: continue - if (isinstance(field, peewee.ForeignKeyField) and - hasattr(self._meta, 'reflection_options') and - self._meta.reflection_options.get('skip_foreign_keys', False)): + if isinstance(field, peewee.ForeignKeyField) and skip_fks: continue if field.primary_key: @@ -151,10 +153,14 @@ def reflect(self): self._meta.fields[field_name].reflected = True # Composite keys are not a normal column so if the pk has not been - # set already, check if it exists in the reflected model. + # set already, check if it exists in the reflected model. We avoid + # adding pks that are if not self._meta.primary_key and ReflectedModel._meta.primary_key: - pk = ReflectedModel._meta.primary_key - self._meta.set_primary_key(pk.name, pk) + if isinstance(ReflectedModel._meta.primary_key, peewee.ForeignKeyField) and skip_fks: + pass + else: + pk = ReflectedModel._meta.primary_key + self._meta.set_primary_key(pk.name, pk) class BaseModel(Model, metaclass=ReflectMeta): diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 23d36174..79d7b6e9 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -62,6 +62,8 @@ class Meta: class Gaia_DR2_Clean(CatalogdbModel): + source_id = BigIntegerField(primary_key=True) + source = ForeignKeyField(Gaia_DR2, field='source_id', column_name='source_id', From 4cda9e41f487b1637b8f7d554f54fe7c47c93008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 12:00:46 -0700 Subject: [PATCH 157/197] Tweak display of composite keys in schema --- python/sdssdb/utils/schemadisplay.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index b769e431..aa103f49 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -63,29 +63,30 @@ def _render_table_html(model, show_columns=True, show_pks=True, table_name = model._meta.table_name fields = model._meta.fields - pk_col_names = set([fields[field_name].column_name for field_name in fields - if fields[field_name].primary_key]) + # pk_col_names = set([fields[field_name].column_name for field_name in fields + # if fields[field_name].primary_key]) - fk_col_names = set([fields[field_name].column_name for field_name in fields - if isinstance(fields[field_name], ForeignKeyField)]) + # fk_col_names = set([fields[field_name].column_name for field_name in fields + # if isinstance(fields[field_name], ForeignKeyField)]) def format_field_str(field): """Add in (PK) OR (FK) suffixes to column names.""" + suffixes = [] + column_name = field.column_name - if model._meta.composite_key: + if column_name == '__composite_key__': column_name = '(' + ', '.join(pk.field_names) + ')' + suffixes.append('PK') # Composite keys get .primary_key == False - suffixes = [] - - if column_name in pk_col_names or column_name == '__composite_key__': + if field.primary_key: suffixes.append('PK') - if column_name in fk_col_names: + if isinstance(field, ForeignKeyField): suffixes.append('FK') suffix = ' (' + ', '.join(suffixes) + ')' if len(suffixes) > 0 else '' - if show_datatypes: + if show_datatypes and field.column_name != '__composite_key__': field_type = field.field_type if field_type in field_type_psql: field_type = field_type_psql[field_type] From ff4fc71a6f3d618db80042ddc4022a039f70eb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 16:00:47 -0600 Subject: [PATCH 158/197] Add transitional_msps --- python/sdssdb/peewee/sdss5db/catalogdb.py | 13 ++++++++ .../legacy_survey/dr8/legacy_survey_index.sql | 9 ++++++ .../transitional_msps/transitional_msps.sql | 30 +++++++++++++++++ .../transitional_msps_load.py | 32 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql create mode 100755 schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 79d7b6e9..dde4553f 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -578,6 +578,19 @@ class Meta: table_name = 'galactic_millisecond_pulsars' +class TransitionalMillisecondPulsars(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'transitional_msps' + + class ATNF(CatalogdbModel): name = TextField(primary_key=True) diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index b7b75d83..f1619a5a 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -1,6 +1,15 @@ ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (ls_id); +CREATE INDEX ON catalogdb.legacy_survey_dr8 (ref_id); +CREATE INDEX ON catalogdb.legacy_survey_dr8 (ref_cat); + +-- Create new column with the source_id for Gaia where ref_cat=G2 +ALTER TABLE catalogdb.legacy_survey_dr8 ADD COLUMN gaia_sourceid BIGINT; +UPDATE catalogdb.legacy_survey_dr8 SET gaia_sourceid = ref_id + WHERE ref_cat = 'G2'; +CREATE INDEX ON catalogdb.legacy_survey_dr8 (gaia_sourceid); + CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql new file mode 100644 index 00000000..f72ae0f7 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql @@ -0,0 +1,30 @@ +/* + +Accreting X-ray Binary Pulsars http://www.iasfbo.inaf.it/~mauro/pulsar_list.html + +*/ + +CREATE TABLE catalogdb.transitional_msps ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX ON catalogdb.transitional_msps (q3c_ang2ipix(radeg, decdeg)); +CLUSTER transitional_msps_q3c_ang2ipix_idx ON catalogdb.transitional_msps; +ANALYZE catalogdb.transitional_msps; + +ALTER TABLE catalogdb.transitional_msps + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py new file mode 100755 index 00000000..f6ae7f33 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: transitional_msps.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/slavko_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'transitional_msps', schema='catalogdb') + + +if __name__ == '__main__': + + main() From 5d72e4d6e57fb0d0f9de3d75b6ea192aa918c5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 16:30:02 -0600 Subject: [PATCH 159/197] \COPY must be written in a single line --- .../bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 4 +--- .../gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql | 4 +--- .../catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql index b6e9c167..047f1626 100644 --- a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -80,9 +80,7 @@ CREATE TABLE catalogdb.bhm_efeds_veto ( anyormask INTEGER ); -\COPY catalogdb.bhm_efeds_veto - FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv - WITH CSV HEADER DELIMITER E'\t' NULL '\N'; +\COPY catalogdb.bhm_efeds_veto FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; -- Query to create a new column with the correct specobjid. 1301 is the result of -- (N-5)*10000+M*100+P for RUN2D v5_13_1 (see https://www.sdss.org/dr16/help/glossary/#specobj). diff --git a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql index e27f3295..1c825ee1 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql @@ -18,9 +18,7 @@ CREATE TABLE catalogdb.gaiadr2_tmass_best_neighbour( tmass_pts_key integer ); -\copy catalogdb.gaiadr2_tmass_best_neighbour - FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' - WITH CSV HEADER; +\COPY catalogdb.gaiadr2_tmass_best_neighbour FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' WITH CSV HEADER; ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD PRIMARY KEY (source_id); diff --git a/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql index 399ca862..bba4905b 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql @@ -18,8 +18,6 @@ CREATE TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour( ); -\COPY catalogdb.gaiadr2_sdssdr9_best_neighbour - FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' - WITH CSV HEADER; +\COPY catalogdb.gaiadr2_sdssdr9_best_neighbour FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' WITH CSV HEADER; ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour ADD PRIMARY KEY (source_id); From 81609a91d8291d51b3d6cd5d034ac050ee896df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 16:33:03 -0600 Subject: [PATCH 160/197] Add best_brightest --- python/sdssdb/peewee/sdss5db/catalogdb.py | 6 +++ .../best_brightest/best_brightest.sql | 53 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 schema/sdss5db/catalogdb/best_brightest/best_brightest.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index dde4553f..9d6350bd 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -666,5 +666,11 @@ class Meta: table_name = 'bhm_efeds_veto' +class BestBrightest(CatalogdbModel): + + class Meta: + table_name = 'best_brightest' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql new file mode 100644 index 00000000..07a85d88 --- /dev/null +++ b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql @@ -0,0 +1,53 @@ +/* + +Best & Brightest Catalog + +*/ + + +CREATE TABLE catalogdb.best_brightest ( + designation VARCHAR(19), + ra_1 DOUBLE PRECISION, + dec_1 DOUBLE PRECISION, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + w1mpro REAL, + w2mpro REAL, + w3mpro REAL, + w4mpro VARCHAR(6), + pmra INTEGER, + pmdec INTEGER, + j_m_2mass REAL, + h_m_2mass REAL, + k_m_2mass REAL, + ra_2 DOUBLE PRECISION, + raerr DOUBLE PRECISION, + dec_2 DOUBLE PRECISION, + decerr DOUBLE PRECISION, + nobs INTEGER, + mobs INTEGER, + vjmag REAL, + bjmag REAL, + gmag REAL, + rmag REAL, + imag REAL, + evjmag REAL, + ebjmag REAL, + egmag REAL, + ermag REAL, + eimag REAL, + name INTEGER, + separation DOUBLE PRECISION, + ebv REAL, + version INTEGER, + original_ext_source_id VARCHAR(16) +); + + +\COPY catalogdb.best_brightest FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/best_brightest/sdss_v_best_and_brightest_merged.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.best_brightest ADD PRIMARY KEY (designation); + +CREATE INDEX ON catalogdb.best_brightest (q3c_ang2ipix(ra_1, dec_1)); +CLUSTER best_brightest_q3c_ang2ipix_idx ON catalogdb.best_brightest; +ANALYZE catalogdb.best_brightest; From b53fc60402c7865e08a2c12becb50a97d8f2a893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 17:15:57 -0600 Subject: [PATCH 161/197] Add skymapper_gaia --- python/sdssdb/peewee/sdss5db/catalogdb.py | 19 +++++++++++ schema/sdss5db/catalogdb/smgaia/smgaia.sql | 38 ++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 schema/sdss5db/catalogdb/smgaia/smgaia.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 9d6350bd..19961baf 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -672,5 +672,24 @@ class Meta: table_name = 'best_brightest' +class SkyMapperGaia(CatalogdbModel): + + skymapper_object_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + skymapper = ForeignKeyField(SkyMapper_DR1_1, + column_name='skymapper_object_id', + object_id_name='skymapper_object_id', + backref='+') + + class Meta: + table_name = 'skymapper_gaia' + print_fields = ['gaia_source_id'] + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/smgaia/smgaia.sql b/schema/sdss5db/catalogdb/smgaia/smgaia.sql new file mode 100644 index 00000000..a3112e40 --- /dev/null +++ b/schema/sdss5db/catalogdb/smgaia/smgaia.sql @@ -0,0 +1,38 @@ +/* + +SkyMapper-Gaia + +http://skymapper.anu.edu.au/_data/sm-gaia/ + +*/ + + +CREATE TABLE catalogdb.skymapper_gaia ( + skymapper_object_id BIGINT, + gaia_source_id BIGINT, + teff REAL, + e_teff REAL, + feh REAL, + e_feh REAL +); + + +\COPY catalogdb.skymapper_gaia FROM PROGRAM 'zcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/SMGaia/SMGaia.csv.gz' WITH CSV HEADER DELIMITER ','; + + +CREATE INDEX ON catalogdb.skymapper_gaia (gaia_source_id); +CREATE INDEX ON catalogdb.skymapper_gaia (teff); +CREATE INDEX ON catalogdb.skymapper_gaia (feh); + + +ALTER TABLE catalogdb.skymapper_gaia + ADD CONSTRAINT skymapper_object_id_fk + FOREIGN KEY (skymapper_object_id) + REFERENCES catalogdb.skymapper_dr1_1 (object_id) + ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_gaia + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON DELETE CASCADE; From 6b861de31751b9769379ba87079d75613b571814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 19:38:09 -0700 Subject: [PATCH 162/197] Allow more custom parameters in schemadisplay --- python/sdssdb/utils/schemadisplay.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index aa103f49..b11de006 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -96,7 +96,7 @@ def format_field_str(field): html = (f'<' f'') + f'{table_name}
({model.__name__})') added_col_name = [] fields_html = [] @@ -131,7 +131,9 @@ def format_field_str(field): added_col_name.append(column_name) - html += ''.join(fields_html) + if len(fields_html) > 0: + html += '' + html += ''.join(fields_html) # Add indexes and unique constraints if show_indices: @@ -183,7 +185,7 @@ def format_field_str(field): def create_schema_graph(models=None, base=None, schema=None, show_columns=True, show_pks=True, show_indices=True, show_datatypes=True, skip_tables=[], font='Bitstream-Vera Sans', - concentrate=True, relation_options={}, rankdir='TB'): + graph_options={}, relation_options={}): """Creates a graph visualisation from a series of Peewee models. Produces a `pydot `__ graph including the @@ -212,6 +214,8 @@ def create_schema_graph(models=None, base=None, schema=None, show_columns=True, List of table names to skip. font : str The name of the font to use. + graph_options : dict + Options for creating the graph. Any valid Graphviz option. relation_options : dict Additional parameters to be passed to ``pydot.Edge`` when creating the relationships. @@ -247,12 +251,13 @@ def create_schema_graph(models=None, base=None, schema=None, show_columns=True, if schema: models = [model for model in models if model._meta.schema == schema] - graph = pydot.Dot(prog='dot', - mode='ipsep', - overlap='ipsep', - sep='0.01', - concentrate=str(concentrate), - rankdir=rankdir) + default_graph_options = dict(rankdir='TB', + sep='0.1', + mode='ipsep', + overlap='ipsep') + default_graph_options.update(graph_options) + + graph = pydot.Dot(prog='dot', **graph_options) for model in models: From 948922847d8e4444268918ee791587eba3c9c72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 1 Apr 2020 19:38:22 -0700 Subject: [PATCH 163/197] Legacy survey fk to gaia --- python/sdssdb/peewee/sdss5db/catalogdb.py | 34 +++++++++++------------ schema/sdss5db/catalogdb/foreignKeys.sql | 8 ++++++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 19961baf..eb3779b4 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -205,17 +205,19 @@ class Meta: class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): - source_id = ForeignKeyField(Gaia_DR2, - field='source_id', - column_name='source_id', - backref='+', - lazy_load=False) + source_id = BigIntegerField(primary_key=True) - tmass_pts_key = ForeignKeyField(TwoMassPSC, - field='pts_key', - column_name='tmass_pts_key', - backref='+', - lazy_load=False) + gaia = ForeignKeyField(Gaia_DR2, + field='source_id', + column_name='source_id', + backref='+', + lazy_load=False) + + twomass = ForeignKeyField(TwoMassPSC, + field='pts_key', + column_name='tmass_pts_key', + backref='+', + lazy_load=False) class Meta: table_name = 'gaiadr2_tmass_best_neighbour' @@ -248,14 +250,10 @@ class Legacy_Survey_DR8(CatalogdbModel): ref_cat = TextField() ref_id = BigIntegerField() - @property - def gaia(self): - """Returns the Gaia DR2 object or `None` if no match.""" - - if self.ref_cat != 'G2': - return None - - return Gaia_DR2.get(source_id=self.ref_id) + gaia = ForeignKeyField(Gaia_DR2, + column_name='gaia_sourceid', + object_id_name='gaia_sourceid', + backref='legacy_survey') class Meta: table_name = 'legacy_survey_dr8' diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 33ab7d46..2ae743b8 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -261,3 +261,11 @@ ALTER TABLE catalogdb.gaia_dr2_wd ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- legacy_survey_dr8 + +ALTER TABLE catalogdb.legacy_survey_dr8 + ADD CONSTRAINT gaia_sourceid_fk + FOREIGN KEY (gaia_sourceid) + REFERENCES catalogdb.gaia_dr2_source (source_id); From b1a2c601dca49aa0048d963b4cd65e0211ad3ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 2 Apr 2020 17:45:29 -0600 Subject: [PATCH 164/197] Add UVOTSSC1 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 6 + .../sdss5db/catalogdb/uvotssc1/uvotssc1.sql | 382 ++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index eb3779b4..a4f6d118 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -689,5 +689,11 @@ class Meta: print_fields = ['gaia_source_id'] +class UVOT_SSC_1(CatalogdbModel): + + class Meta: + table_name = 'uvotssc1' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql new file mode 100644 index 00000000..3f104fe2 --- /dev/null +++ b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql @@ -0,0 +1,382 @@ +/* + +II/339 Swift/UVOT Serendipitous Source Catalog (Yershov, 2015) +================================================================================ +Serendipitous UV source catalogues for 10 years of XMM and 5 years of Swift + Yershov V.N. + + =2014Ap&SS.354...97Y +The Swift UVOT Serendipitous Source Catalogue - UVOTSSC (2005-2010), Version 1 + Page M., Yershov V., Breeveld A., Kuin N.P.M., Mignani R.P., + Smith P.J., Rawlings J.I., Oates S.R., Siegel M., Roming P.W.A. + + =2015yCat.2339....0Y + =2014styd.confE..37P (2015arXiv150306597P) +================================================================================ +ADC_Keywords: Photometry, ultraviolet ; Photometry, UBV ; X-ray sources +Mission_Name: Swift + +Description: + The first version of the Swift UVOT serendipitous source catalogue + (UVOTSSC) provides positions and magnitudes, as well as errors and + upper limits of confirmed sources for observations taken from start of + operations in 2005 until October 1st of 2010. + + The first version of the Swift UVOT Serendipitous Source Catalogue + (UVOTSSC) has been produced by processing the image data obtained from + the Swift Ultraviolet and Optical Telescope (UVOT) from the beginning + of the mission (2005) until 1st of October of 2010. The data + processing was performed at the Mullard Space Science Laboratory + (MSSL, University College London, U.K.) using Swift FTOOLS from NASA's + High Energy Astrophysics Software (HEASoft-6.11), with some + customising of the UVOT packages in order to get more complete source + detection and properly apply quality flags to those sources that were + detected within the UVOT image artefacts. The total number of + observations with 17'x17' images used for version 1 of the catalogue + is 23,059, giving 6,200,016 sources in total, of which 2,027,265 have + multiple entries in the source table because they have been detected + in more than one observation. Some sources were only observed in one + filter. The total number of entries in the source table is 13,860,568. + The S/N ratio for all sources exceeds 5 for at least one UVOT filter, + the rest of the filters having a S/N greater than 3. + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +uvotssc1.dat 618 13860568 The Catalogue (6200016 individual sources) +summary.dat 268 23059 Summary information and per image upper limits +uvotssc1_1.fit 2880 1613850 The catalogue (FITS version) +-------------------------------------------------------------------------------- + +See also: + http://www.ucl.ac.uk/mssl/astro/space_missions/swift/uvotssc : Swift/UVOT + Serendipitous Source Catalog home page + J/ApJ/725/1215 : Faint UV standards from Swift, GALEX and SDSS (Siegel+, 2010) + J/AJ/137/4517 : UVOT light curves of supernovae (Brown+, 2009) + J/AJ/141/205 : UVOT imaging of M81 and Holmberg IX (Hoversten+, 2011) + J/MNRAS/424/1636 : Swift/UVOT sources in NGC4321 (M100) (Ferreras+, 2012) + J/other/ATel/5200 : VizieR Online Data Catalog: Swift Galactic Plane Survey: + sourcelist v3 (Reynolds+, 2013) + +Byte-by-byte Description of file: uvotssc1.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 8 A8 --- --- [UVOTSSC1] + 10- 26 A17 --- Name UVOTSSC1 name (JHHMMSS.s+DDMMSSa) (IAUNAME) + 28- 32 I5 --- Oseq [1/23059] Reference number in the observation + table (N_SUMMARY) + 34- 44 I011 --- ObsID Unique Swift observation ID (OBSID) + 46 I1 --- Nf [1/6] Number of filters included in this + observation (NFILT) + 48- 54 I7 --- SrcID [1/6200016] Unique source number (SRCNUM) + 56- 65 F10.6 deg RAdeg Right ascension (J2000) (RA) (1) + 67- 76 F10.6 deg DEdeg Declination (J2000) (DEC) (1) + 78- 84 F7.3 arcsec e_RAdeg [0.001/15] Right ascension error (RA_ERR) (1) + 86- 92 F7.3 arcsec e_DEdeg [0.001/21] Declination error (DEC_ERR) (1) + 94- 99 F6.3 arcsec rUVW2 [0/30]? Distance to closest UVW2 source + (UVW2_SRCDIST) + 101-106 F6.3 arcsec rUVM2 [0/30]? Distance to closest UVM2 source + (UVM2_SRCDIST) + 108-113 F6.3 arcsec rUVW1 [0/30]? Distance to closest UVW1 source + (UVW1_SRCDIST) + 115-120 F6.3 arcsec rU [0/30]? Distance to closest U source + (U_SRCDIST) + 122-127 F6.3 arcsec rB [0/30]? Distance to closest B source + (B_SRCDIST) + 129-134 F6.3 arcsec rV [0/30]? Distance to closest V source + (V_SRCDIST) + 136-138 I3 --- Nd [1/208] Number of individual observations + (N_OBSID) (2) + 140-145 F6.1 --- sUVW2 [3/5813]? Significance (S/N) in UVW2 + (UVW2_SIGNIF) + 147-152 F6.1 --- sUVM2 [3/5000]? Significance (S/N) in UVM2 + (UVM2_SIGNIF) + 154-159 F6.1 --- sUVW1 [3/5000]? Significance (S/N) in UVW1 + (UVW1_SIGNIF) + 161-166 F6.1 --- sU [3/5000]? Significance (S/N) in U (U_SIGNIF) + 168-173 F6.1 --- sB [3/5000]? Significance (S/N) in B (B_SIGNIF) + 175-180 F6.1 --- sV [3/5000]? Significance (S/N) in V (V_SIGNIF) + 182-188 F7.4 mag UVW2 ? UVOT/UVW2 Vega magnitude (UVW2_VEGAMAG) (3) + 190-196 F7.4 mag UVM2 ? UVOT/UVM2 Vega magnitude (UVM2_VEGAMAG) (3) + 198-204 F7.4 mag UVW1 ? UVOT/UVW1 Vega magnitude (UVW1_VEGAMAG) (3) + 206-212 F7.4 mag Umag ? UVOT/U Vega magnitude (U_VEGAMAG) (3) + 214-220 F7.4 mag Bmag ? UVOT/N Vega magnitude (B_VEGAMAG) (3) + 222-228 F7.4 mag Vmag ? UVOT/V Vega magnitude (V_VEGAMAG) (3) + 230-236 F7.4 mag UVW2-AB ? UVOT/UVW2 AB magnitude (UVW2_ABMAG) (3) + 238-244 F7.4 mag UVM2-AB ? UVOT/UVM2 AB magnitude (UVM2_ABMAG) (3) + 246-252 F7.4 mag UVW1-AB ? UVOT/UVW1 AB magnitude (UVW1_ABMAG) (3) + 254-260 F7.4 mag U-AB ? UVOT/U AB magnitude (U_ABMAG) (3) + 262-268 F7.4 mag B-AB ? UVOT/B AB magnitude (B_ABMAG) (3) + 270-276 F7.4 mag V-AB ? UVOT/V AB magnitude (V_ABMAG) (3) + 278-283 F6.4 mag e_UVW2 ? Error on UVW2 magnitude (UVW2_MAG_ERR) + 285-290 F6.4 mag e_UVM2 ? Error on UVM2 magnitude (UVM2_MAG_ERR) + 292-297 F6.4 mag e_UVW1 ? Error on UVW2 magnitude (UVW1_MAG_ERR) + 299-304 F6.4 mag e_Umag ? Error on U magnitude (U_MAG_ERR) + 306-311 F6.4 mag e_Bmag ? Error on B magnitude (B_MAG_ERR) + 313-318 F6.4 mag e_Vmag ? Error on V magnitude (V_MAG_ERR) + 320-331 E12.6 cW/m2/nm F.UVW2 ? UVOT/UVW2 Flux (UVW2_FLUX) (3) + 333-344 E12.6 cW/m2/nm F.UVM2 ? UVOT/UVW2 Flux (UVM2_FLUX) (3) + 346-357 E12.6 cW/m2/nm F.UVW1 ? UVOT/UVW2 Flux (UVW1_FLUX) (3) + 359-370 E12.6 cW/m2/nm F.U ? UVOT/UVW2 Flux (U_FLUX) (3) + 372-383 E12.6 cW/m2/nm F.B ? UVOT/UVW2 Flux (B_FLUX) (3) + 385-396 E12.6 cW/m2/nm F.V ? UVOT/UVW2 Flux (V_FLUX) (3) + 398-407 E10.4 cW/m2/nm e_F.UVW2 ? Error on F.UVW2 (UVW2_FLUX_ERR) + 409-418 E10.4 cW/m2/nm e_F.UVM2 ? Error on F.UVM2 (UVM2_FLUX_ERR) + 420-429 E10.4 cW/m2/nm e_F.UVW1 ? Error on F.UVW1 (UVW1_FLUX_ERR) + 431-440 E10.4 cW/m2/nm e_F.U ? Error on F.U (U_FLUX_ERR) + 442-451 E10.4 cW/m2/nm e_F.B ? Error on F.B (B_FLUX_ERR) + 453-462 E10.4 cW/m2/nm e_F.V ? Error on F.V (V_FLUX_ERR) + 464-469 F6.3 arcsec aUVW2 ? Major axis in UVW2 (UVW2_MAJOR) + 471-476 F6.3 arcsec aUVM2 ? Major axis in UVM2 (UVM2_MAJOR) + 478-483 F6.3 arcsec aUVW1 ? Major axis in UVW1 (UVW1_MAJOR) + 485-490 F6.3 arcsec aU ? Major axis in U (U_MAJOR) + 492-497 F6.3 arcsec aB ? Major axis in B (B_MAJOR) + 499-504 F6.3 arcsec aV ? Major axis in V (V_MAJOR) + 506-511 F6.3 arcsec bUVW2 ? Minor axis in UVW2 (UVW2_MINOR) + 513-518 F6.3 arcsec bUVM2 ? Minor axis in UVM2 (UVM2_MINOR) + 520-525 F6.3 arcsec bUVW1 ? Minor axis in UVW1 (UVW1_MINOR) + 527-532 F6.3 arcsec bU ? Minor axis in U (U_MINOR) + 534-539 F6.3 arcsec bB ? Minor axis in B (B_MINOR) + 541-546 F6.3 arcsec bV ? Minor axis in V (V_MINOR) + 548-552 F5.2 deg paUVW2 [0/90]? Position angle of major axis in UVW2 + (UVW2_POSANG) + 554-558 F5.2 deg paUVM2 [0/90]? Position angle of major axis in UVM2 + (UVM2_POSANG) + 560-564 F5.2 deg paUVW1 [0/90]? Position angle of major axis in UVW1 + (UVW1_POSANG) + 566-570 F5.2 deg paU [0/90]? Position angle of major axis in U + (U_POSANG) + 572-576 F5.2 deg paB [0/90]? Position angle of major axis in B + (B_POSANG) + 578-582 F5.2 deg paV [0/90]? Position angle of major axis in V + (V_POSANG) + 584 I1 --- xUVW2 [0/1] Extended flag in UVW2 (UVW2_EXTENDED) + 586 I1 --- xUVM2 [0/1] Extended flag in UVW2 (UVM2_EXTENDED) + 588 I1 --- xUVW1 [0/1] Extended flag in UVW2 (UVW1_EXTENDED) + 590 I1 --- xU [0/1] Extended flag in UVW2 (U_EXTENDED) + 592 I1 --- xB [0/1] Extended flag in UVW2 (B_EXTENDED) + 594 I1 --- xV [0/1] Extended flag in UVW2 (V_EXTENDED) + 596-598 I3 --- fUVW2 [0/511]?=- Quality flags in UVW2 + (UVW2_QUALITY_FLAG) (4) + 600-602 I3 --- fUVM2 [0/511]?=- Quality flags in UVM2 + (UVM2_QUALITY_FLAG) (4) + 604-606 I3 --- fUVW1 [0/511]?=- Quality flags in UVW1 + (UVW1_QUALITY_FLAG) (4) + 608-610 I3 --- fU [0/511]?=- Quality flags in U (U_QUALITY_FLAG) + 612-614 I3 --- fB [0/511]?=- Quality flags in B (B_QUALITY_FLAG) + 616-618 I3 --- fV [0/511]?=- Quality flags in V (V_QUALITY_FLAG) +-------------------------------------------------------------------------------- +Note (1): standard error of position in RA and Dec, i.e. (1/n)sqrt({Sigma}err) + +Note (2): this number should correspond to the number of entries for that + particular source in the table. + +Note (3): Fluxes are expressed in erg/cm^2^/s/{AA}=cW/m^2^/nm. + The fluxes were derived using the count rate to flux conversion + factors from Poole et al. (2008MNRAS.383..627P) which have been based + on GRB spectra. This differs slightly from the count rate to flux + conversion factor based on Pickles star spectra expressed as a flux + ratio (rightmost column); note that the flux conversion factors are + valid for UVOT B-V > -0.36 + ------------------------------------------------ + Filter Central Wavelength FWHM F.filter + (nm) (nm) (Pickles/grb) + ------------------------------------------------ + V 546.8 76.9 0.998 + B 439.2 97.5 0.893 + U 346.5 78.5 0.940 + UVW1 260.0 69.3 0.963 + UVM2 224.6 49.8 0.884 + UVW2 192.8 65.7 0.965 + ------------------------------------------------ + The UVOT UV filters are physically similar to those in XMM-OM but have + greater throughput. The UVOT U,B,V are not the Johnson U,B,V passbands. + +Note (4): Quality flags are; + 1 = Cosmetic defects (BAD PIXELS) within the source region + 2 = Source on a READOUT STREAK + 4 = Source on a "SMOKE RING" + 8 = Source on a DIFFRACTION SPIKE + 16 = Source affected by MOD-8 noise pattern + 32 = Source within a "HALO RING" + 64 = Source near to a BRIGHT source + 128 = MULTIPLE EXPOSURE values within photometry aperture + 256 = Source within an EXTENDED FEATURE + Multiple flags are summed to give the final quality flag value. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: summary.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 5 I5 --- Oseq [1/23059] Reference number (N_SUMMARY) + 7- 17 I011 --- ObsID Unique Swift observation ID (OBSID) (5) + 19- 36 A18 --- Target Swift original target identifier (TARGET_ID) + 38- 57 A20 --- Filters List of filters used this OBSID (FILTERS) + 59- 77 A19 "datime" StartDate Earliest observation date and time, UT format + (DATE_MIN) + 79- 88 F10.6 deg ptRAdeg Pointing RA (ICRS) of image centre (RA_PNT) + 90- 99 F10.6 deg ptDEdeg Pointing Dec (ICRS) of image centre (DEC_PNT) + 101-105 F5.1 deg ptPA [0/360] Position angle image centre (PA_PNT) + 107-113 F7.1 s exp.UVW2 ?Total exposure time in UVW2 (EXP_UVW2) + 115-121 F7.1 s exp.UVM2 ?Total exposure time in UVM2 (EXP_UVM2) + 123-129 F7.1 s exp.UVW1 ?Total exposure time in UVW1(EXP_UVW1) + 131-137 F7.1 s exp.U ?Total exposure time in U (EXP_U) + 139-145 F7.1 s exp.B ?Total exposure time in B(EXP_B) + 147-153 F7.1 s exp.V ?Total exposure time in V (EXP_V) + 155-158 I4 --- n.UVW2 ?Number of UVW2 sources (NSOURCES_UVW2) + 160-163 I4 --- n.UVM2 ?Number of UVM2 sources (NSOURCES_UVM2) + 165-168 I4 --- n.UVW1 ?Number of UVW1 sources (NSOURCES_UVW1) + 170-173 I4 --- n.U ?Number of U sources (NSOURCES_U) + 175-178 I4 --- n.B ?Number of B sources (NSOURCES_B) + 180-184 I5 --- n.V ?Number of V sources (NSOURCES_V) + 186-191 F6.3 mag ul.UVW2 ?Limiting mag UVW2 (VEGAMAG_LIM_UVW2) (6) + 193-198 F6.3 mag ul.UVM2 ?Limiting mag UVM2 (VEGAMAG_LIM_UVM2) (6) + 200-205 F6.3 mag ul.UVW1 ?Limiting mag UVW1 (VEGAMAG_LIM_UVW1) (6) + 207-212 F6.3 mag ul.U ?Limiting mag U (VEGAMAG_LIM_U) (6) + 214-219 F6.3 mag ul.B ?Limiting mag B (VEGAMAG_LIM_B) (6) + 221-226 F6.3 mag ul.V ?Limiting mag V (VEGAMAG_LIM_V) (6) + 228-233 F6.3 mag ul.UVW2-AB ?Limiting AB-mag UVW2 (ABMAG_LIM_UVW2) (6) + 235-240 F6.3 mag ul.UVM2-AB ?Limiting AB-mag UVM2 (ABMAG_LIM_UVM2) (6) + 242-247 F6.3 mag ul.UVW1-AB ?Limiting AB-mag UVW1 (ABMAG_LIM_UVW1) (6) + 249-254 F6.3 mag ul.U-AB ?Limiting AB-mag U (ABMAG_LIM_U) (6) + 256-261 F6.3 mag ul.B-AB ?Limiting AB-mag B (ABMAG_LIM_B) (6) + 263-268 F6.3 mag ul.V-AB ?Limiting AB-mag V (ABMAG_LIM_V) (6) +-------------------------------------------------------------------------------- +Note (5): One Swift OBSID can consist of one or more images, which for this + catalogue have been summed, yielding the quoted total exposure time. + The original uvot images can be found in the on-line archives at + MAST, the Swift archive at swift.ac.uk and swift.gsfc.nasa.gov using + the OBSID as search key. For higher temporal resolution the original + images need to be used because the catalogue data sum over all + images within an OBSID. + +Note (6): The upper limits per filter of the summed image constructed + for each OBSID because the sensitivity hardly varies over the + detector. Usually the images within one OBSID share the same + pointing, however, whereas the quoted upper limits apply always for + sources near the pointing direction given, if the images had small + offsets in pointing they may not apply to sources near the edge of + the summed image, which is typically about 8 arc seconds from the + pointing. +-------------------------------------------------------------------------------- + +History and notes: + The initially released version of the catalogue (2015) was done + with the source identifier "SWIFTUVOT" for each source, and was + made available in that form. The decision was made to rename the + catalogue sources by including the catalogue version number. In + addition, in a few instances multiple SOURCE IDs shared the same + IAU NAME. They will be distinguished by having a letter a,b,c,.. + appended to their NAME. Sources brighter than 0.96 counts per frame + have not been included. Their coincidence loss is too large to + correct for. +================================================================================ +(End) Paul Kuin [MSSL/UCL], Francois Ochsenbein [CDS] 27-November-2015 + +This uvotssc1.csv file was generating by taking uvotssc1.dat and ReadMe from +http://cdsarc.u-strasbg.fr/ftp/cats/II/339/, reading the file with astropy as + +table = astropy.io.ascii.read('./uvotssc1.dat', readme='./ReadMe', + format='cds', fill_values=[('---', '0')]) +table.remove_column('---') +table.write('./uvotssc1.csv', format='ascii.csv') + +*/ + +CREATE TABLE catalogdb.uvotssc1 ( + name VARCHAR(17), + oseq BIGINT, + obsid BIGINT, + nf BIGINT, + srcid BIGINT, + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + e_radeg DOUBLE PRECISION, + e_dedeg DOUBLE PRECISION, + ruvw2 REAL, + ruvm2 REAL, + ruvw1 REAL, + ru REAL, + rb REAL, + rv REAL, + nd BIGINT, + suvw2 REAL, + suvm2 REAL, + suvw1 REAL, + su REAL, + sb REAL, + sv REAL, + uvw2 DOUBLE PRECISION, + uvm2 DOUBLE PRECISION, + uvw1 DOUBLE PRECISION, + umag DOUBLE PRECISION, + bmag DOUBLE PRECISION, + vmag DOUBLE PRECISION, + uvw2_ab DOUBLE PRECISION, + uvm2_ab DOUBLE PRECISION, + uvw1_ab DOUBLE PRECISION, + u_ab DOUBLE PRECISION, + b_ab DOUBLE PRECISION, + v_ab DOUBLE PRECISION, + e_uvw2 DOUBLE PRECISION, + e_uvm2 DOUBLE PRECISION, + e_uvw1 DOUBLE PRECISION, + e_umag DOUBLE PRECISION, + e_bmag DOUBLE PRECISION, + e_vmag DOUBLE PRECISION, + f_uvw2 DOUBLE PRECISION, + f_uvm2 DOUBLE PRECISION, + f_uvw1 DOUBLE PRECISION, + f_u DOUBLE PRECISION, + f_b DOUBLE PRECISION, + f_v DOUBLE PRECISION, + e_f_uvw2 DOUBLE PRECISION, + e_f_uvm2 DOUBLE PRECISION, + e_f_uvw1 DOUBLE PRECISION, + e_f_u DOUBLE PRECISION, + e_f_b DOUBLE PRECISION, + e_f_v DOUBLE PRECISION, + auvw2 DOUBLE PRECISION, + auvm2 DOUBLE PRECISION, + auvw1 DOUBLE PRECISION, + au DOUBLE PRECISION, + ab DOUBLE PRECISION, + av DOUBLE PRECISION, + buvw2 DOUBLE PRECISION, + buvm2 DOUBLE PRECISION, + buvw1 DOUBLE PRECISION, + bu DOUBLE PRECISION, + bb DOUBLE PRECISION, + bv DOUBLE PRECISION, + pauvw2 REAL, + pauvm2 REAL, + pauvw1 REAL, + pau REAL, + pab REAL, + pav REAL, + xuvw2 INTEGER, + xuvm2 INTEGER, + xuvw1 INTEGER, + xu INTEGER, + xb INTEGER, + xv INTEGER, + fuvw2 INTEGER, + fuvm2 INTEGER, + fuvw1 INTEGER, + fu INTEGER, + fb INTEGER, + fv INTEGER +) WITHOUT OIDS; + +\COPY catalogdb.uvotssc1 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/uvotssc1.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.uvotssc1 ADD id BIGSERIAL PRIMARY KEY; + +CREATE INDEX ON catalogdb.uvotssc1 (q3c_ang2ipix(radeg, dedeg)); +CLUSTER uvotssc1_q3c_ang2ipix_idx ON catalogdb.uvotssc1; +VACUUM ANALYZE catalogdb.uvotssc1; From fa1e86b5b098a1c134fa6d95963a5eb15bd91c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 2 Apr 2020 17:55:34 -0600 Subject: [PATCH 165/197] Add Cataclysmic Variables --- python/sdssdb/peewee/sdss5db/catalogdb.py | 6 + .../cataclysmic_variables.py | 30 +++++ .../cataclysmic_variables.sql | 109 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100755 schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py create mode 100644 schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index a4f6d118..439c0f63 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -695,5 +695,11 @@ class Meta: table_name = 'uvotssc1' +class CataclysmicVariables(CatalogdbModel): + + class Meta: + table_name = 'cataclysmic_variables' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py new file mode 100755 index 00000000..48413210 --- /dev/null +++ b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: cataclysmic_variables.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/cataclysmicvariables.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'cataclysmic_variables', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql new file mode 100644 index 00000000..c5e2d165 --- /dev/null +++ b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql @@ -0,0 +1,109 @@ +/* + +AAVSO Cataclysmic Variables + +*/ + +CREATE TABLE catalogdb.cataclysmic_variables ( + ref_id BIGINT, + solution_id BIGINT, + designation TEXT, + source_id BIGINT, + random_index INTEGER, + ref_epoch REAL, + ra DOUBLE PRECISION, + ra_error DOUBLE PRECISION, + dec DOUBLE PRECISION, + dec_error DOUBLE PRECISION, + parallax DOUBLE PRECISION, + parallax_error DOUBLE PRECISION, + parallax_over_error DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_error DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_error DOUBLE PRECISION, + ra_dec_corr DOUBLE PRECISION, + ra_parallax_corr DOUBLE PRECISION, + ra_pmra_corr DOUBLE PRECISION, + ra_pmdec_corr DOUBLE PRECISION, + dec_parallax_corr DOUBLE PRECISION, + dec_pmra_corr DOUBLE PRECISION, + dec_pmdec_corr DOUBLE PRECISION, + parallax_pmra_corr DOUBLE PRECISION, + parallax_pmdec_corr DOUBLE PRECISION, + pmra_pmdec_corr DOUBLE PRECISION, + astrometric_n_obs_al SMALLINT, + astrometric_n_obs_ac SMALLINT, + astrometric_n_good_obs_al SMALLINT, + astrometric_n_bad_obs_al SMALLINT, + astrometric_gof_al DOUBLE PRECISION, + astrometric_chi2_al DOUBLE PRECISION, + astrometric_excess_noise DOUBLE PRECISION, + astrometric_excess_noise_sig DOUBLE PRECISION, + astrometric_params_solved SMALLINT, + astrometric_primary_flag BOOLEAN, + astrometric_weight_al DOUBLE PRECISION, + astrometric_pseudo_colour DOUBLE PRECISION, + astrometric_pseudo_colour_error DOUBLE PRECISION, + mean_varpi_factor_al DOUBLE PRECISION, + astrometric_matched_observations SMALLINT, + visibility_periods_used SMALLINT, + astrometric_sigma5d_max DOUBLE PRECISION, + frame_rotator_object_type SMALLINT, + matched_observations SMALLINT, + duplicated_source BOOLEAN, + phot_g_n_obs SMALLINT, + phot_g_mean_flux DOUBLE PRECISION, + phot_g_mean_flux_error DOUBLE PRECISION, + phot_g_mean_flux_over_error DOUBLE PRECISION, + phot_g_mean_mag DOUBLE PRECISION, + phot_bp_n_obs SMALLINT, + phot_bp_mean_flux DOUBLE PRECISION, + phot_bp_mean_flux_error DOUBLE PRECISION, + phot_bp_mean_flux_over_error DOUBLE PRECISION, + phot_bp_mean_mag DOUBLE PRECISION, + phot_rp_n_obs SMALLINT, + phot_rp_mean_flux DOUBLE PRECISION, + phot_rp_mean_flux_error DOUBLE PRECISION, + phot_rp_mean_flux_over_error DOUBLE PRECISION, + phot_rp_mean_mag DOUBLE PRECISION, + phot_bp_rp_excess_factor DOUBLE PRECISION, + phot_proc_mode SMALLINT, + bp_rp DOUBLE PRECISION, + bp_g DOUBLE PRECISION, + g_rp DOUBLE PRECISION, + radial_velocity DOUBLE PRECISION, + radial_velocity_error DOUBLE PRECISION, + rv_nb_transits SMALLINT, + rv_template_teff REAL, + rv_template_logg REAL, + rv_template_fe_h REAL, + phot_variable_flag TEXT, + l DOUBLE PRECISION, + b DOUBLE PRECISION, + ecl_lon DOUBLE PRECISION, + ecl_lat DOUBLE PRECISION, + priam_flags INTEGER, + teff_val DOUBLE PRECISION, + teff_percentile_lower DOUBLE PRECISION, + teff_percentile_upper DOUBLE PRECISION, + a_g_val REAL, + a_g_percentile_lower REAL, + a_g_percentile_upper REAL, + e_bp_min_rp_val REAL, + e_bp_min_rp_percentile_lower REAL, + e_bp_min_rp_percentile_upper REAL, + flame_flags INTEGER, + radius_val DOUBLE PRECISION, + radius_percentile_lower DOUBLE PRECISION, + radius_percentile_upper DOUBLE PRECISION, + lum_val DOUBLE PRECISION, + lum_percentile_lower DOUBLE PRECISION, + lum_percentile_upper DOUBLE PRECISION +); + +ALTER TABLE catalogdb.cataclysmic_variables ADD PRIMARY KEY (ref_id); + +CREATE INDEX ON catalogdb.cataclysmic_variables (q3c_ang2ipix(ra, dec)); +CLUSTER cataclysmic_variables_q3c_ang2ipix_idx ON catalogdb.cataclysmic_variables; +ANALYZE catalogdb.cataclysmic_variables; From d03f69c13ae879654995bf6e6e352cf4af2a4f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 2 Apr 2020 18:08:06 -0600 Subject: [PATCH 166/197] Add yso_clustering --- python/sdssdb/peewee/sdss5db/catalogdb.py | 11 +++++++ schema/sdss5db/catalogdb/foreignKeys.sql | 8 +++++ .../yso_clustering/yso_clustering.py | 30 +++++++++++++++++++ .../yso_clustering/yso_clustering.sql | 30 +++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100755 schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py create mode 100644 schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 439c0f63..556bbdec 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -701,5 +701,16 @@ class Meta: table_name = 'cataclysmic_variables' +class YSO_Clustering(CatalogdbModel): + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='+') + + class Meta: + table_name = 'yso_clustering' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 2ae743b8..be7ce9b3 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -269,3 +269,11 @@ ALTER TABLE catalogdb.legacy_survey_dr8 ADD CONSTRAINT gaia_sourceid_fk FOREIGN KEY (gaia_sourceid) REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- yso_clustering + +ALTER TABLE catalogdb.yso_clustering + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); diff --git a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py new file mode 100755 index 00000000..8c716e69 --- /dev/null +++ b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: yso_clustering.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/Kounkel_Clustering.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'yso_clustering', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql new file mode 100644 index 00000000..3257a1d7 --- /dev/null +++ b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql @@ -0,0 +1,30 @@ +/* + +AAVSO Cataclysmic Variables + +*/ + +CREATE TABLE catalogdb.yso_clustering ( + source_id BIGINT PRIMARY KEY, + twomass TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + parallax REAL, + id INTEGER, + g DOUBLE PRECISION, + bp DOUBLE PRECISION, + rp DOUBLE PRECISION, + j REAL, + h REAL, + k REAL, + age DOUBLE PRECISION, + eage DOUBLE PRECISION, + av DOUBLE PRECISION, + eav DOUBLE PRECISION, + dist DOUBLE PRECISION, + edist DOUBLE PRECISION +); + +CREATE INDEX ON catalogdb.yso_clustering (q3c_ang2ipix(ra, dec)); +CLUSTER yso_clustering_q3c_ang2ipix_idx ON catalogdb.yso_clustering; +ANALYZE catalogdb.yso_clustering; From 2cee5707c488f5d1d88e3cfd43b30d8f9c1cb6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 2 Apr 2020 18:08:20 -0600 Subject: [PATCH 167/197] Add tycho2 unique index on name --- schema/sdss5db/catalogdb/tycho2/tycho2_index.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql index 74c3f3cb..31a1b2bb 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql @@ -1,4 +1,9 @@ +CREATE UNIQUE INDEX CONCURRENTLY catalogdb_tycho2_name_unique_idx ON catalogdb.tycho2 (name); +ALTER TABLE catalogdb.tycho2 + ADD CONSTRAINT catalogdb_tycho2_name_unique + UNIQUE USING INDEX catalogdb_tycho2_name_unique_idx; + CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; ANALYZE catalogdb.tycho2; From e904ff71f17ffadc4ab260f5d94032aee0d855aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 2 Apr 2020 18:41:21 -0600 Subject: [PATCH 168/197] Add source_id pk in ORM explicitely for YSO_Clustering --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 556bbdec..11319be2 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -703,6 +703,8 @@ class Meta: class YSO_Clustering(CatalogdbModel): + source_id = BigIntegerField(primary_key=True) + gaia = ForeignKeyField(Gaia_DR2, column_name='source_id', object_id_name='source_id', From e86ff0ce465367002879faf43967f4ff745b0d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 10:29:02 -0600 Subject: [PATCH 169/197] Update eRosita version --- schema/sdss5db/catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita.sql | 0 .../catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita_index.sql | 0 .../catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita_load.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename schema/sdss5db/catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita.sql (100%) rename schema/sdss5db/catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita_index.sql (100%) rename schema/sdss5db/catalogdb/eRosita/{v0.1.1 => v0.1.2}/eRosita_load.py (99%) diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita.sql similarity index 100% rename from schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita.sql rename to schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita.sql diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql similarity index 100% rename from schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_index.sql rename to schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py similarity index 99% rename from schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py rename to schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py index fd97ca3f..fd94bf7a 100755 --- a/schema/sdss5db/catalogdb/eRosita/v0.1.1/eRosita_load.py +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py @@ -20,7 +20,7 @@ def main(): path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' 'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/') - for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.1.fits', + for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.2.fits', 'bhm_spiders_clusters_superset'), ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits', 'bhm_spiders_agn_superset')): From 73e003ab7dee91965c1478d4ccf4cdbe5ebd396e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 10:29:46 -0600 Subject: [PATCH 170/197] Changes to loading Tycho2 --- schema/sdss5db/catalogdb/foreignKeys.sql | 2 +- schema/sdss5db/catalogdb/tycho2/tycho2.sql | 90 ++++++++++++------- .../sdss5db/catalogdb/tycho2/tycho2_index.sql | 8 ++ .../sdss5db/catalogdb/tycho2/tycho2_load.py | 4 +- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index be7ce9b3..1447a9bc 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -157,7 +157,7 @@ ALTER TABLE catalogdb.tic_v8 ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT tyc_fk FOREIGN KEY (tyc) - REFERENCES catalogdb.tycho2 (name) + REFERENCES catalogdb.tycho2 (designation) ON UPDATE CASCADE ON DELETE CASCADE; ALTER TABLE catalogdb.tic_v8 diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql index 6bee10d5..fd52de5b 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -328,42 +328,68 @@ References: ================================================================================ (End) C. Fabricius [Niels Bohr Institute] 19-Jan-2000 +This script loads the file tycho2_all.csv that was generated by loading +the tyc2.dat and suplement files using +astropy.io.ascii.read(file, format='csv', readme='./ReadMe') and then concatenating +them with astropy.table.vstack. The resulting table was then saved as CSV. + */ CREATE TABLE catalogdb.tycho2 ( - id_tycho BIGINT PRIMARY KEY, - name TEXT PRIMARY KEY, - pflag CHARACTER(1), - ramdeg DOUBLE PRECISION, - demdeg DOUBLE PRECISION, - pmra DOUBLE PRECISION, - pmde DOUBLE PRECISION, - e_ramdeg DOUBLE PRECISION, - e_demdeg DOUBLE PRECISION, - e_pmra DOUBLE PRECISION, - e_pmde DOUBLE PRECISION, - epram DOUBLE PRECISION, - epdem DOUBLE PRECISION, - num INTEGER, - q_ramdeg DOUBLE PRECISION, - q_demdeg DOUBLE PRECISION, - q_pmra DOUBLE PRECISION, - q_pmde DOUBLE PRECISION, - btmag DOUBLE PRECISION, - e_btmag DOUBLE PRECISION, - vtmag DOUBLE PRECISION, - e_vtmag DOUBLE PRECISION, - prox INTEGER, - tyc CHARACTER(1), - hip INTEGER, - ccdm TEXT, - radeg DOUBLE PRECISION, - dedeg DOUBLE PRECISION, + tyc1 INTEGER, + tyc2 INTEGER, + tyc3 INTEGER, + pflag VARCHAR(1), + ramdeg DOUBLE PRECISION, + demdeg DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmde DOUBLE PRECISION, + e_ramdeg INTEGER, + e_demdeg INTEGER, + e_pmra DOUBLE PRECISION, + e_pmde DOUBLE PRECISION, + epram DOUBLE PRECISION, + epdem DOUBLE PRECISION, + num INTEGER, + q_ramdeg DOUBLE PRECISION, + q_demdeg DOUBLE PRECISION, + q_pmra DOUBLE PRECISION, + q_pmde DOUBLE PRECISION, + btmag REAL, + e_btmag REAL, + vtmag REAL, + e_vtmag REAL, + prox INTEGER, + tyc VARCHAR(1), + hip BIGINT, + ccdm VARCHAR(3), + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, epra_1990 DOUBLE PRECISION, epde_1990 DOUBLE PRECISION, - e_radeg DOUBLE PRECISION, - e_dedeg DOUBLE PRECISION, - posflg CHARACTER(1), - corr DOUBLE PRECISION + e_radeg DOUBLE PRECISION, + e_dedeg DOUBLE PRECISION, + posflg VARCHAR(1), + corr REAL, + flag VARCHAR(1), + mflag VARCHAR(1) ) WITHOUT OIDS; + +\COPY catalogdb.tycho2 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/tycho2_all.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.tycho2 ADD COLUMN designation TEXT; +UPDATE catalogdb.tycho2 SET designation = tyc1 || '-' || LPAD(tyc2::TEXT, 5, '0') || '-' || tyc3; + +-- Need to remove duplicates in the supplement tables. This assumes that the supplement tables +-- are towards the end of the table, which may not be true, but seems to work. +ALTER TABLE catalogdb.tycho2 ADD COLUMN pk BIGSERIAL; +DELETE FROM catalogdb.tycho2 a USING catalogdb.tycho2 b + WHERE a.designation = b.designation AND a.pk > b.pk; +ALTER TABLE catalogdb.tycho2 DROP COLUMN pk; + +ALTER TABLE catalogdb.tycho2 ADD PRIMARY KEY (designation); + +CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); +CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; +VACUUM ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql index 31a1b2bb..01ba6745 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql @@ -4,6 +4,14 @@ ALTER TABLE catalogdb.tycho2 ADD CONSTRAINT catalogdb_tycho2_name_unique UNIQUE USING INDEX catalogdb_tycho2_name_unique_idx; + +-- Also create a designation column which is the same as name but with dashes +-- (for MIPSGAL and maybe others) +ALTER TABLE catalogdb.tycho2 ADD COLUMN designation TEXT; +UPDATE catalogdb.tycho2 SET designation = REPLACE(name, ' ', '-'); +ALTER TABLE catalogdb.tycho2 + ADD CONSTRAINT catalogdb_tycho2_designation_unique UNIQUE (designation); + CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py index 85b3997f..94a2d5fb 100755 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py +++ b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py @@ -18,8 +18,10 @@ def main(): assert database.connected path_ = '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/' + paths = list(glob.glob(path_ + 'tyc2.dat.*.gz')) + [path_ + 'suppl_1.dat.gz', + path_ + 'suppl_2.dat.gz'] - for file_ in sorted(glob.glob(path_ + 'tyc2.dat.*.gz')): + for file_ in sorted(paths): print(file_) From 0dff3789f761f34d711f5d3880c8f5d4aa4ed5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 10:30:06 -0600 Subject: [PATCH 171/197] Add MIPSGAL --- python/sdssdb/peewee/sdss5db/catalogdb.py | 12 +++ schema/sdss5db/catalogdb/foreignKeys.sql | 21 ++++++ schema/sdss5db/catalogdb/mipsgal/mipsgal.py | 30 ++++++++ schema/sdss5db/catalogdb/mipsgal/mipsgal.sql | 77 ++++++++++++++++++++ 4 files changed, 140 insertions(+) create mode 100755 schema/sdss5db/catalogdb/mipsgal/mipsgal.py create mode 100644 schema/sdss5db/catalogdb/mipsgal/mipsgal.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 11319be2..400b1e43 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -714,5 +714,17 @@ class Meta: table_name = 'yso_clustering' +class MIPSGAL(CatalogdbModel): + + glimpse = ForeignKeyField(GLIMPSE, backref='+') + twomass = ForeignKeyField(TwoMassPSC, + column_name='twomass_name', + object_id_name='twomass_name', + backref='+') + + class Meta: + table_name = 'mipsgal' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 1447a9bc..dc8cc2c7 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -277,3 +277,24 @@ ALTER TABLE catalogdb.yso_clustering ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- mipsgal + +-- Prepare the columns +UPDATE catalogdb.mipsgal SET twomass_name = TRIM(twomass_name); +UPDATE catalogdb.mipsgal SET twomass_name = NULL where twomass_name = ''; +UPDATE catalogdb.mipsgal SET glimpse = NULL where TRIM(glimpse) = ''; + +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal USING BTREE (twomass_name); +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal USING BTREE (glimpse); + +ALTER TABLE catalogdb.mipsgal + ADD CONSTRAINT twomass_name_fk + FOREIGN KEY (twomass_name) + REFERENCES catalogdb.twomass_psc (designation); + +ALTER TABLE catalogdb.mipsgal + ADD CONSTRAINT glimpse_fk + FOREIGN KEY (glimpse) + REFERENCES catalogdb.glimpse (designation); diff --git a/schema/sdss5db/catalogdb/mipsgal/mipsgal.py b/schema/sdss5db/catalogdb/mipsgal/mipsgal.py new file mode 100755 index 00000000..8363a8c9 --- /dev/null +++ b/schema/sdss5db/catalogdb/mipsgal/mipsgal.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: mipsgal.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'mipsgal', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql new file mode 100644 index 00000000..96d155eb --- /dev/null +++ b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql @@ -0,0 +1,77 @@ +/* + +MIPSGAL - https://irsa.ipac.caltech.edu/data/SPITZER/MIPSGAL/ + +*/ + +CREATE TABLE catalogdb.mipsgal ( + mipsgal VARCHAR(18) PRIMARY KEY, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + s24 DOUBLE PRECISION, + e_s24 DOUBLE PRECISION, + mag_24 DOUBLE PRECISION, + e_mag_24 DOUBLE PRECISION, + twomass_name VARCHAR(17), + sj DOUBLE PRECISION, + e_sj DOUBLE PRECISION, + sh DOUBLE PRECISION, + e_sh DOUBLE PRECISION, + sk DOUBLE PRECISION, + e_sk DOUBLE PRECISION, + glimpse VARCHAR(17), + s3_6 DOUBLE PRECISION, + e_s3_6 DOUBLE PRECISION, + s4_5 DOUBLE PRECISION, + e_s4_5 DOUBLE PRECISION, + s5_8 DOUBLE PRECISION, + e_s5_8 DOUBLE PRECISION, + s8_0 DOUBLE PRECISION, + wise VARCHAR(19), + s3_4 DOUBLE PRECISION, + e_s3_4 DOUBLE PRECISION, + s4_6 DOUBLE PRECISION, + e_s4_6 DOUBLE PRECISION, + s12 DOUBLE PRECISION, + e_s12 DOUBLE PRECISION, + s22 DOUBLE PRECISION, + e_s22 DOUBLE PRECISION, + jmag DOUBLE PRECISION, + e_jmag DOUBLE PRECISION, + hmag DOUBLE PRECISION, + e_hmag DOUBLE PRECISION, + kmag DOUBLE PRECISION, + e_kmag DOUBLE PRECISION, + mag_3_6 DOUBLE PRECISION, + e_mag_3_6 DOUBLE PRECISION, + mag_4_5 DOUBLE PRECISION, + e_mag_4_5 DOUBLE PRECISION, + mag_5_8 DOUBLE PRECISION, + e_mag_5_8 DOUBLE PRECISION, + mag_8_0 DOUBLE PRECISION, + e_mag_8_0 DOUBLE PRECISION, + w1mag DOUBLE PRECISION, + e_w1mag DOUBLE PRECISION, + w2mag DOUBLE PRECISION, + e_w2mag DOUBLE PRECISION, + w3mag DOUBLE PRECISION, + e_w3mag DOUBLE PRECISION, + w4mag DOUBLE PRECISION, + e_w4mag DOUBLE PRECISION, + dnn DOUBLE PRECISION, + ng INTEGER, + n2m INTEGER, + nw INTEGER, + fwhm DOUBLE PRECISION, + sky DOUBLE PRECISION, + lim1 DOUBLE PRECISION, + lim2 DOUBLE PRECISION, + f1 INTEGER, + f2 INTEGER +); + +CREATE INDEX ON catalogdb.mipsgal (q3c_ang2ipix(radeg, dedeg)); +CLUSTER mipsgal_q3c_ang2ipix_idx ON catalogdb.mipsgal; +ANALYZE catalogdb.mipsgal; From f6cfc2163a4172c046e06c9aaea74df311099e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 13:14:59 -0600 Subject: [PATCH 172/197] Add xmm_om_suss_4_1 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 18 ++- .../sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql | 132 ++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 400b1e43..ef904e0c 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -718,7 +718,6 @@ class MIPSGAL(CatalogdbModel): glimpse = ForeignKeyField(GLIMPSE, backref='+') twomass = ForeignKeyField(TwoMassPSC, - column_name='twomass_name', object_id_name='twomass_name', backref='+') @@ -726,5 +725,22 @@ class Meta: table_name = 'mipsgal' +class TESS_TOI(CatalogdbModel): + + tic = ForeignKeyField(TIC_v8, + column_name='ticid', + object_id_name='ticid', + backref='+') + + class Meta: + table_name = 'yso_clustering' + + +class XMM_OM_SUSS_4_1(CatalogdbModel): + + class Meta: + table_name = 'xmm_om_suss_4_1' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql new file mode 100644 index 00000000..eeb098f7 --- /dev/null +++ b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql @@ -0,0 +1,132 @@ +/* + +XMM-Newton Optical Monitor SUSS Catalog, v4.1 +https://heasarc.gsfc.nasa.gov/W3Browse/all/xmmomsuob.html + +*/ + +CREATE TABLE catalogdb.xmm_om_suss_4_1 ( + iauname VARCHAR(22), + n_summary INTEGER, + obsid VARCHAR(10), + srcnum INTEGER, + uvw2_srcdist REAL, + uvm2_srcdist REAL, + uvw1_srcdist REAL, + u_srcdist REAL, + b_srcdist REAL, + v_srcdist REAL, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + ra_hms VARCHAR(13), + dec_dms VARCHAR(14), + poserr REAL, + lii DOUBLE PRECISION, + bii DOUBLE PRECISION, + n_obsid INTEGER, + uvw2_signif REAL, + uvm2_signif REAL, + uvw1_signif REAL, + u_signif REAL, + b_signif REAL, + v_signif REAL, + uvw2_rate REAL, + uvw2_rate_err REAL, + uvm2_rate REAL, + uvm2_rate_err REAL, + uvw1_rate REAL, + uvw1_rate_err REAL, + u_rate REAL, + u_rate_err REAL, + b_rate REAL, + b_rate_err REAL, + v_rate REAL, + v_rate_err REAL, + uvw2_ab_flux REAL, + uvw2_ab_flux_err REAL, + uvm2_ab_flux REAL, + uvm2_ab_flux_err REAL, + uvw1_ab_flux REAL, + uvw1_ab_flux_err REAL, + u_ab_flux REAL, + u_ab_flux_err REAL, + b_ab_flux REAL, + b_ab_flux_err REAL, + v_ab_flux REAL, + v_ab_flux_err REAL, + uvw2_ab_mag REAL, + uvw2_ab_mag_err REAL, + uvm2_ab_mag REAL, + uvm2_ab_mag_err REAL, + uvw1_ab_mag REAL, + uvw1_ab_mag_err REAL, + u_ab_mag REAL, + u_ab_mag_err REAL, + b_ab_mag REAL, + b_ab_mag_err REAL, + v_ab_mag REAL, + v_ab_mag_err REAL, + uvw2_vega_mag REAL, + uvw2_vega_mag_err REAL, + uvm2_vega_mag REAL, + uvm2_vega_mag_err REAL, + uvw1_vega_mag REAL, + uvw1_vega_mag_err REAL, + u_vega_mag REAL, + u_vega_mag_err REAL, + b_vega_mag REAL, + b_vega_mag_err REAL, + v_vega_mag REAL, + v_vega_mag_err REAL, + uvw2_major_axis REAL, + uvm2_major_axis REAL, + uvw1_major_axis REAL, + u_major_axis REAL, + b_major_axis REAL, + v_major_axis REAL, + uvw2_minor_axis REAL, + uvm2_minor_axis REAL, + uvw1_minor_axis REAL, + u_minor_axis REAL, + b_minor_axis REAL, + v_minor_axis REAL, + uvw2_posang REAL, + uvm2_posang REAL, + uvw1_posang REAL, + u_posang REAL, + b_posang REAL, + v_posang REAL, + uvw2_quality_flag SMALLINT, + uvm2_quality_flag SMALLINT, + uvw1_quality_flag SMALLINT, + u_quality_flag SMALLINT, + b_quality_flag SMALLINT, + v_quality_flag SMALLINT, + uvw2_quality_flag_st VARCHAR(12), + uvm2_quality_flag_st VARCHAR(12), + uvw1_quality_flag_st VARCHAR(12), + u_quality_flag_st VARCHAR(12), + b_quality_flag_st VARCHAR(12), + v_quality_flag_st VARCHAR(12), + uvw2_extended_flag BIGINT, + uvm2_extended_flag BIGINT, + uvw1_extended_flag BIGINT, + u_extended_flag BIGINT, + b_extended_flag BIGINT, + v_extended_flag BIGINT, + uvw2_sky_image VARCHAR(4), + uvm2_sky_image VARCHAR(4), + uvw1_sky_image VARCHAR(4), + u_sky_image VARCHAR(4), + b_sky_image VARCHAR(4), + v_sky_image VARCHAR(4) +) WITHOUT OIDS; + + +\COPY catalogdb.xmm_om_suss_4_1 FROM PROGRAM 'zcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/XMMOMSUOB/XMM-OM-SUSS4.1.csv.gz' WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.xmm_om_suss_4_1 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX on catalogdb.xmm_om_suss_4_1 (q3c_ang2ipix(ra, dec)); +CLUSTER xmm_om_suss_4_1_q3c_ang2ipix_idx on catalogdb.xmm_om_suss_4_1; +VACUUM ANALYZE catalogdb.xmm_om_suss_4_1; From cdab6b6f90d53371c265ab12245064ed0d4eea79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 13:29:48 -0600 Subject: [PATCH 173/197] Add TESS-TOI --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- .../sdss5db/catalogdb/TESS-TOI/tess_toi.sql | 27 +++++++++++++++++++ schema/sdss5db/catalogdb/foreignKeys.sql | 10 +++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index ef904e0c..93735dc2 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -733,7 +733,7 @@ class TESS_TOI(CatalogdbModel): backref='+') class Meta: - table_name = 'yso_clustering' + table_name = 'tess_toi' class XMM_OM_SUSS_4_1(CatalogdbModel): diff --git a/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql b/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql new file mode 100644 index 00000000..1e8c5f0b --- /dev/null +++ b/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql @@ -0,0 +1,27 @@ +/* + +TESS Targets of Opportunity + +Original file /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/TESSobserved/TESS_Observed.fits +converted to CSV with + +t = table.Table.read('./TESS_Observed.fits') +t.convert_bytestring_to_unicode() +t.to_pandas().to_csv('./TESS_Observed.csv', index=False) + +*/ + +CREATE TABLE catalogdb.tess_toi ( + ticid BIGINT, + target_type VARCHAR(8), + toi VARCHAR(32), + tess_disposition VARCHAR(2), + tfopwg_disposition VARCHAR(2), + ctoi VARCHAR(32), + user_disposition VARCHAR(2), + num_sectors REAL +) WITHOUT OIDS; + +\COPY catalogdb.tess_toi FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/TESSobserved/TESS_Observed.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.tess_toi ADD COLUMN pk BIGSERIAL PRIMARY KEY; diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index dc8cc2c7..b1a1a5a9 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -298,3 +298,13 @@ ALTER TABLE catalogdb.mipsgal ADD CONSTRAINT glimpse_fk FOREIGN KEY (glimpse) REFERENCES catalogdb.glimpse (designation); + + +-- TESS-TOI + +CREATE INDEX CONCURRENTLY ON catalogdb.tess_toi USING BTREE (ticid); + +ALTER TABLE catalogdb.tess_toi + ADD CONSTRAINT ticid_fk + FOREIGN KEY (ticid) + REFERENCES catalogdb.tic_v8 (id); From c8e978119857767260b9b31b6c56d6296a8af531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 13:30:14 -0600 Subject: [PATCH 174/197] More zero-padding of Tycho2 designation --- schema/sdss5db/catalogdb/tycho2/tycho2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql index fd52de5b..47f79aaa 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -379,7 +379,7 @@ CREATE TABLE catalogdb.tycho2 ( \COPY catalogdb.tycho2 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/tycho2_all.csv WITH CSV HEADER DELIMITER ','; ALTER TABLE catalogdb.tycho2 ADD COLUMN designation TEXT; -UPDATE catalogdb.tycho2 SET designation = tyc1 || '-' || LPAD(tyc2::TEXT, 5, '0') || '-' || tyc3; +UPDATE catalogdb.tycho2 SET designation = LPAD(tyc1::TEXT, 4, '0') || '-' || LPAD(tyc2::TEXT, 5, '0') || '-' || tyc3; -- Need to remove duplicates in the supplement tables. This assumes that the supplement tables -- are towards the end of the table, which may not be true, but seems to work. From 7ed605e2266127e5f658d788fbefba5bd2ff9bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 22:22:54 -0600 Subject: [PATCH 175/197] Add tools to indentify (un)clustered tables --- python/sdssdb/utils/maintenance.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/maintenance.py index 5d453cf9..c15f151a 100644 --- a/python/sdssdb/utils/maintenance.py +++ b/python/sdssdb/utils/maintenance.py @@ -73,3 +73,52 @@ def execute_sql(statement): ' ' + table_name) execute_sql(statement) + + +def get_cluster_index(connection, table_name=None, schema=None): + """Returns a tuple with the index on which a table has been clustered.""" + + table_name = table_name or '%%' + schema = schema or '%%' + + cursor = connection.execute_sql(f""" + SELECT + c.relname AS tablename, + n.nspname AS schemaname, + i.relname AS indexname + FROM pg_index x + JOIN pg_class c ON c.oid = x.indrelid + JOIN pg_class i ON i.oid = x.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE + x.indisclustered AND + n.nspname LIKE '{schema}' AND + c.relname LIKE '{table_name}'; + """) + + return cursor.fetchall() + + +def get_unclustered_tables(connection, schema=None): + """Lists tables not clustered.""" + + schema_like = schema or '%%' + + table_names = connection.execute_sql(f""" + SELECT + tablename + FROM pg_tables + WHERE + schemaname LIKE '{schema_like}'; + """).fetchall() + + table_names = [table_name[0] for table_name in table_names] + + clustered = get_cluster_index(connection, schema=schema) + if schema: + clustered = [idx for idx in clustered if idx[1] == schema] + + unclustered = [table for table in table_names + if table not in list(zip(*clustered))[0]] + + return unclustered From 7b56174d2757b5fa7a95adffbfb94bdf439c4e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 22:26:05 -0600 Subject: [PATCH 176/197] Make MIPSGAL->GLIMPSE a one-to-many --- python/sdssdb/peewee/sdss5db/catalogdb.py | 7 ++++++- schema/sdss5db/catalogdb/foreignKeys.sql | 5 ----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 93735dc2..052dfa91 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -716,11 +716,16 @@ class Meta: class MIPSGAL(CatalogdbModel): - glimpse = ForeignKeyField(GLIMPSE, backref='+') twomass = ForeignKeyField(TwoMassPSC, object_id_name='twomass_name', backref='+') + @property + def glimpse(self): + """One-to-many for GLIMPSE targets.""" + + return GLIMPSE.select().where(GLIMPSE.designation == self.glimpse) + class Meta: table_name = 'mipsgal' diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index b1a1a5a9..a3aecdcb 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -294,11 +294,6 @@ ALTER TABLE catalogdb.mipsgal FOREIGN KEY (twomass_name) REFERENCES catalogdb.twomass_psc (designation); -ALTER TABLE catalogdb.mipsgal - ADD CONSTRAINT glimpse_fk - FOREIGN KEY (glimpse) - REFERENCES catalogdb.glimpse (designation); - -- TESS-TOI From f3ef9ca24b080d8536b8c2622c61f7453da497ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 4 Apr 2020 00:21:02 -0600 Subject: [PATCH 177/197] Fix unique constraint in twomass_psc --- schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql index d5a45acc..7df0d46f 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql @@ -18,7 +18,9 @@ CREATE INDEX ON catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); CLUSTER twomass_psc_q3c_ang2ipix_idx ON catalogdb.twomass_psc; ANALYZE catalogdb.twomass_psc; -ALTER TABLE catalogdb.twomass_psc ADD CONSTRAINT UNIQUE (designation); +ALTER TABLE catalogdb.twomass_psc + ADD CONSTRAINT catalogdb_twomass_psc_designation_unique + UNIQUE (designation); CREATE INDEX ON catalogdb.twomass_psc USING BTREE (designation); CREATE INDEX ON catalogdb.twomass_psc USING BTREE (ph_qual); From 71ec72ae5f7783f2a2b87abfd5d9f5e6a8cfbec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Fri, 3 Apr 2020 23:59:54 -0700 Subject: [PATCH 178/197] Docstring changes in ReflectMeta --- python/sdssdb/peewee/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 05a8285b..55507097 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -20,7 +20,7 @@ class ReflectMeta(ModelBase): """A metaclass that supports model reflection on demand. - This metaclass expands Peewee's ``ModelBase`` to provide a hook for + This metaclass expands PeeWee's ``ModelBase`` to provide a hook for declaring/expanding fields and indexes using the :ref:`introspection system `. The feature is enabled by a new attribute in :class:`peewee:Metadata` called ``use_reflection`` @@ -75,8 +75,10 @@ class Meta: reflection_options = {'skip_foreign_keys': True} database = database - Foreign keys explicitely defined need to reference fields that exist so the - referenced columns need to be added manually. + Foreign keys explicitely defined need to reference existing fields, + so the referenced columns need to be added manually. In practice, this + means that if you add a `peewee:ForeignKeyField`, the referenced field + (usually the primary key) needs to be defined explicitely. Caveats: @@ -89,6 +91,10 @@ class Meta: and the foreign key need to be defined explicitely. Otherwise neither will be added. + - `.ReflectMeta` is designed to fail silently if PeeWee's reflection fails. + Error are logged with debug level and can be display by setting + doing ``from sdssdb import log`` and ``log.set_level(0)``. + """ def __new__(cls, name, bases, attrs): From 96bf7dcab007c11ba1ff296b6e2020c663e4412b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 4 Apr 2020 09:20:56 -0700 Subject: [PATCH 179/197] Some fixes --- schema/sdss5db/catalogdb/foreignKeys.sql | 4 ++-- schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index a3aecdcb..13a35324 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -161,8 +161,8 @@ ALTER TABLE catalogdb.tic_v8 ON UPDATE CASCADE ON DELETE CASCADE; ALTER TABLE catalogdb.tic_v8 - ADD CONSTRAINT twomass_fk - FOREIGN KEY (twomass) + ADD CONSTRAINT twomass_psc_fk + FOREIGN KEY (twomass_psc) REFERENCES catalogdb.twomass_psc (designation) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index c7fdbaf1..a1a49f06 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -8,7 +8,7 @@ UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; -- Indices CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.tic_v8; +CLUSTER tic_v8_q3c_ang2ipix_idx ON catalogdb.tic_v8; ANALYZE catalogdb.tic_v8; CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Bmag); From e00be3a884e4589f609b32ac5d984ec5fa92a41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 4 Apr 2020 13:27:34 -0700 Subject: [PATCH 180/197] Remove comma in table definition for tic_v8 --- schema/sdss5db/catalogdb/tic/v8/tic_v8.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql index 5d918bbc..ca9a4086 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql @@ -131,5 +131,5 @@ CREATE TABLE catalogdb.tic_v8 ( e_dec_orig DOUBLE PRECISION, raddflag INTEGER, wdflag INTEGER, - objid BIGINT, + objid BIGINT ) WITHOUT OIDS; From b48637d1fd01f5667d9b5ebf8ea24daa28f159e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sat, 4 Apr 2020 13:33:41 -0700 Subject: [PATCH 181/197] Remove unused tycho2 scripts --- .../sdss5db/catalogdb/tycho2/tycho2_index.sql | 17 ------ .../sdss5db/catalogdb/tycho2/tycho2_load.py | 60 ------------------- 2 files changed, 77 deletions(-) delete mode 100644 schema/sdss5db/catalogdb/tycho2/tycho2_index.sql delete mode 100755 schema/sdss5db/catalogdb/tycho2/tycho2_load.py diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql b/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql deleted file mode 100644 index 01ba6745..00000000 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_index.sql +++ /dev/null @@ -1,17 +0,0 @@ - -CREATE UNIQUE INDEX CONCURRENTLY catalogdb_tycho2_name_unique_idx ON catalogdb.tycho2 (name); -ALTER TABLE catalogdb.tycho2 - ADD CONSTRAINT catalogdb_tycho2_name_unique - UNIQUE USING INDEX catalogdb_tycho2_name_unique_idx; - - --- Also create a designation column which is the same as name but with dashes --- (for MIPSGAL and maybe others) -ALTER TABLE catalogdb.tycho2 ADD COLUMN designation TEXT; -UPDATE catalogdb.tycho2 SET designation = REPLACE(name, ' ', '-'); -ALTER TABLE catalogdb.tycho2 - ADD CONSTRAINT catalogdb_tycho2_designation_unique UNIQUE (designation); - -CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); -CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; -ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py b/schema/sdss5db/catalogdb/tycho2/tycho2_load.py deleted file mode 100755 index 94a2d5fb..00000000 --- a/schema/sdss5db/catalogdb/tycho2/tycho2_load.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# @Author: José Sánchez-Gallego (gallegoj@uw.edu) -# @Date: 2020-03-18 -# @Filename: tycho2_load.py -# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) - -import glob -import pandas -import io - -from sdssdb.peewee.sdss5db import database - - -def main(): - - assert database.connected - - path_ = '/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/' - paths = list(glob.glob(path_ + 'tyc2.dat.*.gz')) + [path_ + 'suppl_1.dat.gz', - path_ + 'suppl_2.dat.gz'] - - for file_ in sorted(paths): - - print(file_) - - data = pandas.read_csv(file_, delimiter='|', low_memory=False, - header=None, skipinitialspace=True) - - # Convert columns num to integer with NA. - data[12] = data[12].astype('Int32') - - # Split column into hip and ccdm - col = data[23].fillna('') - hip = col.map(lambda x: x[0:-3].strip()) - ccdm = col.map(lambda x: x[-3:].strip()) - - data[23] = ccdm - data.insert(23, 'hip', pandas.to_numeric(hip).astype('Int32')) - - # Add the id_tycho column - tyc1, tyc2, tyc3 = data[0].str.split(' ', expand=True).apply(pandas.to_numeric).T.values - id_tycho = (tyc1 * 1000000) + (tyc2 * 10) + (tyc3 * 1) - data.insert(0, 'id_tycho', id_tycho) - - # Write as CSV to data stream. - csv = io.StringIO() - data.to_csv(csv, header=False, index=False) - csv.seek(0) - - cursor = database.cursor() - cursor.copy_expert('COPY catalogdb.tycho2 FROM STDIN ' - 'WITH DELIMITER \',\' CSV;', csv) - database.commit() - - -if __name__ == '__main__': - - main() From e16d39c2c1bc7241a6b219f75aaa81c3a9d903c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 6 Apr 2020 08:29:14 -0700 Subject: [PATCH 182/197] Fix more issues with catalogdb --- python/sdssdb/peewee/sdss5db/catalogdb.py | 2 +- .../catalogdb/allwise/source/allwise_index.sql | 11 +++++++---- schema/sdss5db/catalogdb/foreignKeys.sql | 2 +- .../legacy_survey/dr8/legacy_survey_index.sql | 4 ++-- schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 8 +++++++- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index 052dfa91..d8e0ee85 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -476,7 +476,7 @@ class TIC_v8(CatalogdbModel): backref='tic') twomass_psc = ForeignKeyField(TwoMassPSC, field='designation', - column_name='twomass', object_id_name='twomass', + column_name='twomass_psc', backref='tic') photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, field='objid', diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql index a595a220..3e8d2544 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql @@ -17,13 +17,16 @@ ALTER TABLE catalogdb.allwise ADD PRIMARY KEY (cntr); -- Indices -CREATE INDEX ON catalogdb.allwise using BTREE (designation); +CREATE INDEX ON catalogdb.allwise USING BTREE (designation); -CREATE INDEX ON catalogdb.allwise using BTREE (ra); -CREATE INDEX ON catalogdb.allwise using BTREE (dec); -CREATE INDEX ON catalogdb.allwise using BTREE (glat); +CREATE INDEX ON catalogdb.allwise USING BTREE (ra); +CREATE INDEX ON catalogdb.allwise USING BTREE (dec); +CREATE INDEX ON catalogdb.allwise USING BTREE (glat); CREATE INDEX ON catalogdb.allwise using BTREE (glon); +ALTER TABLE catalogdb.allwise + ADD CONSTRAINT allwise_designation_unique UNIQUE (designation); + CREATE INDEX ON catalogdb.allwise (q3c_ang2ipix(ra, dec)); CLUSTER allwise_q3c_ang2ipix_idx ON catalogdb.allwise; ANALYZE catalogdb.allwise; diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index 13a35324..f4b49c05 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -143,7 +143,7 @@ ALTER TABLE catalogdb.sdss_dr14_ascapStar CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass_psc); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index f1619a5a..df9b362d 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -11,5 +11,5 @@ UPDATE catalogdb.legacy_survey_dr8 SET gaia_sourceid = ref_id CREATE INDEX ON catalogdb.legacy_survey_dr8 (gaia_sourceid); CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); -CLUSTER legacy_survey_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; -ANALYZE catalogdb.legacy_survey_dr8; +CLUSTER legacy_survey_dr8_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; +VACUUM ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index a1a49f06..ca98b3fd 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -9,7 +9,7 @@ UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); CLUSTER tic_v8_q3c_ang2ipix_idx ON catalogdb.tic_v8; -ANALYZE catalogdb.tic_v8; +VACUUM ANALYZE catalogdb.tic_v8; CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Bmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Vmag); @@ -21,3 +21,9 @@ CREATE INDEX ON catalogdb.tic_v8 USING BTREE (zmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Jmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Hmag); CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Kmag); +CREATE INDEX ON catalogdb.tic_v8 USING BTREE (posflag); + +ALTER TABLE catalogdb.tic_v8 ADD COLUMN twomass_psc TEXT; +UPDATE catalogdb.tic_v8 + SET twomass_psc = twomass + WHERE twomass IS NOT NULL AND posflag != '2MASSEXT'; From 1d864de60b6e35200204229fbcfd05f935590b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 08:13:39 -0700 Subject: [PATCH 183/197] Rename catalog table and add to ORM --- python/sdssdb/peewee/sdss5db/catalogdb.py | 4 ++++ schema/sdss5db/catalogdb/{catalogdb.sql => catalog.sql} | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) rename schema/sdss5db/catalogdb/{catalogdb.sql => catalog.sql} (85%) diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index d8e0ee85..a944bc9d 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -30,6 +30,10 @@ class Meta: _APOGEE_Star_Visit_Deferred = DeferredThroughModel() +class Catalog(CatalogdbModel): + pass + + class AllWise(CatalogdbModel): cntr = BigIntegerField(primary_key=True) diff --git a/schema/sdss5db/catalogdb/catalogdb.sql b/schema/sdss5db/catalogdb/catalog.sql similarity index 85% rename from schema/sdss5db/catalogdb/catalogdb.sql rename to schema/sdss5db/catalogdb/catalog.sql index 72153596..a0482349 100644 --- a/schema/sdss5db/catalogdb/catalogdb.sql +++ b/schema/sdss5db/catalogdb/catalog.sql @@ -17,4 +17,6 @@ CREATE TABLE catalogdb.catalog ( dec DOUBLE PRECISION, pmra REAL, pmdec REAL, - epoch REAL); + epoch REAL, + parallax REAL, + version TEXT); From 8f0501852abcc574c46e65f104281800cd992981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 08:34:10 -0700 Subject: [PATCH 184/197] Add tool to retrieve row count --- python/sdssdb/utils/maintenance.py | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/maintenance.py index c15f151a..8483a50b 100644 --- a/python/sdssdb/utils/maintenance.py +++ b/python/sdssdb/utils/maintenance.py @@ -122,3 +122,41 @@ def get_unclustered_tables(connection, schema=None): if table not in list(zip(*clustered))[0]] return unclustered + + +def get_row_count(connection, table_name, schema=None, approximate=True): + """Returns the model row count. + + Parameters + ---------- + connection : .PeeweeDatabaseConnection + The database connection. + table : str + The table name. + schema : str + The schema in which the table lives. + approximate : bool + If True, returns the approximate row count from the ``pg_class`` + table (much faster). Otherwise calculates the exact count. + + """ + + if approximate: + if schema is None: + sql = ('SELECT reltuples AS approximate_row_count ' + 'FROM pg_class WHERE relname = \'{table_name}\';') + else: + sql = ('SELECT reltuples AS approximate_row_count FROM pg_class ' + 'JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace ' + 'WHERE relname = \'{table_name}\' AND pg_namespace.nspname = \'{schema}\';') + else: + if schema is None: + sql = 'SELECT count(*) FROM {table_name};' + else: + sql = 'SELECT count(*) FROM {schema}.{table_name};' + + count = connection.execute_sql(sql.format(table_name=table_name, schema=schema)).fetchall() + if len(count) == 0: + raise ValueError('failed retrieving the row count. Check the table name and schema.') + + return count[0][0] From 491289a513c669667b833238cc70d28e4462f8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 08:34:21 -0700 Subject: [PATCH 185/197] Import utils to top level --- python/sdssdb/utils/__init__.py | 2 ++ python/sdssdb/utils/ingest.py | 4 ++++ python/sdssdb/utils/maintenance.py | 4 ++++ python/sdssdb/utils/schemadisplay.py | 10 +++++++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/utils/__init__.py b/python/sdssdb/utils/__init__.py index a8ecc0a9..3382d31e 100644 --- a/python/sdssdb/utils/__init__.py +++ b/python/sdssdb/utils/__init__.py @@ -1,3 +1,5 @@ # flake8: noqa from .schemadisplay import * +from .ingest import * +from .maintenance import * diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index d88064ed..822054a1 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -29,6 +29,10 @@ progressbar = False +__all__ = ('to_csv', 'copy_data', 'drop_table', 'create_model_from_table', + 'bulk_insert', 'file_to_db') + + DTYPE_TO_FIELD = { 'i2': peewee.SmallIntegerField, 'i4': peewee.IntegerField, diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/maintenance.py index 8483a50b..71a12fba 100644 --- a/python/sdssdb/utils/maintenance.py +++ b/python/sdssdb/utils/maintenance.py @@ -10,6 +10,10 @@ import time +__all__ = ('vacuum_all', 'get_cluster_index', 'get_unclustered_tables', + 'get_row_count') + + def vacuum_all(database, analyze=True, verbose=False, schema=None): """Vacuums all the tables in a database or schema. diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index b11de006..f8ecf8b1 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -14,12 +14,18 @@ import re -import pydot from peewee import ForeignKeyField +try: + import pydot +except ImportError: + pydot = None + + __all__ = ['create_schema_graph', 'show_schema_graph'] + field_type_psql = {'AUTO': 'SERIAL', 'BIGAUTO': 'BIGSERIAL', 'BIGINT': 'BIGINT', @@ -235,6 +241,8 @@ def create_schema_graph(models=None, base=None, schema=None, show_columns=True, """ assert models or base, 'either model or base must be passed.' + assert pydot, ('pydot is required for create_schema_graph. ' + 'Try running "pip install sdssdb[all]"') relation_kwargs = {'fontsize': '7.0'} relation_kwargs.update(relation_options) From 9b42ef764f7f546a47b0081711e7b4ee5850c9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 20:12:34 -0700 Subject: [PATCH 186/197] Add geometric_distances_gaia_dr2 and bhm_rm_v0 --- python/sdssdb/peewee/sdss5db/catalogdb.py | 34 ++++ .../sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql | 168 ++++++++++++++++++ schema/sdss5db/catalogdb/foreignKeys.sql | 30 ++++ .../geometric_distances.sql | 25 +++ 4 files changed, 257 insertions(+) create mode 100644 schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql create mode 100644 schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index a944bc9d..0f7cfdb1 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -751,5 +751,39 @@ class Meta: table_name = 'xmm_om_suss_4_1' +class GeometricDistances_Gaia_DR2(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='+') + + class Meta: + table_name = 'geometric_distances_gaia_dr2' + + +class BHM_RM_v0(CatalogdbModel): + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id_gaia', + object_id_name='source_id_gaia', + backref='+') + + unwise = ForeignKeyField(unWISE, + column_name='objid_unwise', + object_id_name='objid_unwise', + backref='+') + + sdss = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='objid_sdss', + object_id_name='objid_sdss', + backref='+') + + class Meta: + table_name = 'bhm_rm_v0' + + _Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) _APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) diff --git a/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql new file mode 100644 index 00000000..d5b573a8 --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql @@ -0,0 +1,168 @@ +/* + +BHM Reverberation Mapping Photometry v0. + +See /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/RM/v0/rm_photo_v0_readme +CSV file generated using sdssdb.utils.ingest.to_csv with default options. + +*/ + +CREATE TABLE catalogdb.bhm_rm_v0 ( + field_name VARCHAR(8), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + distance DOUBLE PRECISION, + pos_ref VARCHAR(4), + ebv DOUBLE PRECISION, + des INTEGER, + coadd_object_id BIGINT, + ra_des DOUBLE PRECISION, + dec_des DOUBLE PRECISION, + extended_coadd INTEGER, + psfmag_des DOUBLE PRECISION[5], + psfmagerr_des DOUBLE PRECISION[5], + mag_auto_des DOUBLE PRECISION[5], + magerr_auto_des DOUBLE PRECISION[5], + imaflags_iso INTEGER[5], + separation_des DOUBLE PRECISION, + ps1 INTEGER, + objid_ps1 BIGINT, + ra_ps1 DOUBLE PRECISION, + dec_ps1 DOUBLE PRECISION, + class_ps1 DOUBLE PRECISION, + psfmag_ps1 DOUBLE PRECISION[5], + psfmagerr_ps1 DOUBLE PRECISION[5], + kronmag_ps1 DOUBLE PRECISION[5], + kronmagerr_ps1 DOUBLE PRECISION[5], + infoflag2 INTEGER[5], + separation_ps1 DOUBLE PRECISION, + nsc INTEGER, + id_nsc BIGINT, + ra_nsc DOUBLE PRECISION, + dec_nsc DOUBLE PRECISION, + class_star DOUBLE PRECISION, + mag_nsc DOUBLE PRECISION[5], + magerr_nsc DOUBLE PRECISION[5], + flags_nsc INTEGER, + separation_nsc DOUBLE PRECISION, + sdss INTEGER, + objid_sdss BIGINT, + ra_sdss DOUBLE PRECISION, + dec_sdss DOUBLE PRECISION, + type_sdss INTEGER, + psfmag_sdss DOUBLE PRECISION[5], + psfmagerr_sdss DOUBLE PRECISION[5], + modelmag_sdss DOUBLE PRECISION[5], + modelmagerr_sdss DOUBLE PRECISION[5], + clean_sdss INTEGER, + separation_sdss DOUBLE PRECISION, + gaia INTEGER, + source_id_gaia BIGINT, + mg DOUBLE PRECISION, + mag_gaia DOUBLE PRECISION[3], + magerr_gaia DOUBLE PRECISION[3], + parallax DOUBLE PRECISION, + parallax_error DOUBLE PRECISION, + plxsig DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_error DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_error DOUBLE PRECISION, + pmsig DOUBLE PRECISION, + unwise INTEGER, + objid_unwise VARCHAR(16), + ra_unwise DOUBLE PRECISION, + dec_unwise DOUBLE PRECISION, + mag_unwise DOUBLE PRECISION[2], + magerr_unwise DOUBLE PRECISION[2], + flags_unwise INTEGER[2], + separation_unwise DOUBLE PRECISION, + near_ir INTEGER, + survey_ir VARCHAR(6), + sourceid_ir BIGINT, + ra_ir DOUBLE PRECISION, + dec_ir DOUBLE PRECISION, + mag_ir DOUBLE PRECISION[4], + magerr_ir DOUBLE PRECISION[4], + separation_ir DOUBLE PRECISION, + optical_survey VARCHAR(4), + mi DOUBLE PRECISION, + cal_skewt_qso INTEGER, + nband_optical_use INTEGER, + use_unwise INTEGER, + use_nir INTEGER, + photo_combination VARCHAR(17), + log_qso DOUBLE PRECISION, + log_star DOUBLE PRECISION, + log_galaxy DOUBLE PRECISION, + p_qso DOUBLE PRECISION, + p_star REAL, + p_galaxy DOUBLE PRECISION, + class_skewt_qso VARCHAR(6), + skewt_qso INTEGER, + p_qso_prior DOUBLE PRECISION, + p_star_prior REAL, + p_galaxy_prior DOUBLE PRECISION, + class_skewt_qso_prior VARCHAR(6), + skewt_qso_prior INTEGER, + photoz_qso DOUBLE PRECISION, + photoz_qso_lower DOUBLE PRECISION, + photoz_qso_upper DOUBLE PRECISION, + prob_photoz_qso DOUBLE PRECISION, + photoz_galaxy DOUBLE PRECISION, + photoz_galaxy_lower DOUBLE PRECISION, + photoz_galaxy_upper DOUBLE PRECISION, + pqso_xdqso DOUBLE PRECISION, + photoz_xdqso DOUBLE PRECISION, + prob_rf_gaia_unwise DOUBLE PRECISION, + photoz_gaia_unwise DOUBLE PRECISION, + des_var_nepoch INTEGER[5], + des_var_status INTEGER[5], + des_var_rms DOUBLE PRECISION[5], + des_var_sigrms DOUBLE PRECISION[5], + des_var_sn DOUBLE PRECISION[5], + des_var_sn_max DOUBLE PRECISION, + ps1_var_nepoch INTEGER[5], + ps1_var_status INTEGER[5], + ps1_var_rms DOUBLE PRECISION[5], + ps1_var_sigrms DOUBLE PRECISION[5], + ps1_var_sn DOUBLE PRECISION[5], + ps1_var_sn_max DOUBLE PRECISION, + spec_q INTEGER, + spec_strmask VARCHAR(6), + spec_bitmask BIGINT, + specz DOUBLE PRECISION, + specz_ref VARCHAR(16), + photo_q INTEGER, + photo_strmask VARCHAR(3), + photo_bitmask BIGINT, + photoz DOUBLE PRECISION, + pqso_photo DOUBLE PRECISION, + photoz_ref VARCHAR(16) +) WITHOUT OIDS; + + +\COPY catalogdb.bhm_rm_v0 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/RM/v0/rm_photo_v0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; + +ALTER TABLE catalogdb.bhm_rm_v0 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX ON catalogdb.bhm_rm_v0 (q3c_ang2ipix(ra, dec)); +CLUSTER bhm_rm_v0_q3c_ang2ipix_idx ON catalogdb.bhm_rm_v0; +VACUUM ANALYZE catalogdb.bhm_rm_v0; + +-- Update to NULL in columns that will be fks. +UPDATE catalogdb.bhm_rm_v0 SET source_id_gaia = NULL WHERE gaia <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_ps1 = NULL WHERE ps1 <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_sdss = NULL WHERE sdss <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_unwise = NULL WHERE unwise <= 0; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (mi); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (skewt_qso); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (pmsig); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (plxsig); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (spec_q); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (des_var_rms); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (des_var_sn); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (ps1_var_rms); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (ps1_var_sn); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (photo_bitmask); diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index f4b49c05..d18045ef 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -303,3 +303,33 @@ ALTER TABLE catalogdb.tess_toi ADD CONSTRAINT ticid_fk FOREIGN KEY (ticid) REFERENCES catalogdb.tic_v8 (id); + + +-- geometric_distances_gaia_dr2 + +ALTER TABLE catalogdb.geometric_distances_gaia_dr2 + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- bhm_rm_v0 + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (source_id_gaia) +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (objid_sdss) +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (objid_unwise) + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT source_id_gaia_fk + FOREIGN KEY (source_id_gaia) + REFERENCES catalogdb.gaia_dr2_source (source_id); + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT objid_sdss_fk + FOREIGN KEY (objid_sdss) + REFERENCES catalogdb.sdss_dr13_photoobj (objid); + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT objid_unwise_fk + FOREIGN KEY (objid_unwise) + REFERENCES catalogdb.unwise (unwise_objid); diff --git a/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql b/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql new file mode 100644 index 00000000..5d683b42 --- /dev/null +++ b/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql @@ -0,0 +1,25 @@ +/* + +Geometric distances from Bailer-Jones Gaia DR2 + +*/ + + +CREATE TABLE catalogdb.geometric_distances_gaia_dr2 ( + source_id BIGINT, + r_est REAL, + r_lo REAL, + r_hi REAL, + r_len REAL, + result_flag CHAR(1), + modality_flag SMALLINT +) WITHOUT OIDS; + +\COPY catalogdb.geometric_distances_gaia_dr2 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/geometric_distances/cbj_geometric_distances.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.geometric_distances_gaia_dr2 ADD PRIMARY KEY (source_id); + +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_est); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_lo); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_hi); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_len); From 40ae486e9121d8ee3c95dc43ecc53a7512df776b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 20:56:47 -0700 Subject: [PATCH 187/197] Rename maintenance to internals and add is_table_locked --- python/sdssdb/utils/__init__.py | 2 +- .../utils/{maintenance.py => internals.py} | 78 ++++++++++++------- 2 files changed, 53 insertions(+), 27 deletions(-) rename python/sdssdb/utils/{maintenance.py => internals.py} (69%) diff --git a/python/sdssdb/utils/__init__.py b/python/sdssdb/utils/__init__.py index 3382d31e..3dd95c52 100644 --- a/python/sdssdb/utils/__init__.py +++ b/python/sdssdb/utils/__init__.py @@ -2,4 +2,4 @@ from .schemadisplay import * from .ingest import * -from .maintenance import * +from .internals import * diff --git a/python/sdssdb/utils/maintenance.py b/python/sdssdb/utils/internals.py similarity index 69% rename from python/sdssdb/utils/maintenance.py rename to python/sdssdb/utils/internals.py index 71a12fba..c33aaf08 100644 --- a/python/sdssdb/utils/maintenance.py +++ b/python/sdssdb/utils/internals.py @@ -3,7 +3,7 @@ # # @Author: José Sánchez-Gallego (gallegoj@uw.edu) # @Date: 2020-03-28 -# @Filename: maintenance.py +# @Filename: internals.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) import sys @@ -11,7 +11,7 @@ __all__ = ('vacuum_all', 'get_cluster_index', 'get_unclustered_tables', - 'get_row_count') + 'get_row_count', 'is_table_locked') def vacuum_all(database, analyze=True, verbose=False, schema=None): @@ -62,7 +62,8 @@ def execute_sql(statement): if schema is None: statement = 'VACUUM' + (' VEBOSE' if verbose else '') + (' ANALYZE' if analyze else '') - execute_sql(statement) + with database.atomic(): + execute_sql(statement) return @@ -76,7 +77,8 @@ def execute_sql(statement): (' ANALYZE' if analyze else '') + ' ' + table_name) - execute_sql(statement) + with database.atomic(): + execute_sql(statement) def get_cluster_index(connection, table_name=None, schema=None): @@ -85,20 +87,21 @@ def get_cluster_index(connection, table_name=None, schema=None): table_name = table_name or '%%' schema = schema or '%%' - cursor = connection.execute_sql(f""" - SELECT - c.relname AS tablename, - n.nspname AS schemaname, - i.relname AS indexname - FROM pg_index x - JOIN pg_class c ON c.oid = x.indrelid - JOIN pg_class i ON i.oid = x.indexrelid - JOIN pg_namespace n ON n.oid = i.relnamespace - WHERE - x.indisclustered AND - n.nspname LIKE '{schema}' AND - c.relname LIKE '{table_name}'; - """) + with connection.atomic(): + cursor = connection.execute_sql(f""" + SELECT + c.relname AS tablename, + n.nspname AS schemaname, + i.relname AS indexname + FROM pg_index x + JOIN pg_class c ON c.oid = x.indrelid + JOIN pg_class i ON i.oid = x.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE + x.indisclustered AND + n.nspname LIKE '{schema}' AND + c.relname LIKE '{table_name}'; + """) return cursor.fetchall() @@ -108,13 +111,14 @@ def get_unclustered_tables(connection, schema=None): schema_like = schema or '%%' - table_names = connection.execute_sql(f""" - SELECT - tablename - FROM pg_tables - WHERE - schemaname LIKE '{schema_like}'; - """).fetchall() + with connection.atomic(): + table_names = connection.execute_sql(f""" + SELECT + tablename + FROM pg_tables + WHERE + schemaname LIKE '{schema_like}'; + """).fetchall() table_names = [table_name[0] for table_name in table_names] @@ -159,8 +163,30 @@ def get_row_count(connection, table_name, schema=None, approximate=True): else: sql = 'SELECT count(*) FROM {schema}.{table_name};' - count = connection.execute_sql(sql.format(table_name=table_name, schema=schema)).fetchall() + with connection.atomic(): + count = connection.execute_sql(sql.format(table_name=table_name, + schema=schema)).fetchall() + if len(count) == 0: raise ValueError('failed retrieving the row count. Check the table name and schema.') return count[0][0] + + +def is_table_locked(connection, table_name): + """Returns the type of lock for a table or `None` if no lock is present.""" + + sql = ('SELECT mode FROM pg_locks JOIN pg_class ON pg_class.oid = pg_locks.relation ' + f'WHERE pg_class.relname = {table_name!r}') + + with connection.atomic(): + lock = connection.execute_sql(sql).fetchall() + + if not lock: + return None + + if len(lock) > 1: + raise RuntimeError('received lock information for more than one relation. ' + 'Something must have gone wrong with the query.') + + return lock[0][0] From 37f6cf01236aa75679845e3d46435871261a133e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 22:38:33 -0600 Subject: [PATCH 188/197] Check for locks in reflection --- python/sdssdb/peewee/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 55507097..a03fc63c 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -15,6 +15,8 @@ from playhouse.reflection import generate_models from sdssdb import log +from sdssdb.core.exceptions import SdssdbUserWarning +from sdssdb.utils import is_table_locked class ReflectMeta(ModelBase): @@ -135,6 +137,11 @@ def reflect(self): schema = self._meta.schema try: + lock = is_table_locked(database, table_name) + if lock and lock == 'AccessExclusiveLock': + warnings.warn(f'table {schema}.{table_name} is locked and ' + 'will not be reflected.', SdssdbUserWarning) + return ReflectedModel = generate_models(database, schema=schema, table_names=[table_name])[table_name] except Exception as ee: From 07f900d6194a576b7068c5002711e5e97928174a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Tue, 7 Apr 2020 22:38:50 -0600 Subject: [PATCH 189/197] Use warnings instead of debug for reflection errors --- python/sdssdb/peewee/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index a03fc63c..e1920232 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -144,8 +144,12 @@ def reflect(self): return ReflectedModel = generate_models(database, schema=schema, table_names=[table_name])[table_name] + except KeyError as ee: + warnings.warn(f'reflection failed for {table_name}: table or column {ee} not found.', + SdssdbUserWarning) + return except Exception as ee: - log.debug('reflection failed for {}: {}'.format(table_name, ee)) + warnings.warn(f'reflection failed for {table_name}: {ee}', SdssdbUserWarning) return for field_name, field in ReflectedModel._meta.fields.items(): From 7017784dfc5739174bba65315d9839112dda8d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 01:13:47 -0600 Subject: [PATCH 190/197] Return multiple locks per relation --- python/sdssdb/peewee/__init__.py | 4 ++-- python/sdssdb/utils/internals.py | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index e1920232..40fa3fe2 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -137,8 +137,8 @@ def reflect(self): schema = self._meta.schema try: - lock = is_table_locked(database, table_name) - if lock and lock == 'AccessExclusiveLock': + locks = is_table_locked(database, table_name) + if locks and 'AccessExclusiveLock' in locks: warnings.warn(f'table {schema}.{table_name} is locked and ' 'will not be reflected.', SdssdbUserWarning) return diff --git a/python/sdssdb/utils/internals.py b/python/sdssdb/utils/internals.py index c33aaf08..cd70b610 100644 --- a/python/sdssdb/utils/internals.py +++ b/python/sdssdb/utils/internals.py @@ -174,19 +174,15 @@ def get_row_count(connection, table_name, schema=None, approximate=True): def is_table_locked(connection, table_name): - """Returns the type of lock for a table or `None` if no lock is present.""" + """Returns the locks for a table or `None` if no lock is present.""" sql = ('SELECT mode FROM pg_locks JOIN pg_class ON pg_class.oid = pg_locks.relation ' f'WHERE pg_class.relname = {table_name!r}') with connection.atomic(): - lock = connection.execute_sql(sql).fetchall() + locks = connection.execute_sql(sql).fetchall() - if not lock: + if not locks: return None - if len(lock) > 1: - raise RuntimeError('received lock information for more than one relation. ' - 'Something must have gone wrong with the query.') - - return lock[0][0] + return list(zip(*locks))[0] From 941d427cc21c79190cf3e2e6a412094f4720d5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 09:03:59 -0600 Subject: [PATCH 191/197] Use concurrently for all indexes --- .../catalogdb/CatWISE/CatWISE_indexes.sql | 4 +- .../catalogdb/GLIMPSE/GLIMPSE_index.sql | 2 +- .../Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql | 6 +-- .../catalogdb/PS1/g18/PS1_g18_index.sql | 8 ++-- .../sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql | 2 +- .../allwise/source/allwise_index.sql | 12 +++--- .../best_brightest/best_brightest.sql | 2 +- .../bhm_csc/v0.1.0/bhm_csc_index.sql | 12 +++--- .../sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql | 2 +- .../v0.1.0/BHM_SPIDERS_eFEDS_veto.sql | 6 +-- .../cataclysmic_variables.sql | 2 +- .../eRosita/v0.1.2/eRosita_index.sql | 8 ++-- .../v5/ebosstarget_v5_post_load.sql | 4 +- schema/sdss5db/catalogdb/foreignKeys.sql | 40 +++++++++---------- .../gaia/dr2/gaia_dr2_source_index.sql | 8 ++-- .../sdss5db/catalogdb/gaia_qso/gaia_qso.sql | 18 ++++----- .../catalogdb/guvcat/fov055/guvcat_index.sql | 2 +- .../kic/v10/kepler_input_10_index.sql | 28 ++++++------- .../legacy_survey/dr8/legacy_survey_index.sql | 8 ++-- schema/sdss5db/catalogdb/mipsgal/mipsgal.sql | 2 +- .../sdss5db/catalogdb/mwm_small/atnf/atnf.sql | 4 +- .../catalogdb/mwm_small/blackcat/blackcat.sql | 2 +- .../galactic_millisecond_pulsars.sql | 2 +- .../sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql | 2 +- .../transitional_msps/transitional_msps.sql | 2 +- .../catalogdb/mwm_small/watchdog/watchdog.sql | 2 +- .../mwm_small/xray_pulsars/xray_pulsars.sql | 2 +- .../r13/AllStarMerge_index.sql | 4 +- .../dr14/sdss_dr14_apogeeStar.sql | 12 +++--- .../dr14/sdss_dr14_apogeeStarVisit.sql | 4 +- .../dr14/sdss_dr14_apogeeVisit.sql | 10 ++--- .../dr14/sdss_dr14_ascapStar.sql | 4 +- .../dr14/sdss_dr14_cannonStar.sql | 2 +- .../dr13/sdss_dr13_photoobj_index.sql | 12 +++--- .../sdssSpecObj/dr14/sdss_dr14_specObj.sql | 10 ++--- .../sdssSpecObj/dr16/sdss_dr16_specObj.sql | 10 ++--- .../sdss_qso/dr14/sdss_dr14_qso_index.sql | 12 +++--- .../skymapper/dr1.1/skymapper_indexes.sql | 2 +- schema/sdss5db/catalogdb/smgaia/smgaia.sql | 6 +-- .../sdss5db/catalogdb/tic/v8/tic_v8_index.sql | 24 +++++------ .../twomass/psc/twomass_psc_index.sql | 18 ++++----- schema/sdss5db/catalogdb/tycho2/tycho2.sql | 2 +- .../sdss5db/catalogdb/unWISE/unWISE_index.sql | 2 +- .../sdss5db/catalogdb/uvotssc1/uvotssc1.sql | 2 +- .../sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql | 12 +++--- .../gaiaDR2/gaiadr2_tmass_best_neighbour.sql | 4 +- .../yso_clustering/yso_clustering.sql | 2 +- schema/sdss5db/targetdb/targetdb.sql | 2 +- 48 files changed, 174 insertions(+), 174 deletions(-) diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql index 17846aca..78919be4 100644 --- a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql @@ -7,10 +7,10 @@ ALTER TABLE catalogdb.catwise_reject ADD PRIMARY KEY (source_id); -- Indexes -CREATE INDEX ON catalogdb.catwise (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.catwise (q3c_ang2ipix(ra, dec)); CLUSTER catwise_q3c_ang2ipix_idx ON catalogdb.catwise; ANALYZE catalogdb.catwise; -CREATE INDEX ON catalogdb.catwise_reject (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.catwise_reject (q3c_ang2ipix(ra, dec)); CLUSTER catwise_reject_q3c_ang2ipix_idx ON catalogdb.catwise_reject; ANALYZE catalogdb.catwise_reject; diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql index 2decd9ed..7c94d775 100644 --- a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql @@ -6,6 +6,6 @@ ALTER TABLE catalogdb.glimpse ADD COLUMN pk BIGSERIAL PRIMARY KEY; -- Indexes -CREATE INDEX ON catalogdb.glimpse (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.glimpse (q3c_ang2ipix(ra, dec)); CLUSTER glimpse_q3c_ang2ipix_idx ON catalogdb.glimpse; ANALYZE catalogdb.glimpse; diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql index 4a001b90..334dc6e9 100644 --- a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql @@ -1,7 +1,7 @@ -CREATE INDEX ON catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); CLUSTER gaia_unwise_agn_q3c_ang2ipix_idx ON catalogdb.gaia_unwise_agn; ANALYZE catalogdb.gaia_unwise_agn; -CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (g); -CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (prob_rf); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (g); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (prob_rf); diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql index 3750bb7e..0f06ec7f 100644 --- a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql @@ -5,10 +5,10 @@ ALTER TABLE catalogdb.ps1_g18 ADD PRIMARY KEY (objid); -- Indexes -CREATE INDEX ON catalogdb.ps1_g18 (q3c_ang2ipix(raMean, decMean)); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (q3c_ang2ipix(raMean, decMean)); CLUSTER ps1_g18_q3c_ang2ipix_idx ON catalogdb.ps1_g18; ANALYZE catalogdb.ps1_g18; -CREATE INDEX ON catalogdb.ps1_g18 (gmeanpsfmag); -CREATE INDEX ON catalogdb.ps1_g18 (rmeanpsfmag); -CREATE INDEX ON catalogdb.ps1_g18 (imeanpsfmag); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (gmeanpsfmag); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (rmeanpsfmag); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (imeanpsfmag); diff --git a/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql index eeb098f7..cf51f5ba 100644 --- a/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql +++ b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql @@ -127,6 +127,6 @@ CREATE TABLE catalogdb.xmm_om_suss_4_1 ( ALTER TABLE catalogdb.xmm_om_suss_4_1 ADD COLUMN pk BIGSERIAL PRIMARY KEY; -CREATE INDEX on catalogdb.xmm_om_suss_4_1 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.xmm_om_suss_4_1 (q3c_ang2ipix(ra, dec)); CLUSTER xmm_om_suss_4_1_q3c_ang2ipix_idx on catalogdb.xmm_om_suss_4_1; VACUUM ANALYZE catalogdb.xmm_om_suss_4_1; diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql index 3e8d2544..f51cc8ca 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql @@ -17,16 +17,16 @@ ALTER TABLE catalogdb.allwise ADD PRIMARY KEY (cntr); -- Indices -CREATE INDEX ON catalogdb.allwise USING BTREE (designation); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (designation); -CREATE INDEX ON catalogdb.allwise USING BTREE (ra); -CREATE INDEX ON catalogdb.allwise USING BTREE (dec); -CREATE INDEX ON catalogdb.allwise USING BTREE (glat); -CREATE INDEX ON catalogdb.allwise using BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glon); ALTER TABLE catalogdb.allwise ADD CONSTRAINT allwise_designation_unique UNIQUE (designation); -CREATE INDEX ON catalogdb.allwise (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise (q3c_ang2ipix(ra, dec)); CLUSTER allwise_q3c_ang2ipix_idx ON catalogdb.allwise; ANALYZE catalogdb.allwise; diff --git a/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql index 07a85d88..a244d2f2 100644 --- a/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql +++ b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql @@ -48,6 +48,6 @@ CREATE TABLE catalogdb.best_brightest ( ALTER TABLE catalogdb.best_brightest ADD PRIMARY KEY (designation); -CREATE INDEX ON catalogdb.best_brightest (q3c_ang2ipix(ra_1, dec_1)); +CREATE INDEX CONCURRENTLY ON catalogdb.best_brightest (q3c_ang2ipix(ra_1, dec_1)); CLUSTER best_brightest_q3c_ang2ipix_idx ON catalogdb.best_brightest; ANALYZE catalogdb.best_brightest; diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql index 68b0b63c..2c64cbb9 100644 --- a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql @@ -1,10 +1,10 @@ -CREATE INDEX on catalogdb.bhm_csc (q3c_ang2ipix(oir_ra, oir_dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc (q3c_ang2ipix(oir_ra, oir_dec)); CLUSTER bhm_csc_q3c_ang2ipix_idx on catalogdb.bhm_csc; ANALYZE catalogdb.bhm_csc; -CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_g); -CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_r); -CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_i); -CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_z); -CREATE INDEX ON catalogdb.bhm_csc USING BTREE (mag_h); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_g); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_r); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_i); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_z); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_h); diff --git a/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql index d5b573a8..243d00ce 100644 --- a/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql +++ b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql @@ -146,7 +146,7 @@ CREATE TABLE catalogdb.bhm_rm_v0 ( ALTER TABLE catalogdb.bhm_rm_v0 ADD COLUMN pk BIGSERIAL PRIMARY KEY; -CREATE INDEX ON catalogdb.bhm_rm_v0 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (q3c_ang2ipix(ra, dec)); CLUSTER bhm_rm_v0_q3c_ang2ipix_idx ON catalogdb.bhm_rm_v0; VACUUM ANALYZE catalogdb.bhm_rm_v0; diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql index 047f1626..5704ac3f 100644 --- a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -91,12 +91,12 @@ CREATE TABLE catalogdb.bhm_efeds_veto ( -- UPDATE catalogdb.bhm_efeds_veto -- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + -- ((mjd-50000)::bigint<<24) + (1301::bigint<<10); --- CREATE INDEX ON catalogdb.bhm_efeds_veto (specobjid); +-- CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (specobjid); ALTER TABLE catalogdb.bhm_efeds_veto ADD PRIMARY KEY (plate, mjd, fiberid, run2d); -CREATE INDEX ON catalogdb.bhm_efeds_veto (mjd, plate, fiberid, run2d); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (mjd, plate, fiberid, run2d); -CREATE INDEX ON catalogdb.bhm_efeds_veto (q3c_ang2ipix(plug_ra, plug_dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (q3c_ang2ipix(plug_ra, plug_dec)); CLUSTER bhm_efeds_veto_q3c_ang2ipix_idx ON catalogdb.bhm_efeds_veto; ANALYZE catalogdb.bhm_efeds_veto; diff --git a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql index c5e2d165..56ef15c8 100644 --- a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql +++ b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql @@ -104,6 +104,6 @@ CREATE TABLE catalogdb.cataclysmic_variables ( ALTER TABLE catalogdb.cataclysmic_variables ADD PRIMARY KEY (ref_id); -CREATE INDEX ON catalogdb.cataclysmic_variables (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.cataclysmic_variables (q3c_ang2ipix(ra, dec)); CLUSTER cataclysmic_variables_q3c_ang2ipix_idx ON catalogdb.cataclysmic_variables; ANALYZE catalogdb.cataclysmic_variables; diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql index a690f015..a820f399 100644 --- a/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql @@ -1,13 +1,13 @@ -CREATE INDEX on catalogdb.bhm_spiders_agn_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset (q3c_ang2ipix(ero_ra, ero_dec)); CLUSTER bhm_spiders_agn_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_agn_superset; ANALYZE catalogdb.bhm_spiders_agn_superset; -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ero_flux); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (ero_flux); -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); CLUSTER bhm_spiders_clusters_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_clusters_superset; ANALYZE catalogdb.bhm_spiders_clusters_superset; -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ero_flux); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ero_flux); diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql index 9835899f..0bd0b171 100644 --- a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql @@ -6,8 +6,8 @@ To be run after the data has been loaded. ALTER TABLE catalogdb.ebosstarget_v5 ADD COLUMN pk BIGSERIAL PRIMARY KEY; -CREATE INDEX on catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; ANALYZE catalogdb.ebosstarget_v5; -CREATE INDEX ON catalogdb.ebosstarget_v5 USING BTREE (objc_type); +CREATE INDEX CONCURRENTLY ON catalogdb.ebosstarget_v5 USING BTREE (objc_type); diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index d18045ef..f4d380d3 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -24,7 +24,7 @@ ALTER TABLE catalogdb.gaia_dr2_clean -- Some objectids are not present in dr13_photooj. It should not matter -- because the TIC has x-matching with SDSS. --- CREATE INDEX ON catalogdb.gaiadr2_sdssdr9_best_neighbour (sdssdr9_oid); +-- CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_sdssdr9_best_neighbour (sdssdr9_oid); -- ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour -- ADD CONSTRAINT source_id_fk @@ -41,7 +41,7 @@ ALTER TABLE catalogdb.gaia_dr2_clean -- gaiadr2_tmass_best_neighbour -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk @@ -58,7 +58,7 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour -- gaiadr2_tmass_best_neighbour -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk @@ -76,7 +76,7 @@ ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour -- sdss_dr14_specobj UPDATE catalogdb.sdss_dr14_specobj SET bestobjid = NULL WHERE bestobjid = 0; -CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); -- Cannot be created because some bestobjids are not present in dr13_photoobj. -- ALTER TABLE catalogdb.sdss_dr14_specobj @@ -88,7 +88,7 @@ CREATE INDEX ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); -- sdss_dr16_specobj UPDATE catalogdb.sdss_dr16_specobj SET bestobjid = NULL WHERE bestobjid = 0; -CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); -- Cannot be created because some bestobjids are not present in dr13_photoobj. -- ALTER TABLE catalogdb.sdss_dr16_specobj @@ -100,7 +100,7 @@ CREATE INDEX ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); -- gaia_unwise_agn -CREATE INDEX ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); ALTER TABLE catalogdb.gaia_unwise_agn ADD CONSTRAINT source_id_fk @@ -141,12 +141,12 @@ ALTER TABLE catalogdb.sdss_dr14_ascapStar -- tic_v8 -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (sdss); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (tyc); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (twomass_psc); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (kic); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (allwise); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gaia_int); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (sdss); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (tyc); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (twomass_psc); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (kic); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (allwise); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gaia_int); ALTER TABLE catalogdb.tic_v8 ADD CONSTRAINT sdss_fk @@ -187,9 +187,9 @@ ALTER TABLE catalogdb.tic_v8 -- bhm_spiders_agn_superset, bhm_spiders_clusters_superset -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); UPDATE catalogdb.bhm_spiders_agn_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; -CREATE INDEX ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); ALTER TABLE catalogdb.bhm_spiders_agn_superset ADD CONSTRAINT gaia_dr2_source_id_fk @@ -203,8 +203,8 @@ ALTER TABLE catalogdb.bhm_spiders_agn_superset REFERENCES catalogdb.legacy_survey_dr8 (ls_id) ON UPDATE CASCADE ON DELETE CASCADE; -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); -CREATE INDEX ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); UPDATE catalogdb.bhm_spiders_clusters_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; ALTER TABLE catalogdb.bhm_spiders_clusters_superset @@ -222,9 +222,9 @@ ALTER TABLE catalogdb.bhm_spiders_clusters_superset -- skymapper_dr1_1 -CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (allwise_cntr); -CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id1); -CREATE INDEX ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id2); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (allwise_cntr); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id1); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id2); ALTER TABLE catalogdb.skymapper_dr1_1 ADD CONSTRAINT allwise_cntr_fk @@ -246,7 +246,7 @@ ALTER TABLE catalogdb.skymapper_dr1_1 -- glimpse -CREATE INDEX ON catalogdb.glimpse USING BTREE (tmass_cntr); +CREATE INDEX CONCURRENTLY ON catalogdb.glimpse USING BTREE (tmass_cntr); ALTER TABLE catalogdb.glimpse ADD CONSTRAINT tmass_cntr_fk diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql index 56bc42c5..b61d5c85 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql @@ -11,11 +11,11 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_flux); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_mag); -CREATE INDEX ON catalogdb.gaia_dr2_source USING BTREE (solution_id); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_flux); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_mag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (solution_id); -CREATE INDEX ON catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); CLUSTER gaia_dr2_source_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_source; ANALYZE catalogdb.gaia_dr2_source; diff --git a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql index bc673b82..853e9910 100644 --- a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql +++ b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql @@ -51,15 +51,15 @@ ALTER TABLE catalogdb.gaia_qso ADD COLUMN pk BIGSERIAL; ALTER TABLE catalogdb.gaia_qso ADD PRIMARY KEY (pk); -CREATE INDEX ON catalogdb.gaia_qso (q3c_ang2ipix(raj2000, dej2000)); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso (q3c_ang2ipix(raj2000, dej2000)); CLUSTER gaia_qso_q3c_ang2ipix_idx ON catalogdb.gaia_qso; ANALYZE catalogdb.gaia_qso; -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (umag); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (gmag); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (rmag); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (imag); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (zmag); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w1_w2); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w2_w3); -CREATE INDEX ON catalogdb.gaia_qso USING BTREE (w1mag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w1_w2); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w2_w3); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w1mag); diff --git a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql index 2e3418bd..c443fab0 100644 --- a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql +++ b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql @@ -12,6 +12,6 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.guvcat (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.guvcat (q3c_ang2ipix(ra, dec)); CLUSTER guvcat_q3c_ang2ipix_idx ON catalogdb.guvcat; ANALYZE catalogdb.guvcat; diff --git a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql index fae7e2b7..b92438e7 100644 --- a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql +++ b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql @@ -11,20 +11,20 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.kepler_input_10 (q3c_ang2ipix(kic_ra, kic_dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 (q3c_ang2ipix(kic_ra, kic_dec)); CLUSTER kepler_input_10_q3c_ang2ipix_idx ON catalogdb.kepler_input_10; ANALYZE catalogdb.kepler_input_10; -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_ra); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_dec); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_umag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_gmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_rmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_imag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_zmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_jmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_hmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_kmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_kepmag); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_glon); -CREATE INDEX ON catalogdb.kepler_input_10 USING BTREE (kic_glat); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_ra); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_dec); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_umag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_imag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_kmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_kepmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_glon); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_glat); diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql index df9b362d..3b652f56 100644 --- a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -1,15 +1,15 @@ ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (ls_id); -CREATE INDEX ON catalogdb.legacy_survey_dr8 (ref_id); -CREATE INDEX ON catalogdb.legacy_survey_dr8 (ref_cat); +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (ref_id); +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (ref_cat); -- Create new column with the source_id for Gaia where ref_cat=G2 ALTER TABLE catalogdb.legacy_survey_dr8 ADD COLUMN gaia_sourceid BIGINT; UPDATE catalogdb.legacy_survey_dr8 SET gaia_sourceid = ref_id WHERE ref_cat = 'G2'; -CREATE INDEX ON catalogdb.legacy_survey_dr8 (gaia_sourceid); +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (gaia_sourceid); -CREATE INDEX ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); CLUSTER legacy_survey_dr8_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; VACUUM ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql index 96d155eb..e9b2df0b 100644 --- a/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql +++ b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql @@ -72,6 +72,6 @@ CREATE TABLE catalogdb.mipsgal ( f2 INTEGER ); -CREATE INDEX ON catalogdb.mipsgal (q3c_ang2ipix(radeg, dedeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal (q3c_ang2ipix(radeg, dedeg)); CLUSTER mipsgal_q3c_ang2ipix_idx ON catalogdb.mipsgal; ANALYZE catalogdb.mipsgal; diff --git a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql index 51990c2a..29700293 100644 --- a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql +++ b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql @@ -19,11 +19,11 @@ CREATE TABLE catalogdb.atnf ( gaia_source_id BIGINT ); -CREATE INDEX ON catalogdb.atnf (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.atnf (q3c_ang2ipix(radeg, decdeg)); CLUSTER atnf_q3c_ang2ipix_idx ON catalogdb.atnf; ANALYZE catalogdb.atnf; -CREATE INDEX ON catalogdb.atnf USING BTREE (gaia_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.atnf USING BTREE (gaia_source_id); ALTER TABLE catalogdb.atnf ADD CONSTRAINT gaia_source_id_fk diff --git a/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql index bc931a15..206a3c82 100644 --- a/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql +++ b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.blackcat ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.blackcat (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.blackcat (q3c_ang2ipix(radeg, decdeg)); CLUSTER blackcat_q3c_ang2ipix_idx ON catalogdb.blackcat; ANALYZE catalogdb.blackcat; diff --git a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql index 406eaa95..859db3a9 100644 --- a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql +++ b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.galactic_millisecond_pulsars ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.galactic_millisecond_pulsars (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.galactic_millisecond_pulsars (q3c_ang2ipix(radeg, decdeg)); CLUSTER galactic_millisecond_pulsars_q3c_ang2ipix_idx ON catalogdb.galactic_millisecond_pulsars; ANALYZE catalogdb.galactic_millisecond_pulsars; diff --git a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql index c1e4f819..35154d07 100644 --- a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql +++ b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.lmxb ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.lmxb (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.lmxb (q3c_ang2ipix(radeg, decdeg)); CLUSTER lmxb_q3c_ang2ipix_idx ON catalogdb.lmxb; ANALYZE catalogdb.lmxb; diff --git a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql index f72ae0f7..95481382 100644 --- a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql +++ b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.transitional_msps ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.transitional_msps (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.transitional_msps (q3c_ang2ipix(radeg, decdeg)); CLUSTER transitional_msps_q3c_ang2ipix_idx ON catalogdb.transitional_msps; ANALYZE catalogdb.transitional_msps; diff --git a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql index 7b665e1e..0492c647 100644 --- a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql +++ b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.watchdog ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.watchdog (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.watchdog (q3c_ang2ipix(radeg, decdeg)); CLUSTER watchdog_q3c_ang2ipix_idx ON catalogdb.watchdog; ANALYZE catalogdb.watchdog; diff --git a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql index 59c69fdf..84b362bb 100644 --- a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql +++ b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql @@ -19,7 +19,7 @@ CREATE TABLE catalogdb.xray_pulsars ( gaia_source_id BIGINT PRIMARY KEY ); -CREATE INDEX ON catalogdb.xray_pulsars (q3c_ang2ipix(radeg, decdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.xray_pulsars (q3c_ang2ipix(radeg, decdeg)); CLUSTER xray_pulsars_q3c_ang2ipix_idx ON catalogdb.xray_pulsars; ANALYZE catalogdb.xray_pulsars; diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql index b396ac62..58b300ba 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql @@ -1,5 +1,5 @@ -CREATE INDEX ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(ra, dec)); CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; @@ -7,4 +7,4 @@ CREATE INDEX sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_ CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; -CREATE INDEX ON catalogdb.sdss_apogeeAllStarMerge_r13 USING BTREE (apogee_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_apogeeAllStarMerge_r13 USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql index 05b721b0..56793980 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql @@ -112,13 +112,13 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStar( \copy catalogdb.sdss_dr14_apogeeStar FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeStar/dr14/src/sqlApogeeStar.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_apogeeStar_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeStar; ANALYZE catalogdb.sdss_dr14_apogeeStar; -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (ra); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (dec); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glon); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glat); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStar USING BTREE (apogee_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql index 7936a106..c83a3b1d 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql @@ -22,5 +22,5 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStarVisit( ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit ADD COLUMN pk BIGSERIAL PRIMARY KEY; -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (apstar_id); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (visit_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (apstar_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (visit_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql index f623875b..52b10a37 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql @@ -115,11 +115,11 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeVisit( \copy catalogdb.sdss_dr14_apogeeVisit FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeVisit/dr14/src/sqlApogeeVisit.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_apogeeVisit_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeVisit; ANALYZE catalogdb.sdss_dr14_apogeeVisit; -CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (ra); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (dec); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glon); -CREATE INDEX ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glat); diff --git a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql index 5816248c..5e9c824a 100644 --- a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql +++ b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql @@ -333,5 +333,5 @@ CREATE TABLE catalogdb.sdss_dr14_ascapStar( \copy catalogdb.sdss_dr14_ascapStar FROM program 'bzcat $CATALOGDB_DIR/sdssAscapStar/dr14/src/sqlaspcapStar.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr14_ascapStar USING BTREE (target_id); -CREATE INDEX ON catalogdb.sdss_dr14_ascapStar USING BTREE (apogee_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_ascapStar USING BTREE (target_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_ascapStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql index 64f0cc5c..56b21882 100644 --- a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql +++ b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql @@ -171,4 +171,4 @@ CREATE TABLE catalogdb.sdss_dr14_cannonStar( \copy catalogdb.sdss_dr14_cannonStar FROM program 'bzcat $CATALOGDB_DIR/sdssCannonStar/dr14/src/sqlcannonStar.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr14_cannonStar USING BTREE (apogee_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_cannonStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql index 0c9af804..6e05baa0 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql @@ -5,12 +5,12 @@ indices for catalogdb tables, to be run after bulk uploads -- Indices -CREATE INDEX ON catalogdb.sdss_dr13_photoobj (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr13_photoobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr13_photoobj; VACUUM ANALYZE catalogdb.sdss_dr13_photoobj; -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_u); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_g); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_r); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_i); -CREATE INDEX ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_z); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_u); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_g); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_r); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_i); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_z); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql index 5beadcf4..54f481e7 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql @@ -404,11 +404,11 @@ CREATE TABLE catalogdb.sdss_dr14_specobj( \copy catalogdb.sdss_dr14_specobj FROM program 'bzcat $CATALOGDB_DIR/sdssSpecObj/dr14/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr14_specobj (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_specobj; ANALYZE catalogdb.sdss_dr14_specobj; -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); -CREATE INDEX ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); -CREATE INDEX ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid); -CREATE INDEX ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid, run2d); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql index 111562d4..20e072ee 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -404,12 +404,12 @@ CREATE TABLE catalogdb.sdss_dr16_specobj( \copy catalogdb.sdss_dr16_specobj FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj/dr16/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; -CREATE INDEX ON catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr16_specobj; ANALYZE catalogdb.sdss_dr16_specobj; -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (fluxObjID); -CREATE INDEX ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj USING BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); -CREATE INDEX ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid); -CREATE INDEX ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid, run2d); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql index e83fe6d2..134edfe2 100644 --- a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql @@ -6,13 +6,13 @@ ALTER TABLE catalogdb.sdss_dr14_qso ADD PRIMARY KEY (pk); -- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + -- ((mjd-50000)::bigint<<24) + (26::bigint<<10); -CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (plate); -CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (mjd); -CREATE INDEX ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); -CREATE INDEX ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (plate); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (mjd); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); --- CREATE INDEX ON catalogdb.sdss_dr14_qso (specobjid); +-- CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (specobjid); -CREATE INDEX ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); CLUSTER sdss_dr14_qso_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_qso; ANALYZE catalogdb.sdss_dr14_qso; diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql index 88b43505..b3b31d5e 100644 --- a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql @@ -9,6 +9,6 @@ ALTER TABLE catalogdb.skymapper_dr1_1 USING INDEX object_id_pk; -- blocks queries, but only very briefly -CREATE INDEX ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, dej2000)); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, dej2000)); CLUSTER skymapper_dr1_1_q3c_ang2ipix_idx ON catalogdb.skymapper_dr1_1; ANALYZE catalogdb.skymapper_dr1_1; diff --git a/schema/sdss5db/catalogdb/smgaia/smgaia.sql b/schema/sdss5db/catalogdb/smgaia/smgaia.sql index a3112e40..4939a120 100644 --- a/schema/sdss5db/catalogdb/smgaia/smgaia.sql +++ b/schema/sdss5db/catalogdb/smgaia/smgaia.sql @@ -20,9 +20,9 @@ CREATE TABLE catalogdb.skymapper_gaia ( \COPY catalogdb.skymapper_gaia FROM PROGRAM 'zcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/SMGaia/SMGaia.csv.gz' WITH CSV HEADER DELIMITER ','; -CREATE INDEX ON catalogdb.skymapper_gaia (gaia_source_id); -CREATE INDEX ON catalogdb.skymapper_gaia (teff); -CREATE INDEX ON catalogdb.skymapper_gaia (feh); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (gaia_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (teff); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (feh); ALTER TABLE catalogdb.skymapper_gaia diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql index ca98b3fd..f98e312f 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -7,21 +7,21 @@ UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; -- Indices -CREATE INDEX on catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); CLUSTER tic_v8_q3c_ang2ipix_idx ON catalogdb.tic_v8; VACUUM ANALYZE catalogdb.tic_v8; -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Bmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Vmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (umag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (gmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (rmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (imag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (zmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Jmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Hmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (Kmag); -CREATE INDEX ON catalogdb.tic_v8 USING BTREE (posflag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Bmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Vmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Kmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (posflag); ALTER TABLE catalogdb.tic_v8 ADD COLUMN twomass_psc TEXT; UPDATE catalogdb.tic_v8 diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql index 7df0d46f..0d9f9985 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql @@ -10,20 +10,20 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (j_m); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (h_m); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (k_m); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (j_m); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (h_m); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (k_m); -CREATE INDEX ON catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); CLUSTER twomass_psc_q3c_ang2ipix_idx ON catalogdb.twomass_psc; ANALYZE catalogdb.twomass_psc; ALTER TABLE catalogdb.twomass_psc ADD CONSTRAINT catalogdb_twomass_psc_designation_unique UNIQUE (designation); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (designation); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (designation); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (ph_qual); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (cc_flg); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (gal_contam); -CREATE INDEX ON catalogdb.twomass_psc USING BTREE (rd_flg); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (ph_qual); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (cc_flg); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (gal_contam); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (rd_flg); diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql index 47f79aaa..ded2be37 100644 --- a/schema/sdss5db/catalogdb/tycho2/tycho2.sql +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -390,6 +390,6 @@ ALTER TABLE catalogdb.tycho2 DROP COLUMN pk; ALTER TABLE catalogdb.tycho2 ADD PRIMARY KEY (designation); -CREATE INDEX on catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; VACUUM ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql index b2224225..c2946f75 100644 --- a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql @@ -1,4 +1,4 @@ -CREATE INDEX ON catalogdb.unwise (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.unwise (q3c_ang2ipix(ra, dec)); CLUSTER unwise_q3c_ang2ipix_idx ON catalogdb.unwise; ANALYZE catalogdb.unwise; diff --git a/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql index 3f104fe2..6f19bad9 100644 --- a/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql +++ b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql @@ -377,6 +377,6 @@ CREATE TABLE catalogdb.uvotssc1 ( ALTER TABLE catalogdb.uvotssc1 ADD id BIGSERIAL PRIMARY KEY; -CREATE INDEX ON catalogdb.uvotssc1 (q3c_ang2ipix(radeg, dedeg)); +CREATE INDEX CONCURRENTLY ON catalogdb.uvotssc1 (q3c_ang2ipix(radeg, dedeg)); CLUSTER uvotssc1_q3c_ang2ipix_idx ON catalogdb.uvotssc1; VACUUM ANALYZE catalogdb.uvotssc1; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql index 42a4e865..ec97b18f 100644 --- a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -1,15 +1,15 @@ CREATE UNIQUE INDEX gaia_dr2_wd_wd_key ON catalogdb.gaia_dr2_wd(wd text_ops); -CREATE INDEX ON catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); CLUSTER gaia_dr2_wd_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd; ANALYZE catalogdb.gaia_dr2_wd; -CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); CLUSTER gaia_dr2_wd_sdss_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd_sdss; ANALYZE catalogdb.gaia_dr2_wd_sdss; -CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (plate); -CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (mjd); -CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss USING BTREE (fiber); -CREATE INDEX ON catalogdb.gaia_dr2_wd_sdss (mjd, plate, fiber); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (plate); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (mjd); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (fiber); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss (mjd, plate, fiber); diff --git a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql index 1c825ee1..67373d83 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql @@ -22,5 +22,5 @@ CREATE TABLE catalogdb.gaiadr2_tmass_best_neighbour( ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD PRIMARY KEY (source_id); -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (tmass_oid); -CREATE INDEX ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (original_ext_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (tmass_oid); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (original_ext_source_id); diff --git a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql index 3257a1d7..af3bc9d3 100644 --- a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql +++ b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql @@ -25,6 +25,6 @@ CREATE TABLE catalogdb.yso_clustering ( edist DOUBLE PRECISION ); -CREATE INDEX ON catalogdb.yso_clustering (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON catalogdb.yso_clustering (q3c_ang2ipix(ra, dec)); CLUSTER yso_clustering_q3c_ang2ipix_idx ON catalogdb.yso_clustering; ANALYZE catalogdb.yso_clustering; diff --git a/schema/sdss5db/targetdb/targetdb.sql b/schema/sdss5db/targetdb/targetdb.sql index ace525e3..12ad8373 100644 --- a/schema/sdss5db/targetdb/targetdb.sql +++ b/schema/sdss5db/targetdb/targetdb.sql @@ -229,7 +229,7 @@ CREATE INDEX CONCURRENTLY magnitude_pk_idx ON targetdb.target using BTREE(magnit CREATE INDEX CONCURRENTLY catalogid_idx ON targetdb.target using BTREE(catalogid); -- This doesn't seem to be loaded unless it's run manually inside a PSQL console. -CREATE INDEX ON targetdb.target (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON targetdb.target (q3c_ang2ipix(ra, dec)); CLUSTER target_q3c_ang2ipix_idx on targetdb.target; ANALYZE targetdb.target; From c80a6d741a71579b5941424b3d1b87e860afb0db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 09:28:45 -0700 Subject: [PATCH 192/197] Allow schema in is_table_locked --- python/sdssdb/utils/internals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/utils/internals.py b/python/sdssdb/utils/internals.py index cd70b610..2d28f87c 100644 --- a/python/sdssdb/utils/internals.py +++ b/python/sdssdb/utils/internals.py @@ -173,11 +173,16 @@ def get_row_count(connection, table_name, schema=None, approximate=True): return count[0][0] -def is_table_locked(connection, table_name): +def is_table_locked(connection, table_name, schema=None): """Returns the locks for a table or `None` if no lock is present.""" - sql = ('SELECT mode FROM pg_locks JOIN pg_class ON pg_class.oid = pg_locks.relation ' - f'WHERE pg_class.relname = {table_name!r}') + schema = schema or '%%' + + sql = ('SELECT mode FROM pg_locks JOIN pg_class ' + 'ON pg_class.oid = pg_locks.relation ' + 'JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace ' + f'WHERE pg_class.relname = {table_name!r} ' + f'AND pg_namespace.nspname LIKE {schema!r};') with connection.atomic(): locks = connection.execute_sql(sql).fetchall() From d17adee180ac6203d1fdbe1909a357c27ee15787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 10:04:20 -0700 Subject: [PATCH 193/197] Tweak ReflectMeta docstring --- python/sdssdb/peewee/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 40fa3fe2..b11e03ce 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -93,9 +93,13 @@ class Meta: and the foreign key need to be defined explicitely. Otherwise neither will be added. - - `.ReflectMeta` is designed to fail silently if PeeWee's reflection fails. - Error are logged with debug level and can be display by setting - doing ``from sdssdb import log`` and ``log.set_level(0)``. + - Reflection will fail if a table is locked with ``AccessExclusiveLock``. + In that case reflection will be skipped and a warning issued. Note that if + the table is locked with an exclusive lock you won't be able to access the + data in any case. + + - In this version, indexes discovered by reflection are not propagated to + the model class. This should not have any impact in performance. """ From 169cab9e4ba93a9ff8d5b2b7edbbb0a47f35c3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 11:52:50 -0700 Subject: [PATCH 194/197] Fix targetdb to work with new base class --- python/sdssdb/peewee/sdss5db/targetdb.py | 57 ++++++++++-------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/targetdb.py b/python/sdssdb/peewee/sdss5db/targetdb.py index 52c9497c..a839eb79 100644 --- a/python/sdssdb/peewee/sdss5db/targetdb.py +++ b/python/sdssdb/peewee/sdss5db/targetdb.py @@ -11,14 +11,21 @@ ManyToManyField, SmallIntegerField, TextField) from playhouse.postgres_ext import ArrayField -from . import SDSS5dbModel, catalogdb, database # noqa +from . import BaseModel, database # noqa + + +class TargetdbBase(BaseModel): + + class Meta: + schema = 'targetdb' + database = database AssignmentDeferred = DeferredThroughModel() ProgramToTargetDeferred = DeferredThroughModel() -class Cadence(SDSS5dbModel): +class Cadence(TargetdbBase): delta = ArrayField(field_class=FloatField, null=True) delta_max = ArrayField(field_class=FloatField, null=True) delta_min = ArrayField(field_class=FloatField, null=True) @@ -30,19 +37,17 @@ class Cadence(SDSS5dbModel): class Meta: table_name = 'cadence' - schema = 'targetdb' -class Observatory(SDSS5dbModel): +class Observatory(TargetdbBase): label = TextField() pk = AutoField() class Meta: table_name = 'observatory' - schema = 'targetdb' -class Field(SDSS5dbModel): +class Field(TargetdbBase): cadence = ForeignKeyField(column_name='cadence_pk', field='pk', model=Cadence, @@ -58,10 +63,9 @@ class Field(SDSS5dbModel): class Meta: table_name = 'field' - schema = 'targetdb' -class Design(SDSS5dbModel): +class Design(TargetdbBase): field = ForeignKeyField(column_name='field_pk', field='pk', model=Field, @@ -71,28 +75,25 @@ class Design(SDSS5dbModel): class Meta: table_name = 'design' - schema = 'targetdb' -class Instrument(SDSS5dbModel): +class Instrument(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'instrument' - schema = 'targetdb' -class PositionerStatus(SDSS5dbModel): +class PositionerStatus(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'positioner_status' - schema = 'targetdb' -class PositionerInfo(SDSS5dbModel): +class PositionerInfo(TargetdbBase): apogee = BooleanField(null=False) boss = BooleanField(null=False) fiducial = BooleanField(null=False) @@ -100,10 +101,9 @@ class PositionerInfo(SDSS5dbModel): class Meta: table_name = 'positioner_info' - schema = 'targetdb' -class Positioner(SDSS5dbModel): +class Positioner(TargetdbBase): id = IntegerField(null=True) observatory = ForeignKeyField(column_name='observatory_pk', field='pk', @@ -120,10 +120,9 @@ class Positioner(SDSS5dbModel): class Meta: table_name = 'positioner' - schema = 'targetdb' -class Magnitude(SDSS5dbModel): +class Magnitude(TargetdbBase): bp = FloatField(null=True) g = FloatField(null=True) h = FloatField(null=True) @@ -134,37 +133,33 @@ class Magnitude(SDSS5dbModel): class Meta: table_name = 'magnitude' - schema = 'targetdb' -class Version(SDSS5dbModel): +class Version(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'version' - schema = 'targetdb' -class Category(SDSS5dbModel): +class Category(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'category' - schema = 'targetdb' -class Survey(SDSS5dbModel): +class Survey(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'survey' - schema = 'targetdb' -class Program(SDSS5dbModel): +class Program(TargetdbBase): category = ForeignKeyField(column_name='category_pk', field='pk', model=Category, @@ -178,10 +173,9 @@ class Program(SDSS5dbModel): class Meta: table_name = 'program' - schema = 'targetdb' -class Target(SDSS5dbModel): +class Target(TargetdbBase): catalogid = BigIntegerField(null=True) dec = DoubleField(null=True) epoch = FloatField(null=True) @@ -215,10 +209,9 @@ class Target(SDSS5dbModel): class Meta: table_name = 'target' - schema = 'targetdb' -class Assignment(SDSS5dbModel): +class Assignment(TargetdbBase): design = ForeignKeyField(Design, column_name='design_pk', field='pk') @@ -235,10 +228,9 @@ class Assignment(SDSS5dbModel): class Meta: table_name = 'assignment' - schema = 'targetdb' -class ProgramToTarget(SDSS5dbModel): +class ProgramToTarget(TargetdbBase): cadence = ForeignKeyField(Cadence, column_name='cadence_pk', field='pk') @@ -259,7 +251,6 @@ class ProgramToTarget(SDSS5dbModel): class Meta: table_name = 'program_to_target' - schema = 'targetdb' AssignmentDeferred.set_model(Assignment) From 991e8bbd3fe0e413acb40bbd1786093765857f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 11:54:30 -0700 Subject: [PATCH 195/197] Update schemadisplay default config --- python/sdssdb/utils/schemadisplay.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index f8ecf8b1..f0a3e3ba 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -14,7 +14,7 @@ import re -from peewee import ForeignKeyField +from peewee import ForeignKeyField, IndexMetadata try: @@ -155,6 +155,8 @@ def format_field_str(field): first = True for index in indexes: + if not isinstance(index, IndexMetadata): + continue column_names = index.columns ilabel = 'INDEX' @@ -259,8 +261,9 @@ def create_schema_graph(models=None, base=None, schema=None, show_columns=True, if schema: models = [model for model in models if model._meta.schema == schema] - default_graph_options = dict(rankdir='TB', - sep='0.1', + default_graph_options = dict(program='dot', + rankdir='TB', + sep='0.01', mode='ipsep', overlap='ipsep') default_graph_options.update(graph_options) From f56360a6847ae5a7b2b969b6a79f47d60acf7860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 12:14:52 -0700 Subject: [PATCH 196/197] Document ReflectMeta and changes to how schemas are generated --- .../schema_graphs/sdss5db.catalogdb.pdf | Bin 0 -> 113227 bytes .../schema_graphs/sdss5db.catalogdb_lite.pdf | Bin 0 -> 43288 bytes .../schema_graphs/sdss5db.targetdb.pdf | Bin 0 -> 28598 bytes docs/sphinx/api.rst | 4 +++ docs/sphinx/conf.py | 6 ++-- docs/sphinx/contributing.rst | 30 ++++++++++++++++-- docs/sphinx/intro.rst | 11 ++++--- python/sdssdb/peewee/__init__.py | 30 +++++++++--------- 8 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf create mode 100644 docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf create mode 100644 docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf new file mode 100644 index 0000000000000000000000000000000000000000..312cc1ead19f9917f289d09199a903e21ef7e77b GIT binary patch literal 113227 zcmV)AK*YZ#P((&8F)lL-CB)_O4?5av(28Y+-a|L}g=dWMv>eJ_>Vma%Ev{3U~qBecO^F$8q*Oo}xonU|3$G z8;z6iBri?TmPlE zo==sP`EB=4-M8J}{M~ckJv?{2>3`1;^in_l+IQp5{rB+L{rGivnqE6~$Mf;UuHWtH zm&@+OrF;F@{mtvw{ig3;|7Z8F-S+=|((PW1-S)rS&waOjR%xs+%JUc zGv5EB&m0Q$_Algf_|q%G&VLZkyOel%=P zSq_-9kluG6F(-m;y?STaOLNLEq^C}6fgAZb0(BLh*wF-B(6Autph;g&U*i;1)r+v{Qzk6f> z+_)ff61WG7hP_w;$VWgaZ^CYo)UWwK-uvr`+^M8S5rbN!`!?Qpai0q+`l7L`= z$l&;ozHYX)$1}P5yu=IN0p%0GOq7*z+H4=->G82=3#k@;&9LA^9zM^4EcBa@+Q@sr z5u|28exSVJCHdHQh8>y=gLntco9PdqN($#PxU6T);CoC%0Hxjn6v!^zD&Y$oi%*3F z{29amw@H`mIQc#Zk17;MBT=fXc)|T!c)9o~0tW$+F0HA;-N+sbh#$ z;{V(W?0^(@_i0G}1}qrm&Gx_LJuq|lyn)yxdXWL?IE2}R`pWBqNM`UIl;=-_H6*O? zd;gojE#;bSt4P6y@+l=#Op0!q6q{{qOaJ+k?zDGxh#$cRAe2Sr%Ka_NenUh1^X1YV zkG;z8V+oi_d$G(Su>%JUv>$*s6cumBFL)B(3>AUAKvy0_5kU4>ltLmF(=Q`Rs#G@@ zrJCU7xGM%P&y3|}`y4D3Fb@rO3gT4Jj?9=CA|e(I}<8Ho-fbieXj-;t+vH zVmTYfoT$R2)zkntIL7cP#Nk+ssY;+SSOHX%Tb`D&L5X1ARGvdH%!fi;wAq$mjspk) zj}L`RH798i?6qeo4Evr!oO}chFaTI$wIA^wrnd_B5QrL1LEH2vRr?W5kA8 zB((7-Lnvwh=mx}4g5GS4h{2D&g|B>q$&9d0V}tA1Ng?D1VrN~Ey)~dc>u=RgA-Ci&s*MSc6`tvRO;?N$d`7bXP=n-xKxeZ{CwRw`leK=B_;TsS^RcUHSUC{1XG%o7 zLw`8E7&#K{!5qLuG@@Rl&~Zd$pZ@(I%=#|wJqmqGZ>RTAo02@_yRV3@Q}5Fg|2!9J zEhKpx)o^v7bN|ET;2?I7NSlv-DjL+msWKMRTz@GnfLO_jPV95iNTGAbF2xcZbt=8ttbr}Fjl0@WKKTT-%)!? z7J0HckSZt$UV&7FnzJ~VxF8{Y3@ZP`ZdJA6+aGghtpYsRgt@_#>L99NCb6^6oBNcZR0v|J+vr>mx${n zSOAWQd)?`)z)gK01#uA5_Z>*7LhR1T}OuQ;QQ z)ZkITDzFHi^lMQ@RSPquXL3~taLV<*3~1EGQ2-4Q31pbslQ_hH7OsbJp-;(6_;{h7 zjb#|Ag#oAH0PXo(F&+l>+T8bep{|Pp7@!dB)!W;ao`o9JK2b0~R*?#kg+YWVBCZr6 zQ)kBu^*4YnsR6P8S}FtjM(D;6W4ut`0RV}O$7X8*vQzc}>gYP?W8K^BnC1mu53 zt8jE~^Qx;Ytz{siRDLaFLqV0UHbf)cOIZ!s{=SgW3HMr>jdz9YOefKekXeoeUT48h zB(hgo%x1&oE*)i|wbc`LT)7xPn=vA5i_Zcj9r^;wI%`&hk}b?E1#Zz6+vc9>ytmN_ z6rp@i=I3mdeh}VS7P=^~XiKWKO^p?8kqJ8Og-ozxQ59_)Y!0d|$pzSKxbhu4S2^Z0 zo#oy|NbYRaD7B2Kt@_MuGx-IaSW9L&aeh(9ucZiXoyMQ&;5Fux z()f0q7*F7&PW^y8WDpWw8w{>QlFnH_0wnKiyH9lX`Vk;m8=|h{Bptba1W4BUg_TIs zk!%4;s1X+qRhcs!wG<*;ZxKJ|8rT;SxWDEX>TUqoR{u1l@q_YcWyskYD%1|`I1Xo3k#{>E0t6@}D@rDsCb!6NCHf8^A z3EQ5$;MH`4ndp~vtcl5Sx^wJ{j;|z$o6uGOU>9wg32e3b-4%VF12#VS2*(T18UVWx z@m*P_I$TteoR+CdlvNEI09M(lJLvj>W-S$DR{BUn`u<>*RPJk!4yU5$s)X@A_ULdV zmz3A}O4#mWj}C{Tuc?F$f_PthbQsfQQ2F)(u$yh@a0jsWwMU12n$#u0Qs)f%HB7Am zunYFc75CY^mCupw@cS^#lC${UNP)7QMcF&sqaC@WCVi4o7~S?c%)i0mIF7`XiSkL> z%8@UScEQ`cuYa*e#6D^88G zz|+uH{O$RhocP?E@-&OIP)!hx9v@gd8+!FK*Zl8}H#b}UMTUn|wz1oG zbAK#4N&uK(l;H<@4+bGYuId^v>arL$O>__eJ7oN+7g;p&K8xE5C&_<8+aKgB;}C+J zWxW;!NsC@Wi&sL_-Hle=7QDLWvbrF`^zNH4KN% zsHvdWH+uKSqIYiP4CDM3{D9EG8{ki6R^2cZ)FTXx)}7noI7swWDRFK08~TylcQV(2 z%M=uZTdKc8;x=2LQOT-;d299RJ&h2#7DtHOI|Knhgw5NHZ%gz;F=wvgjD|XP?N^z% zl@_~Ci`T6dTcgB+$c#`IY+)BFbtX4Q&vuvnu6Q78a)zl7=<$moPxu?qfT*!~+iX{P zB6Hu8jpOceeMkU2N;twU%vlvYQ)LWn=ZU*>F=01Z6`MDWMyy2m8_*$(f;gYilj9D^t096G#-|E3_TB1>cKW`A+hdvalU<0QW#S-jW* zKSdTc6jfxg{0V4JgU<6M=n3yLRM}99W&tZr6;f;j({ChT@VX#Bh+IKZeU1vA z+(**N1C8+IF@0278i)Z^O93NEQ}l_0stWqp^C*5Nf)-}$trj^3YDHV&{^DjW1?7#~ z3bQ2bj4_-sN!1k&@sl9`6W)xM%A^z(2X6sdv#3?{F6}RR=u_NV2cxD0Ypp7ZsEOHW zr3?F=B6yHaW--{)Q(RJKx^Pc+=VD z47gDc56_Ukkq~XFlL9V8UZaWg8CF+mB1*-)Xhv<=!MPmpN{33q@BTH`ptsxXifJFDLb_;?Efe8aE9)nngv87P-VEsYHR8PH~uj&6P z4`!v(R@xo8Kf94?A<76Xo}N=CZj*|sjTXQM1SoKx^;1Vgm7Nazv!1q=dVYpgdH!&A z3lQ;3fEA_!c2~-xbb6QaLqym>Bt0oL>xuibo7R?BbP(OG&f%|FfMZo9eSbEFwJCFL z!I38x?XjN&c`K{v=C;)k3pxZRnVFDz-TlM;S&wT=txuqu0nh^ZysoFg%Qdp*M!4n4 z7p(R3H}IlR1aY)#8>RAe`S%KQ0%Cp@BsP95i384?@o0h2r z~YjM6&vUM%=G=~d}*JVPM@+2a6;FYLSVOHz{B@u`>ni)3vMBrhUGxq z47Z3=2DFAiX2JP)Z`AF&A2L&gg!1zTO2XNrKm6d$CyPVollkC#w<{cO2s%}!f;Yq| zqagznK8Jv@$Uy_3q@dv%@2>=isXO&T7U7{aFx~!ipp6xWeb>`9jKUcH6CH=1_D9;K z;4X2WhK^P%&{(R0W)5~o+VJ?|>=rFhli-2Uy^ii9_b-lzGrfK92K^sS$L^s!x+^ui z^L}#}U5_;h@{ukz()K%g$DU&P;V^bbx;fZi4i{PuKu1OW`9Pn0wdnEO(&7E?=)RB6 zJI|*fF-3nR(gfEWVkU?q30jt5krC;88sIVz&cmRt;D`MXqz1bewAq0aWLgJ-?;vTJ zUPv}QkiJZx2j1?X# ztw3h83z>(Vz!?CveNxL(Uq`>An`@44yJOeWI*#1(2af;-9?-{*?qhxVj%J8|)BIFT zJOFO6Eg~OP8zDACE#d{OX84?cR-18U1b~ve$C*ehuHOWdAiKa#ycF=^RU|GKY37`;HC8?pBCa7)h#0MaU>=_w*bjOG-yl~EhvZRxiP?qkO>U9!Q$N(;2ApAWL4zW~Vil>+S`=taM zU$SETQeLQo{lP>LDXBu6A>xoykSm*^Nf1B4jm_#C6+v+jH-hh{$hw`8myxi%lwTRl`9${q=xCFg~!^AIm6IoLa z*@DhGhI9B+U^rDqZn_{@EYQx~%*|V%R@|AZp(UkJDwq(SnIG1g?Q++CJy%;P!H}fox7PV2u{`Wnd1#?H4t3l@Atu8moGgB&jegcbf6n; zY_XrRVHYxReb!+XO!T=7tV{WVvjH!2aQ*p5xTYsa-Rd#|kSgDN&Lc(h6O2FJoeNab zC|;#9U;-B`Ljl=JNu91DyvjTE7k9ULrtaOpD(_ShZkPguWC2eO*VO4B-V{7^19pw; zj#!ZTOYYLMw?tLgWh9DSkW&HMtkvta9_Rwd%Qv0tAXo2VGZfN^X5vcJ@iito^eW!h0+(vZ zP$3mKTm?Pw3p9j9NHyQ$#628&0XSI5ya;)Xx-sqCFfzOHCmFpM8ke{e8mTNo1%g?gUZEamLOOCj5riw8EF?Y-6jLq|%t=hc-3 zEjNtc+U{}H)T<6%O-i!!fqWSUoPo+-(B$6@RPGEa-}G3lKbLR&Lv1vb8F!P)iQ_}u zhew6rcd^%%p9bSDB#i6)Rk2a)AzObo#s*B`uIDEZs-;a58zEp0tK~y?-P3mvuD{@` zVAoShc%XhPsw+Q#DmvZNoew2+BM0tj&M*xMb`AClxT2k3IwTMlfiB9bdw(5ENM#Bh zcLs5Icq&gN02o>lnrsoYBwS2EU&yb;Y{L*iFhb@I0^m*fd{k}>O}1AWB6l3>h0Zw9 zmJ)$ky-cbc19h+Ur}E4MlgR5#kx68J6zHdk%)BfS8E=Om+lR9su`f842Ra0M4^|-B z0QS-dPrco+ZqR7FOM+26$ha_rH}jiR0WmdxUg@t?ayKd>U)=z#tCc+1b}?ijbVY0i zUkCglk;r<2{5}PG%QMlUi_d{eswvwOObA)HD>Bct8+$kUlbdiMG)>RQ-a<4&`~)oV zH>e$6k&8FMMZ$mv^~1YC2+%fUf*4E@Xo`0Ubz)y~KWe#=8+mG(5?Hc;A>d#G7N`v8 z8iev7kpc(+5e#q2wpvu`UZ(zXa^pJkafa9n6*_{?05bG7iFkluq8pHea6KC@LGJ!r z=)hmG*}^*EyX1l2rE-^(n$M@4YOYS%`dyvdq;Bt%n;4QOTwt6)tW6HfhvGNONlKKU z9;V5OohNo)t}!{$!1cDis{T}(Mxy$v5Fdesshufar)63}CPHs79$kN(Hk|nXRK#yp zzHKUIAn?Y$I~zg(=VSQ)?c9d5Q)2ijyYj_%^j$3ig(5^Sv_{peED zszPA$$80?2gF>>G>6N?A8IM(@Q_kJ8A_%gVt@6j!oY^1sT#+>Kgm@{!LMFYuK=5M1 zDwA^lcxyBe_4g0V0xd@_+=@^K}iGoEKwRts+y=PCJ%_!sT^#`ru}P+{RAJ zs2C*@EA6wFx%fp8Ps_~<%7E01jz?CIOL-t8BEdlH%dOZ z0Va{5nYE6w^;Pyb=jcS~{x5~e2RB#c!^&IH{@yZ9K2(lpQKGQmnyon5kP<{QcI$*M zGOA{Hl>LxQMMr7j!ldbus+;g=K0;YJSQRQL0*@W4p`p9-5qA(!@&ip$vn(#j|Fp_B zUMgCaRLZ&X!OfVNnSr^kCKT;_Z1*B(o0V|59rz%LICWn zLbWua#ka#*)KH3}(8c>E$%!o$l4x>B#KYNrfl#Z5u;D}og?W@*0kzaC)Nvow?*R{x z(#dx@AyCnTm(pKK4X$wWGjmhSbJ5rCruu#BDcDF8ROZsoB_a>nm6SqjylN9QC2f?2qq3*4sRvOw^cCof=n&P}tm4h#cud|@sAkX1;*I)3 z$9=JtRv{<=f|-x7{Lm>Nn7lb51|CCB&=1s*Be$Sno{#cx)8kc_?MIqmQ;2tNk0^)& zV*-PTU&Kz|wgNg5J>M&;!a491+(!IxCro$6-d6}HzyL2qKmn<=HPCPp*nBt*=rov` zqMEeK1(T-Oo6e4D>sul(xTnck1@;0&(Hr&cQ{^=>upjgyeou2IOO6VJuS6y{Z^2uV zd=B5W9du=y_VyE2)FnSPtX&HhkV>oUwrnBb(12Qx7n2-FR1@{7Kz6Tj8JoV(K)448 zF1`Xpz}-5ibJ})Xo7y*aqwHz4V3w1ctzZF|^PGhH#CvCCVL6ot){q5lDtNR(`c&(^ zB3DXF&NlBr3$-Gh+d@dadd8dy8a^4WJgQjg{o)8Bfsix1htjYQMnFF(2tEd? z;ZK(Cu&CZZcdVM(a44QVrm&V}g|z@q5hxn6vOvf0m;V^XOhwH*rx3t}(@6yP;g2xX zta_uweH|c9S+AJsjECJe>!7vo*k8 z5J#xBXDF++Dhj8nvaHs#g#))NXAN$UcK-M-Pnymq4f7E=+7@p{8WrHAw@!?a|xJlK+bt#_l*)d9v-AxkhCbPOwk;KOec+3AGH zrbD%cXQJ}{RE+FQI3J9I-y7swlN@3))&f{P{D19=!AkrIM1cK9E>Kyo9Dzm}kf{bJ za0cOLIP(nmlJHP{N>`vx=3-TEmSxJ+@qQ&solvy^CRysx6h0tgQHRBM)_YPCQY)r* z+Z&va&UX^t8b!p4K)Q?>D>Ct&aV&{;j(JIyq;{o9mU$toMm)R{GtIN+s9lbi;)f)eNW8=0fK~M_Q)T4Wru)FgW<#oI|UJvi`(ZzV^a?Y zV#5ANHw^mS;OP3|L|E_L?x`a!`kA)*FmX=QcW}7SVxQqmSC9w#+=UM4hkbvd-7c>5 zLl5x8xe#*++ZV&-ys6pdb0A(&vlp@9M6{(8GKnULI^G>OQ73cRlBE}9{dK*^Bq^X^ zhlG+QR)uryumy+$F`58IYk2wYE7xl1zvgp(=|?{;&^&Uhz^Od*2*De4GI)A1)#?=A^$@7`pI;H* zvOSK(X+uW+rsjgxZadDaQyDM%@GtISS=ZDuBR)@>Z{N_o0~Uq>5qJXklt&qaYv z{iWbrZgPzW4FZbTqE$;hWI~H8we&V~qky}cFq7BP6!(vkIuUF%5XK_}{`?R`%*ak4vLS)B!4N7)~ylF!x4wy#<>U zF|KQT2Ig*(NInjJ+k~If1688VMt(U9J;QH`aXrQ4L8TZsQzKaz=#fT#br^fNM7FQw z=xyr{(`=4BM?=8(I|&1_TZO%=R0k_6VBpAMMJZ@ukcLRyciCMYT%_tyq}pt6Q>0p$ zc!7HA*FrfEQzT(-%>!etkn0Kxmd8vyJiy&pv0!?8srnpFtVVV$gV&3YO8^ikT5XE_ zag>0j0Shw+I16r5^wL)f2!DkBU}4Gv*a*oFyoPh^MLjMOA6(>>o%~`ejJa!>y#j>= z93a@?ed;{Cg4xc4%H=!yU*C;8_us>#ZV`Z(mj^5`F;v$|%6*F{45>vFFA;MqvOpAm zT0)e2RePry40-{^OCX7J-r76}-Q1rmH5 z265EmB&HiUGlK}TfZ*c!F9ieWHB|d^R`J{T^J&yXUR1*x1X)9x?#6s62Fsg~_gD%d zp_@)%$K$DZ0_$xT+O8q1=6q2pejpwJexaWP?WePY%%;PW-kpb!<_ymo2r?ZlEE@pp zx567p*dVW@5gK0c9WDGUUBhQJ#KV5b)iKda`5CJ)werO;bz3~;%_lWy?MHWLK2pa? zPc13qu-iQSN0;e>QK#+Hd%AmZ8L9ha>U!ZgCG;K;5Y%8W-8I|2=({awKqQ=A_=*5` zmkPE|{F4YcwIqB+Hyx+apuFL$=ZUoA)K@ZngkPZLNA)Lb{gJM{AE-ly8fd8fB~i^c z+t2uW!DJ1yuPDH=ycBL@bPxEC_xuY#5AVfaUg4!zuM~^y3VVN~GfwK&InlCP*YHCv zFPVmjD3qWN4*rA0sRUn~wZLwEBT`oGad0gSEW)2NhAk}(fQpQSYpt30KmsaUcO@yv zR&%aMCjbUninPOr!0_P^^c=i`RL zp25Or>(b12Ge@{OP^X>Ok`zn`qSr+BI@@pcGw!7Rwbv z&GZ_Tl-{8gVnDsC4l`e(5Tr}$K{nrY*<1pX#Ddd7ycW1^E7qLqX7wF2~wI!JPo4M*aeT&j_T|rQKUv{R1_p7p(p>MDHP$ zJu58Tq+w}MsAx-#Pty+j>29V<)0S@GhzPnUdRFgujbahE)DdT>n!MQ>@)?&9Kmf*H zje$vYFaE40UW<8a9L~_Bw(fW^BlUnJ8z1Dy8c8%c>?NiO_ev2nLe5a$MPmN7d<>+( zD+^bRyrBL7x(_9*yJ|!w5|@V`dUv^pZXD5_m2+K{g>IkhUpT^@cE=zcp~_OMj8K+9 z<(03b;6hkWFHm`Os z#4oQzD4bmKS!M!?$E zDSWkR5hNGX+wwCP?fV=XzXTJ-e|cS&SUrwpl2`{d-2hZj$3X_51_SVljL!>K>hYco z8j0ta1xntawbXimLmvl9vqP9GODz(6ig?YMD4FoWiG?Q0@+qsx(XJR)vB6hbo^1?) z$MRMtcPqvd{tC1xk5q@pQMAZRI$kctU<@h`V}~jGkT~JMM_DNYp5#~y3YFrS%$Ht| z=F~{`p4fRz3kjHw+HGOeXICRUXzlDX8su(%obR=d4rHhbQE@O~oN;CJP=&{3J@gU4 zf*yJj%7eaGH2B?V(oMR7EC`MvXdBWBV_+OS?ekLxvu>-MDcecqOxZZDaHdq3rPv|Q z2Bf|@UHOGLOJ9nr?SB;-xy$HD+Jndi-wqPN^W{&@BZw+T*@A0yw>1k7>+?hwmw;%*vl`N3|Em}jN7;x)|+Y#;Kt~;Q9 z3oj|)R`Rj?wkE_np6Rw@fqDYI;$snfmVV%BG2K)DN~+TZMF)1K+WYo7U9>HUMSyKi z(NM#@5;VF|SP+P?5(J!ut$2DHCe+=y5C?Ik+kXYuPHaK)9( z^1d9+Ezmbm9>xUz#@oBi)?!7em^W*8<|y#w+Hmr4mB?*)cC=cf#awQn=n?79y(foiQ)~IK+Cp^h?KAOZ+f`!YV1WWxb455x$$4A>#;tD|| zknK6Nz(PI3AH7!0PSyDyf4PfRUN z)+HPPJsL!FfnkeuEi~p}po8`GAX4ynZ!;iuPT(smQwzN~o@wf4o&{P7jJPh8$ypEO zjvJAkm_c+X4}ikp`brAbg65iW^42~EiSB?PRSF<1c!h`}9HsZ zxwLaIh7XjbeOkEa`xRbGZ_8)$3tiU9@;iqQ&EYip?fpp&r6qEL;zC9}DznT^DeeL) z&7wQft^*Sl&uk-vmsX4KV5PvV92fM-9BU$<%x$(m5#3yKGB?sP13;ffD{pZ)hKy6_ zE@-0R@T`;DHGbBvUl=k}oiuk$iQ)Jknh2 zbucEI>e0Q(R-Ty}hFrxLZ8OeYGB z#~ync>mE}H0!Xgh0HlpdY`YzB%|3HaYw}Fsyb2UqIMe1i>$EmA6K~V>v8U1T1=)vZ zfm)00Q>Vi!uBZxV_q0XMq>;z@iIcNgY@ohl9;;?zmwjJ!08ei#MI!sX8SA*GwGc(z z@V(&n49`2pKm+YFD}YwMnvtbLhW-OBCYyiRtUuH`e}pF}$r{tgYZIS|&01Q$46SHs zrNUYk=n`-9Uieb&1za0+4dk;G*HAX#6>+3ZUvs ziCM$#3wS^vit`VY7vYKU3&^8coRz4IW(_(BM%+;b+Or@t%Y)54P{IKYFd2jZU(2Ke zpbbpu$_U<%pa$s5`96|bP|FUf#n5->{W+r;G0_G(5AQFuD&}$sW4gorns1nUB+=6y84`o(4h=JW zs&Gtyc;Y{N1^FNem~OVnN9c5R`Zsac-<|nN?!N4(B2ya*4YcUxOhZiii<@whu=&mQ zAJgg6kWT33lZXSv6!MKjWG{tkyCJLsvL9S@Aq;%^p?J~7(S8`H-IJJf8Hwftb;@S= z6nhui=`($d?tM6{GsWm{K>ctt>>FSW9Akn|>}XmlkCSL@`Krs0JxMp1A|oXJLTK~4 zPGu3wKS*k*T0ccTE~mg~ZM}`#smxaq`5)j|$f_T|illtI1$Hs9J^e&95^s zgU@sur4eaZTXT58_W@_Rn$n0fBA@Rq&U9_15$Ez`kB``Bo#_TkBd+kl-&_8=1#(H2 zgyS5`>y{^Me9ZhUFP5`#h5kOl0mEVw%QWz$V|T0XpRQOiG3;u(s# zs&h;+-FlrJa?B#jaTWR?;S<7PQ}D%W`=5dsQ{e*-SQRR`MR~KFVphAcgaLKJMV!Nl z(=z9cJBaiW3ZK%Ec#EiPVP&3$GmbJ%oWBQ7`&3?y_^>!BBx%{`;qB`L#Fluye2A6OND#9h?uh$eIPHur+ zn+BnjnczZWOIak4`aC@#j%XCHd6!q#=Vyv5C%vXFZLY9*grFux5U;%ZPD;J8Ww0tj zQ|eAT(7ZktPcM?vr7wxmER;x3X8Q##G_X4f>;VT*5L}eCfKAZx25d4+nKsU~37fjn z2(1S<3aI9UEg%;3Ul8+-uE?ewV&zqJ7Mn!>!5nJ5@;3*h11q2~_rN6-)=S#aZJe7~ zqZ!23t`7mxDB-KF z4@)Ef8aiWxn}+JLWu=RGR=4vc$|-HDIxz)sx0Xz#;TPK(CcY$e?_WRoO@KO`a)Q+VSTTc9Z7(^J2KTW*bvrd?z+;1}xv zD)e52nwAoagOb2U{(ViHdMIxuv!nz93~>oY0j>){65ft9<@P8MbGjo};a@k~ z;Ia^q0?4!jAtyR7Mckxr_DeA^P3M4D?n)Hg=^`F*Zw$tPI~XJil#9e)3Z^Zn-|N_D zVeMH_ZXX3xuwBx3&;r!S$w5|V=ck&P^y5(8Q)WSr$*fP|))AltBH32khPQMd-I4--nvOEvl!}R`?9ND@v|@HaH31kV%{`D2z08mQiwBS`)Q6A*fJ>7qId251$i zFi#t;P(spo6p-0?!50q*KiJ&YjII9PCb5V8{x2=AQ} zV^!QOomu1Is5-Njucr%vjAT|vqf;$Kjb{w4rIw8Zmv!M?pXyZwRHL~~l!Nlq4ZRiwo5`_G7=&JiKebIGKW7%^QQ&nmB z>aw)Y57$(-RwdDKSTaNR=h&}(UxWhH}=CRr40u$V#BOY zrIzy{>5i@=7L9aSM=T4hd0UTSTvh12cM0o~+Nlk^PKfgE2Inbr!7dc0tb?n)nS(~A zAcDzx)s4yQd0R}DKCWa=?uWXX_b#RyF5R)OT##_4dGPz)<=Bk}y3i!l3=Q1mJ6*Rm zb@)1Iqjd=6Ri}n+a-CBQM0^j`lNV?)AMviAT_BYl{@j$SCCtyyrU zh>3_mYoXl=5kZC8qABbf&@ zh>=q(5)g5hS1jUfAP-X@ShT8+aqH9^2T#>;s-V9sl#n*z!b*((q!&Ib0bV8a)wO^a z{csxnX8ZR}3#h~VAK!c;*4-#zdYkqczxQ+g`GxoxlKPSg>1l#Yh4E5sN`=UR3Sqvd z0j4Wpubd?)@X)m_=yO`!3N8^hSgrIoEu^j9LZ}ns1E@_Fcm*MV55{Ls=U~PSMdD=s7K-HK(`tj!*SL3uXDUv>JXJAukH~Ah z#M?O%blWBpTz6%PC3k-*-ZIHD-*WN%X4)bTZNukt;!vq}w&d>5>X-+S?K;T?|G;B- zs|DC=`ZYKT-JdBdX^TQ>xX7iW(v#)dlHdlC6_UG^S$U_ka`psfRvt7S6imIUPtKhY znAe!ok36I=L589n-I}j=rVdq#+xI8R*#@dSo^m&LogKf>49w%<(mY@%` z4kCK|}H6+gj=alynP*cu~@9-~B5DEg|A|10-t#XE zBi_4|nr`|XO&6IUbW(N9=B821(eKD%yN~H`H;sx)2^Z~g6!?KP(bi)mN?Mz6dDeW! z=zq>XDXevmn^K*U1QF&;h2ySBg~$bf3N$H4WGzH52v!-4aCY!d6MX8J7q@~|5X>P~ z(?$qjRaMCZmZeLG=z&$Ik^mN?^-C&Yk<6|CBr|3OKJu-g8ASO*xRc;Bynu2HT9Tt4UTI4j;#4eW_@(g@rlg-^ctz_h3uPKqk1OW()V=Vd^ilw_zy2m z=L@|;nZS&c0nUeL4X+o=Dcr*C`xIM1rd*n7{pQb^5y%t$=QU(v7~ zntX9~i_{o|V7{qGbTZE5-6!*n;v`o%$$raQ#D2rJi0P+8ugL}QN2Xzx$-`>1VS@Ma z;(VYrz-M!I?VIh7F}(t@+>f6$P~PC-C{Ajf2w?CrvgPIS2uXpPL_zzBrhycFaTbO4 zn)E?8i{g0Z4MB6hBbJ4A4zMvIst(7gql(Q1BOp!1XOA@AD|#j6MK5P@VNFjC<0s$( z-B0A1(NN0r5sKjL)(<7}4!kA2R3*Y*zD^*IVTIYsmM-4MT~NW#EilUCm-3a`@z@J| z&k!vyJwlYqW&Cc7yE};}e$)&tu1o4dQ5<-p>?q2(UT&wmeqKTEc88heHd?q8F9<;V z0U>?F@bo7bnH;9S5EaH{j5-J9P35#zDX&rh4KezA1kEi(ssL>-1bC02xt&NApp9!l zt8fo~;A$}y&>fK?hI(%jmw42z7L+8+kv8+d@$)J2M-PKqpCq|U(@iV^kYv)4H8Vph z0BBfxTRlQgB?9r+40_$wz9?Um2;=0IEG1{tAT zZ#4?jCM7XtY2_o}hhW(p%pV`@dALwIXe!1}_ZB=aJz`VeRx@I54U-_I3)&tu!-PN$ z7C49|n-4r2gso<}z-e|T$UwB7l^+R)$u6iFhKP9)E8z~NwKHoH)!@K5LAZ%qm^MM> z-NIutfU0BhHUPSfPpTJ4D)b|G2Yy(tk8Igol1JrHV71f+_ z8S$I!d(veM6~E0}9;N;)K5-tunLcsxs8^z&h0RascWb9idTF<+@UgJ>iS6BHyF$Lf zv^i+?v@+{yh!jy(yF$fc0wt+g11;F@oZnq*=Z(#ot8TVA096_0Awpfe=V)3 z*DH=HELx#R*rWqj(Q3X5Dy9{jB(jcTD~xiIxAs8`+oD1%EAl;n9HP-ba25w!1J?>f z?eH7HTetPWr8*I!C{;KhA6^K*b(A_~!zxOxvRtEEP!%LRE)BtQ!8{mK8axoDNl%Z) zlY2#n>E;&OP>HNtZt~H;bP`iX7m-IZVjzyuUda|fLm2?>`GZtCO1QBRDu{Eo=FC$Sr;2W8sw zFb=!T(|=U7pbVFu8V#t;<#<$UIKqh_p8!tsF&z!!ND#jyEgb~-bOJc?&#y>=rxQRT z8$Dw~rvm;BY-}*mnwqO80gIDOo%IZ<3y`%rp zU5TChkIsL!=3eMvqd)XS1?m7wK()UlNfVJb+gA{%SFgZcyapQwo)PXY(p5)VKl-`Y zb(A7M#gkqUu#;0~Y)z7R96`ErX^@@R$0 zBV;W+dH*PB>Vx0|7JzX`YZjf<`$W|(TFImOXJlTGw%oz!8g4AuY_BRuw;v@Bq9?e@ z2vd-s!xdcUlfoyC)4ZuzgHIA_&wAllXa$A`BNu5KG2$0iYJ;@4jn zB8H3(QC3yL^3R|@@J&L^IC5^{NKNDNARKW^O7ap$=`F`85<|w2A@2E-1Ni(g>m3Iz z2{vvGNgf;6EO3CXa(R)z0|?S&WHt;9fGhmj2xakC5&JU!o2|TOkRV3zjm^Lk{>!&ubdt~Ne7Ix+4^Mjw ze~z{4@z*&?`YFMGUxB{|>NB?PV%ENpPrfer$GG4o_?P#JtifN+8bXLENaA^Qis12W z6@=-gc0=MiWkLtaUkmOEfHx@uWO|tpDsmKJ;nR&8p{g}&Btq{WsVU$sTSRxym954y zA*TB(KAjOlM5>YIB}=AZokS$tOlGFhnBtC01!$nmagIx^^VWgoGdWS(@~V<>viuI@ z{x4y`l@NI|vN=t)d^$T4h}^=Fe6VNIijxWS@1SrU>-&|&(mO-aGOk6e{}y6h3ldSw zO1%!{{U6VN6odX0j#slQN}AFUz5!5`@Q#;a`2o=mjy}Wrn}bk z29JDnhlspX9af!!9zen1E=Ff8FYE{vgvX#$`7ZJMfM^zYQTBWWRnfg!o>Gdgs_L2f?|LUltT@o2)Qp92_^+zv+rFI4A4E^$ga;CY~RAs zAqhg>n^qp-j2K~RyF)lESlrh3kLHwA5dQ76)U-OJAyO;`LQ0C~1h16hOkrv?houJ4 z!uKlxZmCAoJk-2x44&sH98!eNFqCfs<8BI^8<3g@?)f{Yintm`qA8QC0n0K;oec{z zsm`j}S{FG~qOmDKrPBDih?*&{UMkTqs>EfLyk=q+d-5sdH4ipvxhc!5fXgV8Jp#21 z9E*eo4hE_y=O*B-U85oldWQxFr>bl8DU4GWIR`Ci^twMflNszm69q!Ak(G8T<8Aje zzsE)PnX7jSLRkJZA&gWiP2jGC(Ji#dTMGrF&V93I;FuY`g@^XkX^_%trr4raB+jgyg_&>$X|WtKIFD5_%9(uJh>n`HHgrd zWD^s%)lHbB)f8{xlG?>U7cmO83lku&UBF?8I)Dw~aQ+#uL07;^6-9Y!tqgxNwtJxa z8wKmrENsk%JVjLLA9d0&r~nd>JLtpbUT|_Pv=ZEdyXrQDMVI;!`4o35U*WP-m5@-WMYD2KNvh!Y+b)q-Y9` z<+xxPyw=w&Y4{-9HA02-hkS-721I_6P_W&Bm_GcV;{JhgwAs785J^;ABNz@$MfSO2 zaw!;Ttm{c1X`dGTmEHVueUJvr4d+Ae9(r)ILXX2? zLlGg}Y1z{}xzmY0Nxiu>bK!;y4WB#H4u|8BF3JrAT%7FCTsUtr+gu1+mFeb2mYT8W zzPR~$G>_EzG%+jEA55L*lNQlR3^G(MXbCUiFSY%_(~W`Y8Ki8}6%SHF^D@ZW=}YMj z2qUDv6)3jYi|N7(-8(Wnh21Tol~+Af@ZmazeL(dYv+mPy)q{L9InkrW2bhW5oV5;H z_XVf}+&9oJv<@fy-8q@|t;30KqF_-vkFdN}6d&h0hPhxsZhUH7{j_C{IU(#JHy51% z@>E4A5@zs7X#Ke^P*BB}aEU@2|F~<|Ggb64iL$!ZC@Zcf36kZ8;uJ~Tc!V#5`Nl86 z8qx-W2tsj4c0UBmt;M1SK$On!Vz(sksYWYwwEy!b8@dH`nrb!tb>qiS|cd$o^qOct^`fM^bdhN<{7@`g#Cuw~7KU+uA*TCvA*#>BDM!}@Ji~H!{u}a{9Q|L4&;OEx4&9Q@ zIAY%R-DWBX&s?+b44YX;7n4|Z;u+gTys8w@VwXl}d6RVoT50N~6Fotz+{25IM9F0A z*;cWSYX!GpUlpF&UR}?V6wkyuuSb(Q_1+gd53VIQFQtgd;EkGsPcgK3OlZwwOB)VM zjF$AGr_eTsg=2-y$#&v;Z35s5f7Uoi0DK01LFnf3uWQL(JVGH)>4Ex;t=s4P6XmWo z_`3$&d}Ox<|CLt~qQ2XEU)f@Fb}0AVTKKa$EAStzj08d5UKd4B_Ft4`ts<=K#1=y6 zHs3U&vk0-Q2-kIDgN)virU&^^?3h@DwoYszM7g`zBE-EV!cdJW(|Dys=w9YjBEn&h z3M5%1a57~3z|~z%!Xn+$Y(NGc>6UZ`&4y_8@H&RbYF}0XdWlv@l?wge61z6eNSSJA zQ(JQI@bN@8yCt28RRC#vM{6UJP|N5$R@BP1qv7&S!rNE2q%#k5 z^4DU^mis0I1=qbCcpuMoqwZ;j%4k6G)UH1zpWQu>@24}8()WwYgTg*Z53<^Uay7-O zu0;#PNe^EmuI@B@=WM55iH8B$S^rZoqRi>UE ztu*xv)tswt2@6*4$Ud)-pm#Qrj)Zyh`X9_C6`AtH)We)pCTcttN%#)A0%Rg5!Vy4~ zyH4eFWbY2+a{y6qaX1V?GTsD8+SHV(WX`&baTA>3`jNb~@9Tj?KV&Th5~ zAs+&))?0|I`_dV#r%v0Di5fz}2-KzCl_5)$S)afd!bn7?!7(8r9EiAFy|u z=mQQ{d_b-0L$W)@1=0|l(IS{>G%ISS#*q)+l%sG>B->8-TnDfxJb((28%`$&F?*V| za%!~nDmkqK4eyHmVQ87G_lL8NIX0jR?Ya2=KtM(9IRgMh0#G`* zTN0z}fp{i3=`;PZKJdW-!VC^?SyMQ`U#k0M;jnV3&@Y=LaWyQ-O!Q`{19ec11muHc z9|-Ib%itA1(VL(3Lq1p$dm%qfp;+jb%`wD`{mW$V3@P@@I;r^$Neux>FSJ86 zF~)g5TO{?nDGLleej<$bzNn=C5VvvE7vq@(ounn-`Jb ze<5oCiOfZW3s8|gZJRA{dU}9p;B9yUB2RIbR>~EEvYOI+-jtE#>3kXZQyznQAxzma z{^}43hzb5(8~y>qH`~8=5;}C-e|+T#7#l%LAefVij>fgK7#$G+T_D>HwYxXN$MgePzuj)!jC&f1LTh5_Fs!EmQ!jcN3qwP6`xkVg z?3OT6mC4PJp=M!spfni}N^-hkpd*xFyx6l~QeIL3cNqvcl7&5eS@ui}7_;E2#g-6d zn4Y#6yJ=#~-gS`>pvMc*L{fQ4FQ))+N+;zcjS}N2nM4kWU#RFtv8l1@moTN*!=v!V zuinZ>4)s=z;N8EV{}aG;CYqK(-fSUMKbCM#ZTzJyq3?TjYYW7#8By_;0QCWQF++7( z*M6YO!j$iy$@;nxN0jjwq*V2w&CaV5OrIcWWq5^08IicHD=GCVpba>#{@v=0$>@C*fT%j z!snF``Q#)Odb;>mfGFrEloH~F#^B&5|G<_4R$X3z88U9}+*7Adv;?JqUMO3n%=KU) z+!JT^OL0Lz{;B*{kxFc~O|^+AlN9kexhN@r#+$}<*?T*T)`!iIfEHa|oHJ>bAwenO zb14e)VL-Uhk@C-*I8?#g4nNtux@Fi3viR^506mrjNdRG(Js^!X^+eYW6(I9#htn)T zPmsq3888#WYMXkZi-=i>+!8Q0sjw1Y+Ve9EhL{J~3%SHU+al&QaqT7knfI+OJwY}p zAu-YPKtVRIL^BNSKu(yG=%o{hrag*4lY-v&iY7#W-N`f))5~xr;-Qy`iZcF_6kTOK za?YgE4akB?6#+$)0>DK)jZBhJQkf)_5!lM}$^!2#28sgdSLH}l3h2DRe!3!tKhlik zxp2(vGZY&-fZMR4Z%`0Vu5!Nb;Q2^bJm<*Ft|K!t8`rz)g~~uiu2iew(1>d!#`c8Q zLVA5#b)@d|g4kBEL}S6m`gzg%lqAd}OcK^?WodS=%@j{N4JmV_KyMF*Mei$AfSSwa zQZrKl3%~|aGWE%P-9R$v!f8P=5VAbWeN)&bc_Cf_ve`Zw_NLX)9%&Ix6I_2m&hXCq z`o;jtb%f0ssMKBsP#}x?dQb?)|820}bJ!+p)=U~#NlQwVRCw_6iWmc9F#rcVgEjt? zHHF_}+Syh}543Woz(i{#sG{SXRIZt`*-#v4YfOPu3jkCy(vzK_Z6*(GhnYh+vl13` zuo6W&O^{S|?Q!CAp!)Y55+&|-#lwQ+?gXQYo^vo~1bc)ZLnH-dKs_Q$xK+A%pmO{i zqFD<_D?}y%+1y*bBh3;=OH&F8Sp}{xEHP-Qrnc(>)$Q&8nMqK#bWH@!ROk+Lnabq9 zcg>-A8P0_MM^fEzHwIARfD;Zw0%3T!D3S?wx+>Hv@$25V16ONbGho0;lX$=R!J=oS#Avw!3tQ8+%4eMJ^XBQN17BKi;n8yx4@pe zXmu#AeLPmsDH_cVmN{L(HDhiHD_jN zS50iM)F=pmr79teDDXpA2K2lZ#$_UE;V@tG%wbLDs z{lzsXxwTfbY^r?AgLt*uxjvzX?let%I(5|lOAdW^>8LR&G{#a(4xj$~@wfl$Y zT<;D2ARSlynRnGieuUfyt*Y7)aIVoQZ3=x8n)sm;|H6_bynFR&vH8VG?bPye8R(`< zPd!z2eJ`}x%YmgA%aEwO{?A$7h@chFRoOgDcc!vf3~uqWadojHzajZwN3) z+3C?ejHFvLlW*+_BW{5lf?)wJHq9@^2Z(82cTC*5V`A^cmba~Fl1UYU1;>TC+cubBRRB;+5>EcW$JKmIj)kr~?`u-!{;w;V(^q_5L)L3hHPrP+GJN z6hxKsF)eepg-qJ5R+zA3dC^7ylm$Wq8jC^3CR}?e!eWry$l>i*7?2ds4pAJ1jKGzd zC$@|tL>})3n`&9nIG&0z%x&;^pBUbkhq>(@9z(F{3T847aUigsnO9mZxAH@B-GbF> zN0l)Iv1(AwaP>Cc`apfF1#kWQO;;rM#A-_!KK3Toro>!rmY@4LxRr>9l#yqgnlH<$)X{tOiKQxqS`75dV;%c>L(! zg0L5%6$p=Xa9yy>U@YY$ahMM$72tAU1YH!jLGHpII=<$bOZ63Z-BITmG*UnZng#;J=p~QN_Z<<;Tc})QO?4v8sVQH38AU(={t}ZuxHs2$gT{)bktjkmfzjx z&U7o%#gQ7GvuJf&blnhIY*Aa$0>;9fzvbgX`FAr~^CC{vZo=F$HsRm^h>AF{%;=m6 z?MN$aLu}XHGasq{y1++4Sj{~Ob+uIpZ?-_#;BcEhQmp!m*5mq-I=D@Qy-5{>^;I*d z98?rmM#>_;6AQ=hRaioWMQ&Twa}}247v3wDABLBD50DP1JTF65kt6*}VVfj2Nrptk zmk-}>5~4e--&3Es$uX}IX~@L02GZl9Wu!2XuG`L#VoNQC^_me1-kefvTq(HFUVS39>A;@m z$eji?U|Xhg`NlujWVIzNn0{gk*Ygxz&|kiiKRxkV$m~wzxkmo^)wl6y$!2G}g<|;j zG#-9O7@Y=cT}k^q-)vujhidv3zou$Na0UdH4~Ze<|K#AKslczq7A>|N?)TH|dV*={ zzZ_|;QHI>nQ9pg~L^kI%xGf0z<29&f<;*R&9Vap&P1|tdp2b@mnj2P9d34O*{2((4e1ru$+X{^%EoJxEB!!H&#S<@OFlLB@(u2RlPbS&qs z;v>dF7_^;>nFY;cS?j6}iRt&YvL{Wa! z#0t0UC9YYjXt78Px3Z4J5S9np-^jn^B;Vi%<97=|tE?{yKX&JWgWhZdUafsJ{~xQE97$ z-9bZtBN_^eEbl?-AW&rhN>J=v~OP9BqLV z{dM|yp<9m*&5TY1$`)>tPQ#Gn0fJ_$&<0mjrMGr<^LU}ey%1MGUEyM_Y&iT8EjZ8F zvj!tX3OVHrm9`N7crmNa96@YC(?ftZjqJn71Blk5m{SpO&lx}qhdj#XU)RhR^VF2x4Db{o93V>Or ztCpEO)5FwiNKwjkuL(;uSXE%A?Jj93q)I530TnR~<-{xvz8`{rb{k;wg=!dANHFlF z(3maQ+Z@{(a=5z}7>;MROD2ygWbjFr1$Q*XnK3l6=XA6d`_1->tlg`Dr2k(uj6U`C z+v7R6p2iy;W0?vV4d(>|_Xx2FRFiqLvH7ZTUfuTfEQ+r=mpq=!du-&s6oqnt8<;fX z=IEb>6BWhXZaPlL7LhX6`CM9p(``dAG!@%0_cTRJ0Osn;K$Tce)d|}GTJyT&dGZY zgnRgUW7=jTiS*bhkjO2%$*WK>#leA@ps^01Hg^lq+@_m6Xpdl@@m*8!9z_Zt)>mfN z<#=0x$^fcg$_g`a!3<1YpJP5Lcu7Z00mv=QVIK~7R82Ox3itzEvZh`jDuS^cRzzN>Qut}$t8+>h%uy>tz-IZwk7(2;2>G)T$;l>$#;G!OsYEi3q?wjDX-)5 zpf6@=Mpbk-+xU}K(Jh+g1qM_|uf&#!C)Ov9)>-*J0^#cS=k7SI6$)`ud?$7_sU|s7 z7v4&w0)>lm=A~kl?V^^5$aY=1oskptX^BzZy;;PA9TI{-@L;=eB!fT6d-35#Bla=% zA9y+AbxN6ptCdtNUTvhx%UN0`STAR62XE;kkL3+d2xSp2hO4p=8Id7aeo&*0h^bxs z7!|X;dWxbf_Jhx1|Jl38K(!>8$|v`&g2Lhv8>Gy%)!dO*=18Qt^GaPPL>LLtLF-u< z;2ADLNw67~ufkG;l5@ET^!kO6x@md}q91eRJQ|ne@*@QQ`w`PkOSOX2rMZH$W`-0i z=qi$ORa41PzEv_=CA4IE)xJfTKdte~yUO2+lziVfS>FF;p!I)%m2T5mo|UMS2K#~P z%Uo2$M57yss?Ygvkv$yrin26&faz0*PIWkYZ4w$>iSum>HPaHb3Z7P!TiD=Jy-Dlx51%c}(NySqwQ%h6QJ0|5EBRASnVmM7mg)vC~K*;0N4C6hyU9Fv7; zv)Ik@4)wTxOliBr)h$00W1On}H}#@V6&U3Wst-6vUQO2h&Fj}g+M@Rji9DP0RxiN8 zZsY~7Yu+98BC(WQX4VO(>~>Lg_(%B`+*TWXU^zy2!^@{@^jQ~ zZI*syhJJriO~t9@0!x9FDDI-{er$*yRnKsWsI9ncK+4E_*@!caNUrNn{co&u8BOyjl$I{6!-pZ(ED9l6j3)FQZAF?GJefnF=3#?`@2Yf9B8-|5{)#@ zPZ1==Wdv(IF49PQ4l_UP7Nhpat?cN3eK+phe>9NHxac>WGeMLuMlr^l?O3h2QccYx z`av3=hEpb(@`&QxNmWz9?snuqGKVxA>*6XPy6ADi-H)RhqNnBg1btQwNjX{PalLFv z|Jn^4k$d26dOn7+7(_jUOO8fKLUnDH_M z0Dx$X{}~yE4_h_p&zTs;2f-R2NFiulYOLh+)AvJuU>r}GsKw)m)46uuAv69UK}vrt z1}Xn7wnE|c{pBI8yEkKP&qFu@g=?k$ePAArs6 zY~@|Z|K6il>v`aA?wm*Aqj<&+yxMj`J_C;LF~{x8GkO2~gc;*Nk=!)n*yg6^UNYR~ z-E5=tu4$$F+OcWJotw*>_uv*rijk>cdf|&<`xsGl zBYX2)G(&6?Zw_YE-OiN7X55xzu_x9|BWmg^J<#0o1Pf=cWO$0 zIG8V!!L#$t))>qZLktA}M{*F*$6G=*uq*zoJqCqs19BQQ+TQL)AW0b~Nui4iBq;|f z=nehCbbGR-yWMWsynetyVcJAMp(Sl@7!PzfA?r25VuAntcS7i|d*Z}@{rY#r^6dT6 zh!N9sscVm(OLGq8xq%gLw*T@;M-vIo-8Lef`zmoOAO|kIL+TWP?3AwGlhZ!X_Ybsh zVHmq%^YkCxEBn2w07@U*`;WOj3a=kcm#_K(06kSY~&K+5u4(gDFF~c0oYEL-6?l0i<>4;~<&_j`_h`zgi)(BbhRgCOte4q)B0e zI2%lxO0zFTTIJg3FXe^zcz;xMlWcVf(G1&!IWjP&JM9Oft8wTnGp$qO~Dv_4GD5R9X|*F@c`gM~UfF0c})r)u_*?;vBRvZqY>R>I2! z+iXJ-A2iGfM5XFwgm+I#u*ty{!V3;UJ>j)%;MM9X-9l`~nIv;lOuDB{UQ zhi*8~pxq-eupj6XG}53O+_~p)Jk#f@Qj(Eiarn^HwqbWj^83FM*#xPD3KY#VYk4>rM zxn!SIIjVArTr$GlZI$C(QqWSD8Ei|WGG2g5S|IEh((qZ*iD1^%f$akfhrA6j{+sRJ zbDv!yg%!BJ5dKj3?-PN~S0z&ul0{^g5E>c5 zFPb=W-GB!mLo2-s$Up`S7R^I~R;&b3#_BfNC$OS@UWM=&SYaCHg5Z7F-Z)@I+rH+! zn)Svtr*ZRR5)m^Y#TCaSRcdsi9ba>7W@^MdQg!+4X#7gi^YVHHTJx|q3i*XeCtBpS zj8?bR-13lJYbPd+y7g5LphcUxYBUO}1s9!R+egw^3;bwf7uc4@QV$sGD(Vq)D-#eA zKjMI>`}B2nh0$pU((Y>1oEU?ccE=~JSVV2}fV0l*$@fVwS$;^n;D z0*I14kcwkRb*sHZDzqr&ArtGkCQd)m!YKt=9Q}Y)0ORhYqV-P-pecf>B-JD@jw-GMk(LD&Km_p$fZdCD zw7jPPnL#{|3dGypnRu?&m6QViTEra0D*$#U;vHxSP60BR9|uCgsSE7$Xz1G5;z0Xo zU_}bC#e0W~w#O9UO2C0bx5;#E#B-o^E&wNucpy*j9WGkRQh+M~r!uAxOK-PV8J_$= z?Mg8m%9Ep-cr8FKny*TE548WJ0FZ}cvn{yo1wopJJoSX4wV zK=0#(fd!XcMYv|T#>*_S52e5;5W5P%Yf_K#Jd5<>)I;DZxa&OOu7Zo!m?(zEa3!Xz z8o*n{Vk0dxsqlyeS3M`(&EOiz=2yfJ!r4UtZ`D>Kg%mo%EdluS7!o*vv1m^25yg(Q zMWjGP3&x^pxeJW6*P{`}qLSPNMq0Je2xCzm?*ZffQp^jUzX^N0i*hJUTpmVI?(4Ft z{kfPBXxNJKcNf^`1iipp4O>y}?gHD9yhq*$YS@bMb{E*Fy}tw-4q@L{GPS2!|8TWw zG8JwY0iZjlEqj`!4}j9N#Q>-!L1<|=*!RT*fgs@+2c)So5nik!6S1t z+_plrWZ3EPROYaAp!T*{I%gR+SGg6?3|IiVNnX9&vb?TTse#6^$3UwZbrq$H5_}i& z*caVO70^ninG34mDZWrF!;I4NxrYmAtZ`9L3^_^_&+L!&a{^q*VJ_ z+Rcn(*6IQXB`-xlM*$0t00OWEP%j{^39$8;%=JyqDLQKbc?4Jh6iic?4JhbU_wFgHRRg>j6C%a~N1zOIBHE=adbZex@aWl*y1)6JR}n36yos z%DH=v>VjKrS2tS-#~k1==dBXp&Fg0#(;zX zu%{{I1y>A_0C)g6Bg1zW37Tr2r7-P6#Z3aKxG=z*?ZQD$wY_Z>2^wsk1$dJOxZ5t~ zCXo!^N+f6yc|jzy6z;aU+~scej3$Y%gEVX5g7{wLjh%(7IwslCJn*b^1aaPMBXRD+ za7$<;j7r&r%;)q^ZMxM)+A<+Kn)tnr3ji1>u$g)9T%Zx&>$p&D7h}(Jb1u+u?gAGc zMSil99{}`{A9xU%nQq%0HCeuWM+3DBGzY*Rjc?zn85Chw1h+KMjY6StR;ptPhkr*D zzL6o{(R}L~k*Jp8n~DU*c+2QM1LiA z@y!-d0@Ok78>)NF2!7CTI!Gq=wfypX`30}2`-YqC4>@C#ZuHz#4j|!ggE3P#_*?y0v<_*M{v>ANCL}J zr8mwJ(zpYf6N9;_xPkOD+{t=={`o6hXqoQwl|snD`O^b^C5?sYcXU6qNJ5O@ppVf# zQTkZ9&I@@J=7-`G?M3FS_&X8P@@SVt#)ujcPOM1#1%adrJWKo2>QBQ>vCVXmQzXdW z!-AFz(kk%F--}x=w|I2=3s;QIQ9aYu2qwDm^%I~k#ll$k!8j}DD^&Lzrq9wvV`qyA zdCHuJ{}G}3e&FXS)A@n}YP)LSI$2NV;9`PVdG-(&phc3(1!xntuLZV}luu%IC2U5W zSyewR*g!Ub1~;1%l`xF4i-$4J!|Cc$qmb5BoQ3V_xgi*fJSFN)4Uyc8jP*TM0!OZh z?9FKk$HSvx{c_}E*jkSg1U92?td0^=*euW<;2t9ihR||G!TO@HKks6H3?( z)0wqT84EvAtf~z|pmW7l;h5G!A2|n#Ey@l$>SH=}(af1lIB|Bx(>x?d6qHyqGm$+1 zQ9@IsI>yy`kC^x*9=ovFx`h@#07q){6M3a&*@Z-=Cs#<#D`(j63!V7OLcFormS;pT z_!Yp$Ng7T3Q9dD+0$kMxIW^D=`9c%WEuwoW7fIdn76xsJvRw^jy@<3%awk^?jig$6 z62zKmrXmTkNa{Qa0LXWcTvrIqOD+`g39-|=9E{gRcqgO4sb~j4-r+aseGZ^`#LnVj z)eV~YY!+m=7~5?D;k&}{pE-wben`a@|3u683hadu0xeUe4_*mfdLpg(R1{AKe>GyoAuE!KmorNPG|c%Y3Je}7S4SWJE@?%)>ka-=1B1unqJAya+)TE0GW3;d4m z;Yj=KK-ttiz;=Q8A`PhE<>-rI(vLprDCRz?vz$UzG!sD_&4GW=!RH6S2eT_%$>yp@ z<9&YSZeaU(0KY8oy?nmP+IOUp^953}_HDLMIv806vkzfnm=VC}+vC-t^O1(Y7Xf1} z0AL`dFgwTN6s{I8f20*{1-Rb9$=mF0AXWEIl@a}s#t{}k;+%f7E$PLA-GN^CBI}14 z`YNI9k>*31_~xoWu(X)m5;l^-jLpU6R`nJdY*=Mi;RDLBve3_A4ZbWD?Lu=_5_KQS zcr4vP+Mq%?seexn;)T)|otDd9XImCsLVV4qhrVb_|3gp98m^__9>qaI|O`WQf~ z*X}aiX5h}ts#NJ@lmv6-r?q5++gh>c{;kxjru#C*10{od)u>xF;r5*fF2JV# zGmh)fe->L4?=;=wO*VP4I;_nUPDM|v{et{szBZcxwn zVmoGH^W7pumngJ1DZBGDM%6`GI3TGkn(fVWD&gWw`Hs*X4Z=bnjf4^Qh{Y>7qOawN zuvGM3Kp!7Y#|^illIFsy2zs-(8%YS{o-3$omthbqpdr=>pQ6s_(eSMB*X0B8IdavU~frhYeH2i~0J(85zf4xd`41rUreSzp6>!heRnEOj=u>QR` zk(B!74u>YW*j>nO3}(<@2GUDL>8gU-P6|1O_z$9nXaPlh`L+B>T~+9^R~2G&K*dm; zNkrR@DwTWSoTA+(X1X|vqDltb?V_bkOa5BI*qvy>RrXrKW?Bif8}}XU9Xr3+?+04A zH1srCiS9LYdzW^QO{Z7fF5?sZa-io9CwhUVmC-UCau_rf=R`Lj9yZI{ljt$@X4-H| zUpBlr(U-+bU3q&_u&+0|t)WA=19d%D^DdutE20rk}WmLCQA0K;B{cNrsmmLGPq}PqRNz0x>EtpE0~{GuV8EO8qzz0-k(MC^&sUtehzB^|0cczFMR1hAG$4aAQ)J>{ZUTAX&cvc z`{O*)LN0e9CNHT}*9+FrE%Bn;A7`V5Llsd0VG0?{1aZ>iE^$@)5=gS}DY5tz?$$Gq z(Qw@zq6%qi!4s+^uzLb*oH5LUK+YNx!@CB=a$Cm}f~RU=H>*y;=S8R3Ns3tLR*kgi z6b|YiLuiDssgATPGzHQwAB{7#C)cYX+)ttXGrAn$hZ31Bi@^+56`QOOgOhI6NF}wP za7D7Rp#D^Ju|U@4n!k|MwQD3c5$zgc#lAzF01>u9y`#LI%3T|OL6nYghYKC)Fs+Pz zay0}NgA&){NMA@B4LbDI(V#$Qh(Oo3mlJZv-Kl%4r3#`2!+q}EK%E|Gi6 z8w`6Z#Hf~>a$^(|=U8rr5I{EJX6xHl;^f_aMy-zJ<_CjeM4Wr5)v>%j-U6gZoLi{X zk((1TYNZE(B0O3Uot1iZEO#7eNMN5AAZa~@7a=x_Trf2gBCOw&dpr;DV!2Ua{)mT1IZU;M-k$Q>9o@*Mp$yh#8Ho=Y|v#4FCY|Y`u=+2HU z0rpnh1h+;wl-~I#-PY!gZu@FJWp4+YS+JynD#SL0A`{?_dMm3@j6)?ka6U{#ffB}= z7uw~QiQ8h-M!l)kgqEo_gEp25KnwV7g}XNB8}%C3H0b-YP1`KAO3%FD3e>ZbMKQ&h z^$N427~M+Oyuh|XiGv1#Zdn3y@@j<%G#1Xb3f>=;ptS;pYqcs9DC~19V2e=J7~4B% zQF~)!tn$bu@~T~)LHk1b$GUc}Pww1~=GtKgG=bUeF`*Pd-OG&J8#;qm|E<~f*fj~-8SUMk-#qP+&O#-n&p=KXyRCLc>^hNwt@oi-r=ZleS2B|fy&`)VdX{)VZ)of- zGcy`Xd{5d(+8T29N^Q-=Eq!j0u#a0TiSb0UB_^i%f~H=Zv-f(-aGJAQ(0jZVpl894 zw6vM#nJSJO&0Powa!_x!MW~_E%1LDu_EzSmFYhYOAwq+e(TZi58w(LbcReE|ERM9G zG2(IU!dZ@N!T>9PR%LncZg7NHC;(^T_@xLl<8*s9seRVTl2_xBBC!0Y2y|g)cO@dN zoNybGbD{E0IdKE!w({4$ygnH^fs>rSKuKww&pbSK2In4C@4mb~xei99lJ|gdS6-i7 z2cy!qTe|vPd3|ynY|1s>g?{BV%5^X*9l0s}%4?L1uz_r|E`s+{sHmRWLlTTjC)v$NMFv*&=f@X--oJvw!!IlmrKUk85+wPaU*Ih%f*a#e?C>t+65iu z!>Jb6?Xb{JN#0~!)PdX+p-r8J?V#E)FOxLk*3D3U{eCTt1uv6$KD~{EKX1+Qn4GRc zStn@@M zxAjVWwnYO_Kp?@^cXTGHid9&(pv%WsNcl`wf_Rn}odZTt<}h=tzbd-UIl!uUy&H&&@*|nX)iKUBHswpzhb}!}EqxY@1#m3% z;^Qx;UMd{%A{TlBv5Umiyvtqrn=vUC7SOAuKNM@YVt>^R;)~PEU3aODd67tGP>Y8g z(*AB!=DM#l+DHK+9mq!Z(`?N*v5?%xV!HSR76}Ie60CDd)Te3~HwX%bMwdz9iAC1^o2TxG|l) z7UZ2oo$UMtj~DaXP3Uf~0?||lmD}}2zTr*-qb?08VR{9RzB>0pWEj|vh5|+PE(2d} zzvIlGCiKf=k=z{nn}GL!8eeOq8n#B7?|ytEA%Fhjq_%WkE};D+NoO9^3y*JUH4t{; zK<6bmODhm%F)_+K7#yp;A;KI1LGqgdE~_jKhSO1x;?A(pQPGJ5KxtyQdwW9W)*u`( zX)ku1frKb&T>Oz)78`tzpC)sz(wUbrSubCE4y_aeLfdZ2`8NW%O{Z20 zyajURdnC63N{4eo7mnjOI7HM6rKB-Yr+J+ud$3;53uwnJImy<)V{op2vVyt$tBzt3 zLx;>5WLrs0CtJi|S;EGZ!&fFh2Jlh|Y1ZawT#f>XM(j(q)p0+Ep_`o*0u5xM0(F7S z-T`XRHtt$Jc?pWr=Yr%33eUREl>2C{%oYHxyRNc(koJy5G3N3oCtJ-HNCWx3C!(i%n)A$= z)BPS?e!z^;;tFcVaEr6g1r;Y89sJrl4`o9{5P3v~N;xFt`Bi_+iSQGQ%+E=XtDm_b zNMEOsr$~*sIa`UUs*%@nkk!QvPw_Yy$}nrhQ4p<<5VFfYaqrXUV{9TPp?f4A$vV<6 zN@6iErBFkW3+35VOsj@rVFSVX?_!~ zW|Nc)Lvstl_8`ILp*>m!+eZ1WGi4U3BcB9M7ltR6gq$Fm%C0Suf8c}+QBfw7Ls11y z2}j$_?3Xc`05G|~(iaCIU@KS(jcpGP=Z4@mM_a0N&5!&R&@&}RpAQF#1&t7O1d=jg zf+|{IDzXA2E;efS&;!Hq>*8bdwV_;D%Y?w|o3Bw+s;QF#)t-eh=VJ~#ziMC&XNS*D zFXz)!)h$uN#ztRfc}pT$msPEgvx0n&@qaq*$>4)#71cA3=a&BEb*Po`9ftef^4{N* zeF9d?440)p_VMX06=tX`uH)3IDOX?&J7oH_C_HAbtl{^psxd)&^YOMRbaVfluG`;_ zK@e2#G>fT2s)o7rvU_3FVKU}Pve*GxC2O|46dC|0E)O~%BQOlvae!T}lKr?-BtrWp zF3Nc|AxDSIPhXU*$F*p% z+I1>=g7C|Ck@hIgk-eZ;?z6fxOzNy`wm)$fjhS zA1P|E$<`;2gB-vq4O2t6_!qn>@!aeq8eoblLeknS8>D%;Jz=wosVg_JTvl2FcLwcu zxN9X=#cg?i2S6C6 z7wn}HCgNz5QplnGJ%>=wpxi3)la1w-d3#)m>bn{glTz!NvF!0}s-=;`{+lhxW`~3- zt-K138zs`MeAYReM9c08{+?}M}w&! zSH2e96|IP?GEX2hPN;oSuB93|s?m|en)zvmKqjOz7gzd*%%%1_(#xaTuTaH9;f#JC z*QXjvic%W4zC+KiX{*hXH7+h~7Jlze-wUIkgS`*x`H7#0mzrT#Ph8)9--V@LS28vX z7+xg+e{gX1Kl=iQFZ;zSO$$?~PNbS4TkRxM6O77|e-i==z1`GGqMQWFp(@o53gpwa z>eIK+J;%?MoL>Sa9QZgsk@I%>FC@MU4;7!KMA5#LizjQQ^JP{;kkNNzr`zx{2fx>x zrx$Ot+@|@`x?wodQIr>s>GyP?I)BT4j`o@W=i7`N;_WY*$?`05UMx@T=nn%KkJY4g z8N))eNr_q~AJmxi0OK`C5mBwjCI&5jiH@5)^?KUS8>eyRbyROX6Bo^tte9$~t~`@` z^e8rbb2AH>x-?kvEExOwn)jO8y?*us)8D323uuL-NR94(k@)%B5193Q&Ak)85~&Tc zzyI6qeZ%4xmMl!0wpA`(evVXv70Yu0u@<4^%cCprAkE!@4X;|aL<8hw0FDl_z`Zgu9GI*+}Z|3&~^ zK~PYQ3}&E$4N5t2A9%ey)H>)DfmXU%z9#D92R`>RwXh#-sh<%O zj7-7Gq*yyoEiPjwqw{6fRT!E;oCB#v1mmD>SK`^w>XJvukBDM6B?D|o1)dOIict|2 z_0Gj5T03X_m1O0u+5AUkuf$iw+UKxtp0A}yf=CGYJuz)^@~;eVJ!9m2BOJbdlE=N2qIv?fd(WnBo`qtFalqJBnQ;2Cg28d7*VGzY2l($weVz@vhkh2 zHM{{)L%5g~(jACn6}~c=fR5El)pT|n;iuN3YDIq^zKBS3zsDi)8mv$!cd+{wJILu;C}kw;7tMKHHHJTFAC14 zP;wPr^ghvIXvH>osj2lTYYkn;m<%W8L!nYVaFPw&2`fagT})rEqXw>(Fp5^2xdl8 zA(m3SV>CQuT2MYx*&uK>E^5n%`8@n)fm^sXK|KlgGOWuq@~2^E((pZcebJ(JT*P9qa$iamOx3%3~#5FsBR-j&bkY zc+!zcNeNm+avlBBhYdpI2Lg0a6F&nJRT}KU6;Ed>%_gz`j<_%efdEhq0`X=LdA5uv z=eB{=P!V_T^3H&q6k*9+nEh46`jwT*J%P3g0*&{t!w?65c_Oq;BzZ~GzpLa2TG&QF z{BT+?gl^bDLi|8dX$KPLUt0x^D6)q(kw^`f^?vkXu`-F^*(RW)zy{|=LHugbFysT- z%rU4^F^1^R)iQ(dJmQ8r0Y?OAOeFz9Q4JDSb3FrX*g-)AGht$3PX{t!(h#gWU5U?M zosz;27+b_=xQn#a%vB2n(NUt(sufU0K^!dAIB@3xm1+<9&$J~TUsb5!h-Mry5VBGr z5EuJD&t08@fw*Y1Bbv(W_|~LS;4fR4h)>8Kst@3Z;oLM*(OADY{&s;N#;b%3wK=5b zvrRXK#^3`QFvY`PwX+bPfj(6oArJ$ca0wnr)0L$01O3-uGfxH{t2J;$_?Kn=18Fq* zhD5P=fOHmd?AB>SvyNLOAc(QQf(LCjsqBlF7aD`2|Ht1JyHxi1`Sbs~pqF5vt`=PY zh;cU+@ma`ol>yQJ2!!yokIH}1qWbCd{72%rdXnBi&zoC$JhA2A3F#w8mhL%;bg6#+c>D8HqO)W~YWgv`_tE z^Clo%XhgnH#=HUzG-+RxuiJ8GMr^S4g1%ut%cd^JsTGOVA4ZoSr+>2iLNMjZu)=gY*%Qk_QozV2qh0agz^`3{HY(oo)ePotHlR zaG96E0gK=foifjGCAWrf5w9BxC<}`tAcu2F} z>21pZ^_(s{t?>D}&(GYU;*v^-KkV2kWrOB)^bPHEmz;;d*SzqY-kJ3C9ll_M|;kw|k+_+di; z5JF;5hIDGI9l5%SXIHo497#2f`mj+PcBZn4gwN9EV#otcL`kpt)-|@U8n_pcBxEIooKy| z6_u>VyuD?I-OHu*R?{PUxo$S{KUcWw*>%n(^Vks zYVdcjbT(3a*_uWF_jfeV4I8;lwvxlV?U2EH)$!*{on)D+fg8m;V{M@w9~yV?6|aFz z69&;V`w^3wi(C~3Io~bQE6$2p3RAw&*gqsGthqX?j6x9SB`?PsYFi(P?6;LyefNN) zV|B|Lj5!Xq&$$Wh7ZL2E*sdAbWymUtfR_i8*oF_@VwT8)&EF~tVsdp)T(;2+q4E3~ zU3iUF4u!>6U+LU9>khR@*aZ<{Y$Hr+;h#^~?o)5_&@~PH^*#&*RsHm-s~&N6aY%9< zF-qDEkS=pf*alH#4puYquc3evGg$nPp8zHg@$D`t4RCo~(rOg(U=T1i{3rYkXbu}W z5K;^O1gIc7LN~=(HR6rsX%sEk5M})EnVNBlr!;3ZZMFrQS2?1TF9kMYs?5+MjYkO| z_reKCAH~(gC2XU6t$Oi}O5`vvfSceuh| zX13fLw#B*^BB=G~WKBeNqtKGpcV<>~mX041&1_GyPO4(tN5k+0{>9y*r58A$>E-FN z+>1jK%J{wKIVw>)VVa`Kotvr!lbwSWdmG}>aVjXL1RJZOp*7y?kvscFO38Usbgs%| zA#FgU3JVz|ORA=TB)cbAi&h}e_g`Fz3SK82xjI>Oq5Bq-H(qI05jiq;-p~wN(qBIR z$t_J51QD-e`4o==q7nd@?68{5z1YvtK2Hd_H$+b`)?hjCews>(N$`$(m`}v+p0{|r zuVSJ~ImhOo38`%}QYrX6G0Oiv#Nq=AE?M6nl{>pua$9+03?Xq&Jx{-0{Dh*4LPwoy zpdI-j+gJRdGAinBQ15J8Qp-4Li2<>0%Aa`A00W%@?4FyAm+}>y; zj7aM-;0_<=BU@ihVIs+vzW zR8FGe^xJ*$XMpKW@FcxV25Wq&UW49cD!UHj9MF-5dy!!pn2JBhiyWY?M7}jXFKda0 zqmesPY^kwgctbj&dwd)=AJB8@;kdJcE3uO z&G={#hkU)`O7G5*!8uGssSBFUKFkNZTa#pP9vP+fH`N#rE5VAps~dVUMf`g)^bs+nWKp6jR4goJ6#k z5G08h>{iO+ungxOqNCc@z=4(1jisVnb<-d_t_v-NGrZJ{z`~T!@)2`rxB5(h7X0@S-un|#S{ruRqU6BaB@e@UW5M)eUh1seR|+)+ z=ewVL+-7qOTbAn^lh4WXJvE0f0)mKaE9=mJ8kxqf=%CWCIp2dq2o2!C zJ*Kl33D+#kgs1OeiJ$Xly^gn6hMqUFytY~c90si_^L86z9=%N9_L4=Zf9IZ-&r9&2 z8_!|o+T$u0Klg}0k`$J_!1nV0&H?L&M!0}N`_a>AFV(*-6+6~U>*6Xv7f#%m3+yG4 zf(-MWy%Dh&26fck1U}5pKu0;Q-DpNI^B4N!Oq_qjPH!ZhO_8n~U20|33=Nn&WIxEP zGdg3>Fg;L<;!$YBLa%<;Xpv;FujF-$if>;)`Ph=ylTS`_pn@Fg4DgLhxe3h=G6)9< zY2x>YV2|SbChv%W{|bVtIph2@=g0V@SPh-}snGw;JQ)I8+AAV6{Fw(K8~~D*7sn^m zIGai4G9PM_E{D5bwMOPX85(>8Pyr_afF*(kYvbqRzuSZp(0U<4Be(YWezo&wLrdy> z*|+2E%9zfqFazgTB1#9H>N3N`gOz8RLGM(ZL7$c$f#0^sbljk1w#fDb)*QT^efj|d z{Wlcs%#LTiEXF55kNtj9PjHqe`fe|V&)&ror+hrp`jo81$Nc4z>07WXDO_x{_&PYp zJ;%qq&Mxn873)mtAXnMM}KX;by5d0YlSAtQmlp;=;)cdL~vT3Flj zqT|FJGQHV{D@^x_3axRMClhXNPseC`N#E(<`<*Vr+h!wo%g;~X8^%|E0U`8FW_y;9 zVH~C`V2C*wSR(7Oz~^<@t=HZj;sK7+9R1-OiAWZWxCz-tU|?KNg&o=%E;rwykjDvV zbe|9wU$tq!#>**p!Z|(kZ$q6R=Ru(=f-)fZlKq&W_9`D?-)UHkTc2| z%VM&%2ZS?zJRd{BKbN*DX`j#6N|mJ&;$0(k`H%Y<7wokXi;@n_R^tPI(Vto^*8FTf z%F)VD!#7Ml%KUHgmB4)2GxPFM&|rqjgYYb@DsInPp_ir>i7=lqI_{V9EbLAM!Q|qS zH!G4Dcm~DEO3E|1=gLQ~+OI}o3ae*S$ysc{gCpe?5piM%H$Db?fddcd8|xyu)nGEZj$0v3-F0;74Ly*JF?A9eoxd zFtHV9IoI}YtE}FDqE&xpNP#dcV8QzK75{@~YwC>KJ!J!dGc(jAQO^7=l~~c1Q<8G< zJPQ6dXlshrLdMP4QJS_6n1)s$rMhCBz$qe)DZE1n_4TTXm?z2rpHx}4DSz#05bPej zpYAG9_6hKkU)G?Fz%fiU*`*XP@IYO2Lt~cTLzsaVn37dFi&@5F3#0c76LCY^A*dFJasF-22<=8;5SJ!lO=rUfW#bF9;xhl&Q-ONGEEG zzW3;5f*{fw3!vS}!UgCRLiXif*@9Cc{kHT*_;!^vP)ueqek5G%m6MqHDGVY%!rKl@ zttVS#Y$+>=;ra^I#~N5{D7Zu5@#U)udvpGcd0)2LY%ATD0M{ju0W*sE!NXl<54!JgcmZ zIp-3HA~l~H9$A`)4ea}S<2M4FE+=->v*A}~|og5!vA2Z+cPn8;`q z_D<2rLp^-At5#TK%{&`GB16`v1c3wJh_8pmj73+UxC#1~f78qJoV?P5)SCIsX{v;Q zSa0yHaeuITRW(?90twNzdhrND0ATzYMp_(;6@z#x*$Xv@KIp8BWq4r%HoY2|WUec- z0hl8U5bfO?uN+({pJngwnP@Z2HhQN#M+=EganKHV*-t`tnk)YdjAy4OjdDiL?E)x$ z+urGIp-y*U{{9Y#skosI7UssG&5LUA$zgZ%OBN2@BRv5rR&w$jI5q3$c<03z#i^n* z#5KX70dQMXDH(G#6RvCz-7>wC|K${*nDynIB_M2{sj;}yEWsbL?yV&_uT`ODPxYz7 z21K)+9HhjCe-A|CUGG8T9d*$wwSAt`{o&;KR?vUU$2(vraR(4_s=kZg|05;u4R@~E;n5wy! zj-wzKlmmBIHujtQ3)3+o3#v=8*oO4%Ujv)l*M&qAHlR8|_G zV0NR^!&M&y*+}Ml*I5(qk{i=DB{Hx#ZJ7KtI};Z3z~sOC#4xh)5)PASTtzJDi|phk zt!@HMLB1+NXMv2CQ^S{`F0bOXNyvaW?TLp^Dofi$Ippb)d=;Jk9QjBil}^6F58wn^ zOFD#YD;KtuEYWdv-K8B;!bJ&4q%LTTgAnpy8Gj6-;BM61#5~cBBPq&ar@uH{ZuHzD z0Lkf^rp-7Ca6UP5^+dLD&M9jRKW+gF=wokH-NZVks3beBx|;yFKj|ZOxmLFmYNWy~ znK@W=O}1IeHli^Fx2N3?V_n>ev$|baPuYD?e*aeTD&3R0yza|!6ZcVbuhoOzLiL0n z3c8k`W689ZnaZ(FhOGqq5O99yFKhVS0T2JVj{q##H6MYcX`%D*IH(%qyzzOQGZmbEGd2F84HqFgx&=8+xP6NfiUQ0- zOOSc&iL(<4*odDA)U@`;ia?XNKU`fD0ooxK@@49DgRKD_r&bk2Y7nxDB%r)2xB{)2 z^e6qpKe9Yhf3A>Oh1Q2KG)YKK4UfRGg8;@ue-IlWTjhSb>#LD?){H63e>vB-Lhzsc z;`*P^ytN;+G_`?pn>)Lqa+>^c<({&S{78O!_^NTjKdR!R$jb)=~$+98{(3(RxJJQOYUIGXx4}IH@1Gn}C7MES? z^C|=+VE-bl!slroO~qQ{kpmR18p7$)*tk58_z@C7JIr>j&UP{g3mXmN*%}3V?RT!i zZTi7!ipi+y5yS~dTc@&cc%t*cQw<$;m#PyL#NbHC$Hjl$bt(TgY$+zBJPPPM4e{&m zl6d7v0eD1egfG%MnFi*G$p=r?OcL>K@AAkOgF!~n$oKm1uyKC)Srz)xbk%RwZ8o^4 zzyJizu`GL@5sJ7TsS!kAC+rGmiwe9@ZB_st;7bl*~p(PHZIO0DdLfn7+|ghom1Ma z-1NU}fZC&uvcSW63+YkMCjL$0|NXRrY*qc+1dJvBLFqG4 z1LR8h2Ou#U#`l2r*O*mIe3I$I7^qdB-!>EGz{$PbiwQdJ+OGMQNDhezAw^4$Gs? z=HWS~OC<3um$YfPcQT+i4G*(E5Lty9ifQKX_e2(ziEBmEyEli)&F3-V3!UpeW zLsG{a&$#?h z7~fp#Nx~XR9FdI`64_624E?FE0JPrZ%~tTzE(7tNOwPL&H1r|_&Wv+_FY)?$1cV_K zP7KkMlqeY?>UG>xb1b;*`4zg)_14pn!`sUOR;Ml=vppTmoa@A$D_^!-Q>C3QCVf+B zlY=PSG#gHPurKKPyQG$6S9uuIwmy0UUE^58A5||w0137^AyA3Bfx2*d5WEA3ya*>L z4y^+70jhL5(RHO(C>in4aB1;Th*DDGXwQ>XB7kg!CB@XNloQ!yWm*P>(snUrE zU9GmvxxOpUPqBW@`8%0p9A{<4QBtljV%-swP-imiUuudKz8A{cE$(gup{q#?izVi=@g3YO zfQ0BONCA_hQV0v^JNsT5X1o4emhPGSYf-JMqu}RrE$V}$W}`Wr8($NemI2PL{MaMlCX`>l^ zUt47)O}3jGScc7~+Sj*OBrT!D^_dz<`C*~Cb*9nK+Zb+(BmcxMqIJ)?O+$^t6piPk zMbF{Z{QZYYb_FcrJu6cdLAQ6PUxVvVlxzxV3~antm`)}RW~>4#pazU&8O@wtOb7;H zZK;eE?0Zp-o?&r@YH*~Fi(^1kc6ywE14Ptm_+U1p79Er}fd%H|K|$6K^H=!@gXrVj zM&G#Xh;#i=pO$-k9T@4)eYq5VdKPyTaeDikGX9!d39l~I?!uiIlC;-O*bcghaUm9e z^Y48b!q$xK*=mBmn)gw^dH)-LRMH;v9FHIr6FU@leY*gm2lqW*fLA)bP4?3Uw zt@3(|2)$k+{B5f=_W~Fi7A+%6y`Plx3WDK803LH?xhcgfx4S)4MP3O12|GO}Ju$+n zPdll?BD^?gef;?OAJb|E$*%w~%T6bh_G*dJ24)#Q1<0eb&Ox8{H1=L!)$Uecw7ZQe z*rRe2Cj3$|8z%iZzP$^8AW}LG?cCO-r@cbc&xmK$5}ym2Q%)s5c{i?;{*Lv$aO2k& zDb&geg}iB>bpm3os4Wna4BB!Cjxjbn$r-*o0=?VGe0|~rbb%gK`m|3S5WN@A4Y;Y3 zNR{Y{m%m0Y7AsMLjP~N4rYwhjHOf=Ej{O#GwV6Gc?6ml_K+{;8u41{7N_Ll+IT}XJ zI=(_Shw2fvM*aT zQ=Vj$9g--FeYLPqUmqwQ!Tn3dV^>%9Q(I675h zH?o(8b!)c??a7I0)A{dd^-Y5jjjWoo9djFs{tPb-$M&%1de>ZpO%6k=__@pT9>dV- zmgzqp1Uc5v@#19J%Ehlweqp9-SwHg(V^8wpZqBRKiNMrJ++sB)I=rVh_=oMY;)Wrw;NS$H({1TF#jI2+ZMSLra$Q9&mI8eJhgT?i`0DSJyq(Oe+YfnTI54yqAqa__-hiX6JVVjHr=slX%u{&p8Z$N98rphg~v!+*Y zK98`OO?a@tpRiheGK;vcwfPUALqEs&h8R4)B9xgRL?68cf_Q&fJc$+ac>6DNYJ>3M zOZVJMPPrS?_{W^MIAQ3MC!Q74-y{8P(W5b>c*zCxZk2u){i1nLmKW%$R;4QPAn)pn zH{ZQ675}=nLq`nkX(E`u+y)I5ZuMXjx`1;ZjE8?Jfw9P4qNP9TOF6I&Wv;bc1%F=% zWsb8#8w^N&M&di0 zwG<-3OR62md%xW9*w<_4dqQg|K!kLoM<3(A3ON+=l;etwi@2-Z#OwavVEx>8x}=JP z>sZayTM+$D2mhJ)GQd4Zy#!0$Q*R+`CEPST<7X4dY51oZN-wDSB5F+c)@*W}>SBj` zmbxs277e(pBoA4wR)(dY#a(A1C1$6)KLaC8=v;vLUEBCsgJQSjJ@e&DqL7p{#@$S# z1=kN5!T8}!ZEO_WRFwCtR>j6-ZpqA-Tgo+`BIwKFHf#3$wW{mK+c_3LtM@jRA?wX& zL`J8h2kum>>q(*oMEu-O3y5ELl8dEi{ECtPt!jd%n|!;_2wunjUI!uTlmM4ayQOi- zc&l^jx`IEiO+PHCDZhn!-=tD?omp&3gbN2JMTPLw>U4e4FKqNx=)Vg6_Rzm{E~i?S z>Lb&Cv{RAW85Y3n{GmU=Et&&GD(;-7?q)yc@zkNxJ{G}8!K-@o8A>qhbIj-n5Z~M=Y^ps zLVu1-DGnI_&H}0=UQ-Byq)P5v8uT>$HQ}t$u)X%3pH$Vi6VHKU+g~#_Sp^EqIW9@b zj8klH70~75VTT%~`>d2vLangPTdV{ThhuMHuh`$wUr*woovl~WwmZ~#4_cYr?N#GJ zTTzgb6LS0Fruo8|0vQjG0nsy~2;L6Rv(6$^gzc~ETxn99R8cHAn+I&z3XJunDKT7N zxL>!u!-J_43qCP&(-jS{d(F7s6R!qpvu5RA`~AuMdlfxUZxTW=(y4y5 zm7Lk0to~PfhGI&V>y58{DPx0+5@3A5=?yA>y6P1WpcL>P_;=Ym9&K!|32bkIGo5`a zb7jcAOD>#jHxYsn(dhuIrlsnFDHP}os+&@5+V`YY3ImxMEZ8OcX8PKZ(DtFZzf0_c zLfm9sFwy6c#SKQb-yA2Y6LD{~%CjUNvgT*xFe5NJvS6F*!<<9xnh}t}1K z*c>pA?2xE5R$+tKOi@WtmS^k}AAKz~0`y&a`%AP4=4mNPdLMd5yquSCjzo9@zb&>Y z@N%FswDUt_i_)gVPBnSc)H32`maS>esqYxXE8QlQ()m8I_qqVJJbe`{#2W<|PWt>N7x(F@6zyj(a;(7FP&J%tT|! z!{4OF$#OcuU<1&Y+B=thpBcAVLkY!m>1WRK)-9mxw3ljGclGWVm#zH}W|Gy(}Z_>Sa6}7%^D^=Kpy9Weo>LE7mo9 zYaEot8T;6$PJx#(ngbLUKv>s|cLM%Q+TAZ3IDZ&A_GSqW2MWUF|M)GE{TrklIdh4{qRs0Gxm-GFgRj#nQ@RDVRIVhZ{}TUe$42Cg+O5ZP$*(VdBZ;E-(%c~K30Cn z;6I6PT7@bE}$S{=hXl6_&eW$)#%~Etk)Yr| zU`w+OgdAZ1Be&wWg-85Lsk5NdJqaZNvFEU2vGBEbWq|8RL77jZxmlc6omIV9IY zY#tGLrdQPvTfLpR*M6609dbLul3&KfzFx#}=A42bLY4oIE~%5E8GCdKlJ-(QNawN9 z0oAv1rYliUU~XnqOWqDWc63SNX$)HI|J-)Lu}+AE2uqNU*D~Q@P?r4m^S_|~;(~5f z6kzB7kBp4Cv()69U&fM+tNA*#csbEIOaFg6PWFLPt(VpRddSO({J>uDA2{dQ?=$+* zOXX#}oSWal1U;-Iz}|EX`prC-i6Zy=$_S@X< zU+w>mGN_ZHNo)dD8877Wc({<5jiM-m2}*jtjYtgSZL?-9b({(SXq@4@^+`F=5?u@O#*7R1V#2sT4=GD^R3~==XO*tAp~gG$xeUl) z0mfqOCh%A`u1Me3<2_J70FEX>0Fv8QSxy>>-ZKaJlEHEH>h#Sj(fCaujE>jiy^c3b zq1Jx(dC`HsGx&nSR8Dhs|1I6C2nM}U)dA$d>6qrJ`tmxnmiuw(R$O zuWA4jeYv-Yh$SpFgC95xw3 zWr?^zkD6k^kl-Tii1!yU>lBkwEBg4q2UizQLbKRG?O$Ch9!ti5igF~?Amer85wTbp zt;_lhq^NW&gEsW6aB}~Ji`aSL;!~4;*X1JF{dV$p=;ST_wtzcvIk^nSC88>-M{!go za>a(WJlzZnzS;T3w~|@^%lBSyxuyWD%aR#HNcLMY?rCpY?UNbm13XF6r!SwSfP$?3 zmo0twbkEgv0w|HSW}Yc7m0nYaP<}1@Ai-%T6=kSX>Z=I1c(SHd`M?YZkDF5q0E&k2F3<|*^1 zm?>oz%umSI=A~FpuKr_E4bBlr zKTY6f<{HN{W(q-_dsk2BkEDDb0^O^c*b(!RY;;WE&~N5x8W(#~(GOPK>uJKC(nu{} zfdBS`d00PMi3;DhD`3K&5lj-L$PvoF^D2=O0l+3Ph3jjtgb@QcGVQ2(KvNCgdO=%s znnA9tNgFHG_rdaHB$bGdOMiq0$-^PL+1UJhSC7&78tO6s*`I#6b!t)v83kj$RD?xL z(;3)C-l-(7>plJ>@E!9!r=CdPw^I2ynLk969pmNNBZ@&m_awIh};fH4!SYY#w>6~^ zy+tOscI4Qx!?GKcpqSe-*JY_|Au2Pkq>*-oRj6dn2C}hy>?AZ@vbjz{@1nKY;h6ad`cQh?}Jiu zbF$FZy_n>5vA5WkaI!PQy+|HlvzzC>A;LE|;rNZ@LtT!qoA04LWL;PMTr?T~?FY0m z{Rv(P5T9%{-)CjxNYx+$>gC}M=y?M$gpVr{lDpl^a${sxF;>JZ#OvpUkKAgP?{x^8 zXqxFPxB4U_#JLfq@NG8!*39rlbAyNK2J+=pjFac5-Q|f|8Z6rA?Y7XHBLI@kEccc{ zFZt+b%V%oqh_}_U`;E#fV5Udz*N&yS<}Y8jd?=sbVQ~e?V%DL{b=>Xm&4`8ugP#e9)2`093M~wEc--nGv5a}g3W8+ z6}TN{0E%PP&xwKc{aDyDTaMo-iTQX)4~QT@;kBPKnV&lSh!=cqPVp)J({2@U9^~rH zjF>1ODcZ}f#p>Qcy2Y1mV-oq5N;S(L7jNtbZ&_0efkcj@FN0ONS$v9q1 zt-O{>d9*(oe(vZJb$d%&KgU#eZ!0}HUOj_;iYPA)g3(fyIo5)mw(@;mbE~3mw#XPZ zYibDX{j9;-L|wm-qW`UdhMYL3?GZ zQeJ+`W#Cju(X;2|J~Q==KH)7Nv--imVS!r@Bx;}2rCyAIVFUz>HriE{Ok$`JGTo)m^h>rO|T2e7C-NmS46OP!?$uj!_*26g79QLeZ z8VXGmIF??Phy{H!{Nrvj2`5fh8P{A1>!pWwA-VSI;qk%~h&B}${3?8kE5s1M^O@(X zG0BIPiFNuA<%h|uef;J1qn$*Ow#~uHGh-n44$=38SDdw8Si8#_wpEH9n1tFYcMng!B~JuKU4G6S;LA$@fvf z&|OVrMsF4}ex>ab|0DlHJQ*$E;K^xZHz`{{#9v$zsUj(@RvsSMv6KNq1k7(geEVU3m_Aw ze)_b}hsj&L0;Xxw%B0M&D@0m8#lI6bSxT+^m`QA4sV$8-k1_@yQiT%5E8HC^_rY6Vv4AfVm*kYT|XiuP6Mg&ukX23A*n|2Oe<*3xZlNI7{(GZ$Ce(YrP;RK z(@=G1Txl4_ER@MQ)r(KsV#9SPYgY#u+HPT)z$a~>Lu4>@qrbHbT5^8rL~+aPb)jSE z4s`Yh3+C>Lo;PPVaF!9y>?xL?&_s%;P(B}2xod-YW=xR4^TmQ;Cd(r}1 zS!@2l)4_w#u;Y{Wg!@Qhy_a>2yAsPUEWSm3_iVk{6gW$gWManLS)| zHc=X$g|XF;$EOu4b}^GZj=m+9M)%;!?708CLpwX0QPs=ldCoe;^h`YjYkoxm$t0ZU`9orxwgdj0MM7aXOae^Mp%ze ziC3ILv_NH&;aTAj<+IHq$5_M*Zled+D|yswP9x5BL(;Xb8xa*Ct4$thi*l9MiY)5i>cIpqN@8uIql5!I|Xsi&#qH+q8IU4<5 zC}X){{BKc++im>X?!fNW%HwDSBw62k*@}zR^5}73P)JHj^yWX&NGQb-`9N$fim}7H z)1TyapjATU26D`m4Mf73P}T)|oJR5*?Eb*Z+HE;b5~6!?ipkimkc!Rbg$3!Ncn|qL zd8<6;-mwG;G9SsD0yVtTc!7H7KZhNxH>&t4y74hq_=%OwQTZ8^3mNSuG$w@cwFK(2 zvqhpGm8v@dNg=}mK;@CTzvQqkwSa>X@0c!jrc{!Y8Hpd1!R{zx?FFZB%-TGv<3a23 zj@VaOavsHoF#8~q1jhpPp7Al*8}P5|R`vG2x)fBg@2Km3iZC$(H~=NIA=*W+c$YW_ z#1=9l^xaQnBRfa!vItyT3Vx*v$pbl9hSVIYAAX7KrYXXNngvU^%0Mll*v#n19tl0f zm}JJezm{^!OFV{QZ*)u!YAFZV{?gsH6qMq#`g@1=D#i`3P#m+{pjtitVgt*!78a=y zBOS&8|M?-AT37sX+bqKpTaP>1MV zByc}mMB1kHNcFkxed@9Pw)k75!-b7uvIol!O<_#pYw<|eK;yCx)BF)^=-&!zZ~1!i zLjjd~*8mf$eVPG3eUS#MY#&ZHI)}gT99PIpO;%X1q=dAR$+0<9=mxJPe89DgUK1oM zbdnLt7h_{%idMwAgS!=ev)OFSOr(g_cr~1r=1D4dUz`QtpSp+hTYL-YmM`A8N6|@( zr;hQvXINlP>k^9}hXz_3QMbEDNyIBu=p51*2kiWUCvGE^u#}-#XZ@3Hm?ZNkY7*bX z`8@y3ftx0Iv{O{qlQckxqT6re6x#0xO%i*&AytIDpUg;hyPL&J z*kK3q$V*opeI*Yi+wLJEkTE!As$8S;tLyM;^1ZO(>w;osJPfHqqFfQVDLlwqC$)D= zP&W6&n#CCIVN57EdAl(aH6SoX31f>2lER_kACAmXZvd)|0fqg-Surhr9f_1Vn+lMK z^C01Hx&DmfhV_o27~#;8Wt>j^SltY>@e7;z0+|>U*V|nZ)&(2{;TWiI)_gNVS8rHv z(_oO_Y4>-NW(E8JpoAYxn=lEvSH}Ymje_=%V%+3MQyue6FQJG!pOLseBhx-QnLXi< zkVsQQwdgRU{tb_~i%x#UJCRBHoM(ZcE)UWh5o7x8pM?qWq%qA7KK7Ko%v;Jd2x0@S zZxB2-IZo<^e*utZ<;2P~qCxMo{pGd5tZr!f%|3}Je->%>&xV=N`aHCch^X}#V=cka z80%7-@Lka;c?@b*^WeqKd<~JN3R_;F{P`i`$mZWxfQcM$)X0XNi2gc#9(S9XE(uN0 z5M*s(pb)1<$&m5J0c2+dERU9$8c;wT{*|85Fc3py(OD>@P#j(Np)b=;^yzZ!G#gSS z=+cPOm1(OhtSfGt0s(qkG$l}mI*K*6){}-5IG#vJhIR#T^QGv;4d?sx3}*$@XJ+)CD=+D11iEqlr@oq!Yqx&(Ea(d9xATfvt+fuZWa(0r5?_hWyf zw^YOnIZqRQAx%F-Ldqw@okdTQP!PeJ)qqwG{aBdcwsPWhDfrydRxJcfTHK z@AQ9M9#xB@Si0_X^{o33xP6YrN1jq5_*}>5gb7se_ztRSl-?Pwgl6kVn3P)i;2@I$ zTwyDM$}e2%gBHjW#h2|V93S#U;u`$;{AnLm}A zH27{O_tKMfX^H44CEr6SpJ`n4`aXAey?;~8*iAJKw4`UIS;-PwjnNG(#p;`k?7*UX zh(f`T2MDt{s;F%X5G&M}|72CD6h3G`IA@7^AmmU1!}(xP#~u86v|2+>sOFOE#@(;d z8SpI!FWskD#VIk)TcVzvD{E~ktlf*?HG+eF*9H0Ij*A z4bqV^1MIHG_r;+k;xq2VsC{S%DlBx+&s68@6ljJWwUL z2;IVeT#KT>`=P^9F1W`XvsNT zxG=7%yn44i7Voo(LfBMENz(b7A<_Dl_F1#&3L_g$g#5{!$t$kPNcMgeMI=llC(n-H z3bJsu#&-a`m)AU7zZ>yAT^;_w2oWRFA}jd|fApcPW9gt0T-&|ZjD#H02vQOg7A|bx z5-Bpt?)ZLi$(ayhqr@z396t^weXCW8po1SvLI4PjsT!*#T5lkcbSZR6yC89Y49dE@ z4E(owK*W?U91En!Dh|Z1q+AUFEfk>$^bg38w|6kEtWICXZVH`w4fa1K!?$UXKvtwh z@d;%K1}*L?CVuXm_d^})ALYR_bAuG^TX-!eu|1~QtQwtfD+Q? zFk}IcxMBf=WfmzUKGziw_`0%KOtwjs;fN^(#Y8eNPn&eI1i!V{neR8W&ff3L1y_UC_Br;&#n_RUFbeJ?G*sFAjom9$8i)Ms01_Mf$_z(zTV zG6jGU%l!whmrtxz7qa*%OxAjjBLD)J{D=Sewks0!8Ac`W1@b7Rpr|YUN@Y0R zqWq(4prHT;NKOC6H6Hl;a*@Yaz&ikboERoHwq}H z$jy7HxU0@C4?#=Y&ORD4kaAYSL=ZYXo*e~7_U(H{y`5WE`@J8Y#`WVKInqsybuO&|N` z)N6#G$uBuT#IA}e!wP{0;qv{B)F-N9G3r)C732)O9Aj~fuWAdv9j@G;BUyarw!K)E z4YvlMFueOVC$?Yo;Gl4Sw+PpIB1HW6tb3Ml{5vL)zW&JlXy()zA<3V54KTqqM{^gh zXiZJ(!6u$ZK|R_~D0ToY1>Hv-+^c~u+-Wd5C}>diG)(0J(Lcd#RMp`XmeTBBeo@+A zcY}E400`5JnQMfO7&nb3YJdf!o@P4|Yb8a%gG&imp^vs*FmdeaGtdmivYJ8DaxHH) zuUIiQ(6F)z6$!qiHGxs(iz=Lt6h{T)vP>X1wS0m?6eRVay#^%peAFO(U;2Xd7z|h` zqk8jJQu4}JE*I6Dq2X5wYq}Zc&}Gc4ddr3m+N~+GCYk{mQ`KiEJNAqfEESFt;S-Gl z0#e&GA!(!jyfWu9zh^P5|G_ITcX8MR-GVl`uA$ZR?SJE+2@Uuc9N#+>GRq~ZBxSr6 zA5>}j#(fVgO$trx9#!Gqz=Si7&M1!i)4jj_6_Kt1q%JqD^!BRnEpzp!iuEp1Rcv>0 z0SMSXn?%5sPW2$J$0tsM-0tKI%Eo>@1df|2!1aQ}fthwJzW?w!skD$HU`===umEaM zMk>EZ)9fA)CI*0*fJT*?YYn=kRu_-*4;8YT?u$?ETH3UxTW>DK++POLZ zg_ovp`}uUeTmYwi4_DsS%QnRk=ek;LHF2EFMT+8c&7{&b z{Fk(V>^I9dzfcZZbewWkJk0pg%=g=j=2)0J+7CSdLA68Hs~#RnrO2;xba&QgZSOvj{Jy?IlzHk8EY%rQ$uS`MvdjR0}e!5U<0E%P>~ZgCp<5 zuKw2w2M8x9EMEh2P~P8kN{N2a9@A_2Lh|j3c+6nNLt}95)4P9D`)# z@{m0J-``6LO-DcnJDm6}CZxDOL5}q1LUWbkBj6yEghRWKAD@(^0XS7DL1)0+Rdq9G zU-4qo4ri1cHXg1_g!;Rjfptr)(XikV%V^1C*?4fFz39PZ8fU>H-X8Y=ClqJJV}y^( zG+=~(0wXdIb>n)$o|5I3Ye_@d8PNl2+QhrlnA{RwF~Nz`@(O}ugsu>OszA8NYJJ}p zSNG6isiaZc+&4(-qL$#i#NzHX(3+lHW%$|9)dYnXrE> zXd`JX+dr{OwQ(MPZFJ`Da>SL8HWVS7s7ywb`gd^9>Ky|nGkZ50Ba@MeqTvT^7@&{x zPX#qJKa_O54EZSEG|MV-|94_c(!N>Wm~}`I_*aIr zN#V~-(2>?_G}j91#4Rs4;g~ln);9BQ5HOz2qW^I&oaD(l{pcWDViTuO;1fPJboEtk z`V6p!3MsF^lEJ%Am!ZDpi>bAESQJfNm$4N;@F z`QR6s|2uZ^JUToE@mhgQ3O*zWC(+)&F1Tm;4vi>dDX-ItimfV{7mX~@vJbr{fcxm~ zLX!Sp=i*mpnVtwrC+HDbrTA5Ah{3O|_PSx!^`F^r+3||gA2&r`!KO1vrqS-J4RO+V z>EKGk6VT~5q;ZrQB}Im5kg9$VPx3%|Qn@z1)9VE`qp8Mijuciqn(0(^?3^pWEW$q< zy&lyD&$|BKrHc!co&P6vF%RN;z6#F#a%04imbZ&m;la2K?1Y3oqw3&q4d!yuM=whW zL(xA^`UbSSvrYS5FLAr6KmdyRpIzElquUDLKB37(SqbL)v$UD<5r+?FGSh5)+gRkfZ8r-evdrSP z-~n!Bd@Ot7i9&(pE|3|mZe6mbfFazWt@)+jUb`=8L?_I;*U4Gy&GJ0!-52SP$Dbb{ zd_v?cpD4G_pWng>f!pII3q&)eFGS(ABB933Yj)1MTO>8ohHGmFH6c8lof{Xqj?5iZi0s9?I=b9( z|FHp?D?zuMV?L5CU{g>5g`#oBSwMRce$0R)X?wvki0V@pXl!6 z7m9=((uIFbRggmA)vmkq%bP;~+T&$IgxFetRs4NnkbuQUH=6~{AAlk?&G=M59i4W3 z!sa#F`N0^Z=p=za1b`nT0z$L;`r?=at`E7=$>a;=61o%1i4^MfWyZZOl&1(uHHd80 zP3uj0laXQ+UBFZllcBfg6SCWHQKq=Z%ym%H@7F(<1{xh}-D(i#%F?&vo%R$g zXx+y&3(Yu2PA}7c>pYjA^mm%&v|AjAwZ)VvIzwU+nHGnY+v5LGYs5u$aliJ8Nx@~4 z5Wh$$84iK_IWfC)fe8L<5Yz+A#=WfK5@hg1iTxzxZm2@O6*gw`n*jae{1f(FS$%oI_Ut+`So4els70C5VV2TT#>x=$}u?XO5}NpwXY z2_|j?0GWi%A+}gwJao7?H{h&uWPJvPk4*)TF2e`@JfV7g1|3OSTQU|zF_asbnm1-x zy0A!@FMf=;r9Yv`4BWRXu9zaK&bG*BPZF5HZFv+0h}z?QC7#n!)g^wEzdukB+Pg}7 zjzjtdhpHz$B@kw-AnF~E?;WDTTrLes*U&Q}N);DOsZd7QWtppz(;{`t8a{n{y-2CE zE{ffSBy+RnNE0Y7`V|}ro(vvmErluFkASFZXod6M5a9H-Rusp>qostmB8&vmIUl`H zjYRX%#7Q~J9g4a5#*1u2DcArNqJ-8-^nys)15W|*9|%V;qj%nyH*oH>G7qD zU`y+j{efmJY5|?5jrzud$1r43mn^Y%>MwaI2KNw}mkH|V6V9Gp@l&AP)Ki7L+{6K8 zhOB&-)Vf7PnRUbAhh_f3X2Xl6xolR6MyC~P0jUOSrM$Qdds(X`)-u#_;bsNbGTaP` z^JqjFbU`u$L7v$|`lAg`?v)9&XCa_cxRghnS^e@Czg~mtGf>ENRv&oqjEfO5XMkH< zDMl)xhT;i1(jge2gY@i8{CWnAz9pBF*RNMnX4_{T``Wep)ceDwLSHa7O|MR6sP6+Z zCdnUo8LUzQVOf^ZE6g{W9o+{PI=GwSQ>hol%NzhCX>k|mOHGu17{A z-}|boklRwoSU?}nZPt4R0xbZd{Yx8*c+@t&CkB{f!!|8blGzMcX1%Du!esv;6{squ^^g9{S)t|YNBF-`G*3~m3JmboywdW**850Bx+yZe|r@de>!eLtgb~kBMaAN4I z|7q^8s08e^j@yAwT{oaLd67Ka4}E|y|9SVfk1g!U3$QUN&ov9lGwM61X%ICps<zSK&eNFVLi|w(LW`^XW^xkQKvvL1ZQvRYGYm1!uKLQ(ufj z3N7TR-vHab34|TNRr*bCfm*Q5HNV3b;*uQ6(a4^G4*|3fo!ySuF)B8kr!D&$0uG<*>c7OesgrlH?X z3M{D|7VDN)EfKVz<>1)n6Kx~9LL(Qb@2t@A$|EWeYOQ`}`Bmrh%AzLLAq%lHT8 zBVTS=G;Mm)UM5p=ks9Bi)RWFafO=XHXf@}|Mou9glFM1AD}rzTkNhB2=c!d0?j3$l zwcfEtT}w^_H$zl3Z$SV3O_Yb z2L{|3OF}-!21v5OY&FHlWxBTHV;T!|IL@-jG`W8Nz1}7a2vsGY`RUUW`0z~Q!I2|X z#=BO?k{7GMKa}S&%x@Cu^1S=z=L?JOj> z+PyHpJsN8UlZH2Kf#WO3w#modeX4oi$qT7gB}^!?gSX&;?6jVn;o|UbU%n1x3F$>X z_8se#(}XGJ4&SuLqxVUbPh@gQ033Pww2T{$#=whL4clhugRyr9-uOe@N4?RJbq+i4 z-DfMbZE-0tLl}FWy#|5pQ7P}u-|r^=nhqYt(QV`S=!(~c!%Lx7_MaQkiZ}N!^^RP2 zAqs#AepEmU=zb9@tQgH6XXU*B&1lk46Q>95Ck%wCdK{Q_gbIU>#wME@QQKI&mNLD4 z&p8FzqF`9C1=1X<^9hk|lM!t#A5zMyThl{|xR&R>J?3zp2d>omHk2W?eNk)8v0l0y zX;mi(OJrc%dXg2bREK3qtCIp{ZBRIGrYdAVt%LzQT0MN9@UDlllZZ+^1X?P(o{IAM zUwEa_A#CbA;%xN6uqb0(*pi$CpchG|3tU4S%aRnjy&qxa3Id)&?qQWAXV&Wtse+y4 z)PMx8hetR_%B~agfNqb*$z=~B3`N>`$u zuxqS=`Vk4Ep~f%uIpXO?lvr{1Z;nJ>FOSUh2UmDCyq`gQIs0nd-dJ#f8D1Oei{&@% z^7vwUy+YG`uB&A1-@P;+SonSm^>USL`5_sEo?=f+ooO*Q5ZN?hi%PtZO~&5QBsF7U zDeN5M$+EdKFPqp&ME_3U4+TXlNn&F;x|3R-(pAUJ=+RVwPO;q{MoB36cR>KHmyknW zu>~)JzoOrCK)1!}0$$^HqGo_`Ik68?VBRx01}#h_gK{44j&b^W!AJx_GO@zD$>T~* zq3SeaRi(LeK^)f11Rg6zm#s9!HBrA)-H3-hPAw>(^J=Ax4Vy20!#AA8 zdb$f;b^y%pN{EbQj%T!YU@dLcvYOp8`b7BcA|NYpFNWAvMdx zY#{)F_22iH*8>-a_fzYgU!s&?keSStbx464DB7ej*r>Z8jMZX;oyR*lj-?;HOnzwS zMv*Q`JSMFv1AR%{wHX!Ah?h(-`n=?ChD#>cBqE4~{A)T5k}>=H54|(_3IQl1R>nMa zG0I##0mck=XlSQ>qSeoHgS|6Z=3$H2XD(V%UU<#Fc_R@( z!PTM@tsJ%JXtobpU?hh6TI(tm&koA(np#$I<{0UCD|pSl7Ir16jxDz>uEcxThS5&- z?1cMQ#py!-;$V1&ZH#M5Av{}w4AJ94?9&6=`q&v(-?AT@B*D$Jw4cxKP5cS{+vm~0i zj$J>~m1wrhx;gSnoni6rpX*|docUFBaR^i$%w>AOp# zWQ6ie>q4*ZrF&`Y)g`RkwU>2&HYo0(!?@rIwrZn@vW<;iGy!|%s^U@l}PU;Lxt$3uhO5I^tOI#ArXT|cAt4s&0z7fXw6{T z$}DaFkhxkg1E-jpbP=&{>C?#gS#pv6mu#yO^WY~cN9{wP1DKR=;MCL|tC8ecTIq2G zGvA+9{k(`fLD^)=oc^x)H2MJLjeO4!4U&L7{MIHJ`FBC^nA!%Lal1wL9Q5OyXHXdQ zQ^hmHeoV5#(bdlu*2Q4-OQYxFCaS+RKJ>~z^|NL4#}UXLDEckXT8rB9vrKg9w`41| z)0N8EK-B@Ma}q$AJIZP^s66!84Z#6rs-TUO;5jR8tXNRNhl%Kx7wJ5)>0-<;JsDYY(NGTCaaosHKqqmd7ZKG@)G%tIMI5-TieHKnteO z!d|xU_xL%X#cs;Z@LGk|{BwVU-T``kA)41mx)V=N+Gya0_V2M zNbD4|Uk8O*^aRAE&V8ebL+EW;Ip_vXyO@gok6t@SXsg+0m$z$HqnY8sJhQXc`XW16 z_rg6Oq<7WdIa67UQooob9@Pi5_ld&M;H^Uq&#b(Mhaylg`HGT99nM)7Y8=*PIioSj zdCe?5eW9_pA*WWcmP!`mXyq~5=$J)tM(0R(tHSxLyrPI+M*ZJLP(%bS>%Bn*;^c7a zU^{OrhG;IC#F#MUx~L8g?7U(W1_+eSSPZf;@-vZr4$|YV{lXs(cplbXj(~CaxIjUX zW=E}N`kgy=+q7FX9~uP0nM&qHYJnPl^X4*@L{W5bx*LL4U_X6E4xOU9`XvzbE+3W! zIedR6SOU@t#nfb$tfrjP&X`EoOaR{=Ot-koFbtU$H-iGJ+ujw__5{ z)fGv9(j~8V7Ch%MwC6@wWzozh=v`$$>!qTtNE09e)R!WUPPXF)IjMykTcVci%|;G0 z?Si0naVQ`X%?C!{fl3+KFn7N{^efbmLxDSh+ffT@%bGtaf`ZTxKE!?yT2*7!@NYXM zQ=yLJSu`~5k8fhA=# zrNrk1XSQiI`9Ba*M@BRBs(2M8A}Ut4bFyGgQMtneO832N{ov-GD|P@vcWd9R%#i6x{hKb_~W2yIE7y~=`M=!692He1+kM@ zGPXw>uQgT?Ae7)FjhLXjLO)6wz0L3ng1q+gbvabNk7QaJ8Xl1=UjAM=w(E~s&x!F( z(m`}sf!o^MN*~aDzb=|1>R)E{&Mt)c1TMwHIfD+Q1gn<%5Li%K3*W6rRBKTFvH8^- z>yW5%Y6$?B?cvjdg-DPXiD>%4dM5Sh-UF5W@+f4d$1aLTa%}5|LU{ZS#;Gt#c<<@$ z5>=kwi$IQbq+UAD9YYUBh~_JYyAKcP&ghc{>mXS{;jm^`)Zk&>R)v%1Wx|@~LL!rh z7!vW~|9NrXWmA>S-BpNYGfxouW7mmm&UH3!LrKiOzIbpu%!kC=#H)QPM#i;Boa>?g zGMCgnrh%!65C5ck_63uRD#;1KcN`5L3zJQ;zJZx?dnZzvWss41OUl=5C^qi<9_H{- zc{9KgGqql&@=f@<7vuh@O4q~TYp%fgtH~Bl?}cetgT1nZ@37@a00vvWt{zjf zBwHYFU-$lhySZqtnc1})K9dCRee#hSS@Hqof_DBFp89C$Px5H-p&CqD7WYaOA<%K2Edh!3=r*iwV5or;%wri#GaPN`c= zr|rN`c4J@4e-poPhkOyzPKt7>usc3=4}R?`s~(G#JnLx==VV=Os+|Mv;$StcvCLrz zp=1ir^A})Z!DBw=hIgs?k&rHmhqpdCVuqIod+?C?Y{%7YE#N2~gd+^$ia)XqPts&# zX>Tj0VlwwmY4xA&NVIKJZIJT4cy5=-jGosC4l<=qk@cB;a2VMcYwve?#}KpMzeSM@ z5ts)t9XF@+CO(SDFr}bg$&FG(xN?V~{y4Haf*-8gYUXJHhOXLJ&!drhX~O$!93&D+ z#>6Y!OzHy9IiJw-wN9W?grrGcV4DnQ$k0r}IC5Ni&v@n{BixTXTvXaPsZRvu2=-xz~>&AX%tC zc=~j;*tLzFAvsJR#vrfN3519+cPKc_4;)bh!Ad7|eI6OAfW<5FZ6|k&m|+V_B_BKa znhWSIt2{GRW^CPhi0qQwa^%rnmNo$%LPR5va!b}VeAw9PDMW$TKcern_&0$lY~Rv~ zvPA`T@FIn3>T*x26Qm`Q*``qo$8Wuj0PXs4Mvfu3D*@VuHlt1H&LN%pu!%~DQC-^i zZvRGmJUk$+8u^ME9vLxBS;QjoE=wftu+ z`Ft*GHcm+WG8p`z3sAR%FD#TAVj*)RM`WsACQ!|l`^OcKnl~K1E zXQ<`Z$zHGrO-H#bQ5kGVxQ7)e?cC-hM;m{J8r>(1sM~2Et+2iIu$*7(u?{u$%6$(c z>bAyS>D2%{Val4~ZNA@kcB71>((bcJb1QZWRxGR?<_KRA8mM2r2|xGR5mF-~6jOT$ z-4e+A5#XBUMLJ^C)oubCfOh&a-gw8BpjnIW5JIDOO@C&({x{eQvaCxEGYB8$`PjAn-bF^c-#Da-6wS=pvBb&d==2nZn9L^BTjt-u6?XOu5qTiSOH6c}Gb zpQyM0Q^w~xY)ZvK(_y1yLA=zV{zq}!_Y{4GXo{xE4s8F**+y!DG*HH3v3|fA{Im9i zwOIQ?UC-8x?m+ku1F;2A{GWtxs3}j-UaEGQQ47Ko(6H@{98Ks?pM?S*x6dxIJq(m_ z$v%mm8Gmw~7v9d^kka}&iv5XofwI^n_sIcE0qGkJLZ0d|@ze$W<9GK>?16n^fcVk4 zk%Caepw0f(j<4z8U6_@IZuk0GX;<50mV^Hz-m`^yzfEiZ-g5EsgYSdW_nztNx|(;m z_&73==e3{cA)2qId*r_Uul4z6to4|g5GIf)BCSS;N%(>VxdrDiy$TJkQF2A&aZY+o zsZqjQpeH$NB}Ezf)TcXLM)-L`q@y66StUoiLBY}7--|*>`)chugQZObro}uK&v_mN z=k^o-Dyh{4)nLj|3}IVE6v_$_a|#YEIFe%A$F189wt^!_Qnve^`I#H4<>6;JN>F9= z=N$tmcH_~m_Ix8QZ}M{xDISBy$brCWeG&fG{VL00)0DKPnTnTNRxOsUiS3484M3BO z4#qZLuHnGCXdJdJqAr;{H9L7fOneX<$VDH#i7<3&+|z!-PG)3>!(#{qQsp<_)3g2n zyXc}Ht1uPPN~Q5^;=QgkGhov4eyE-ER33&|g)$|Dn1f4d^kMk6_QPU3J-z=KOo#H|)XeGNW z0K@0{l-?X^pNpgnI^`^IkZ#V6t?Msv(2h776bPs?CaupO-LX2x%Ev^&Eysqc>vNF+ zNoo7-4HVy3o_Q9v@lS$t3HJWKY&88u76M8i_p%KAlBMt>>oQufo_Ytw}T>v0eY zve#+blk*S65)%>M5ZIX;G4+|pu1vZ@E<5MRdDIFR;6tvYjAUH3uZ6pBf|1735rHKY?NRAg|P1Fl>>MQe%+>rjldq0CWrKpNBR%0j61(UML~7C@<} zdgD0~dXhd(!CMbpuvXBLAd5r=7F%zwV+YyHZ^s(&t0|3bXt7T8D3K@@?liai=gkO3 zZ#w#%L&6pNf+1Zg##EbWSkjbg1hJWTrZR(bpI3kRg=@NxZ^}e6Em-jc2RYUTxoZcZaO_qXuYFu%edpu zH-W6}$WY8ooG8U^nPiXNzg9#xE11p*+X7_aCVydr+e40guQpnB2#(HVMzXGu$zKq1 zTDNx_Zo_A$Y2Vee3;=Y4eA0`Ag|?sP5Z3JJyG92tl$J+dVZ#=_9A7)|$**Uzx3Xwx zu(#T|tDjM)?Ex}p?h{F7qq#`-hhtuAS$Al1NOoAp9W&yQ8H5# zB8~c(f3;OL*7X>#WHj5+)`frfDq3-(v=y8+J9E$!SSHlEqe6wLmdunq#G%cN1$zg= z11dVIjEwguFNwJxS|El!ff6PNkLrE_!zsv=qsb=m!G&#=jr_dznmJ~dV<1BeFlsNLHEBKjt%)C$?xby(3R6Oebi z!-e9g**b2^Bd1tHjjWZ?LA)eB^QX2P;;Ks=e<{g82Z;`8DtMQgXa_w2V%Fg0o;cNM z;P7N=S~$!yP)UC3#W^fC^O~-JGimg=^6KePL?DQS92UCU22*I^%ZSI;xh7$0d=ZLG z?*0IBM1+d>hLDJNMvCq;EF$0Gjrd@3u3_{82CldjtbdZdY%JClglr;)FzTr?e(A^=z^ol;R7nuEC z*Sr?EH>0ab3h08W zd5VEFaaPGg29jVeidqO%Y_=>*o$D-WI`qsVLHq@wu{!V5=fC`40CYf$zY_4~2RHUQ zk5^U?H!rHuqiMV8ijNbQZ@g|Rh-1NZ1yS+G zX;WO$4qm~^ytsi^^oUmIoUhF0)C#!jFOFzxor0*~N-69s;Htmqhy5UvwfUSzg+%Q0FXj=QzXkHdVpR&AbX&LRhP>ypctRQPeZXtU|!oJF7zCgo&@Rh7@c${+S#{QAK=&|*K zas*3%gHgHXYsi$=HB-@9XMW1-61!odi}ONGmnfjazSI872vT*A?)7Jl2#SVAwQYjp z7b&l6|0#G&%*`uNE+6X9RVJQz^1b`fM>MgX|SVez_#~<)1TpaT+Wj~AW zoPIiGxL=1Dv_I-}TNyPl(Rj*BJzYLTQm?~%M%NS6LX1uvLu%NbFC3ym`*Yo0ew!`1 zQ?aloIvPH#C_ICn2v~q*{#9svO%--(_-L&or9YvD9(7(UF9+Kb4W*|i1yE>`{Q^Y2 z{V~IP6{c*29X7hbrL?SXET!q|k5jHHe_`Hs9p0L$WM#EN&R)VO_Gl=nA~4R`Dn96R zWA`TH)lnjsifx5(eJL-FI!%=KVk+52UK|b4Q;8P?iC)Ny5@nP1B61uO^ndcw3Aqlv zHoiH-<%XC>p`zP4u3MMz`I|Ql6}^*2yyqXfs5pQ?x6ah61i=EO-Kthm8d(&y#DBDP zDmVLHrn0lw0v5i0G}KQXfiVjoTvw$sFKIlPMb&#B8Sgp`f5sJlk*barO4xQ@qH6HQ z-#3!$3hTI(O?B$0oY2lz3M9Ga5{E-U_2OZD4(pn!Hujz zI!z&=1Oq=2Km#sU<=2M&a8zf+OK`GlzkftwfpY0@!e>~w+?d5B4E@rru+A8!R8s5I z55u19ghCU64yZZ|iuS9C%|eD2ws>>d2G)nDFgm^jS8;UnL8^5(hX%trB~M|HNjU#| zVh2b%wXUBl-u5n~u!tEX#wnY7(lB;S;;x0f0m*_tlM8dByPT@!Cju)lh;&EdK^dEV zIE>9nj)9;&i>C#dPpzK1#y{fD=i}e9F6bwLw`X=N41n+XV-iqg!b)b8 zzIRKzX$NIx4Ibgm$QsP9W7&4C>xTpCh9$C@#a)8^64^eqkC1@=uqB22kvrGM#?DQo z+_$pP;f5c%K_CAty&mMU8NS88H;KRa;Z3DOhOukR`6(H}ed6&(-vd)y;@3HOLBpk&Rq zO5}R?&9C3fXMTO}+<;t5ok-AMP)aC4_kZ#JnGxbLzN~q*L zYbCEl$(?(4TS^YYdI8(I31;)tCt} zj)QvF*aT{ROFX>7aGeU|M;XMA%}>WB6iG>UmK>0gS2rvWC3 zl1AE?6iSDoQBDz~M$cC}b#2vkPF1Pqfg_l8xob~nu| zKga{#qUC$ImB9sHe>1<3%Dd31&=Rs5^C5Z#d|gd#)kI`+ z@fwZO`ThBOBNGR3Q%6_n9KVMB=(={QCiOSL*&1CxXx={I&F$TQw?0%gI%bresSt$2 zwDhoAD6L(TacDV0l zQ66oFuyRD;)n96SY&)=2TtOjo1P1ttOXmw4jFVQda=xVL_uV_iw+rsuRPXND)eIU< zch_;O#v0R4uSVf(D*QkH`zPowERr%a`122NBDux{-`(F0;26HUzPoyQWsS+tVjuoU zb3o-6ujS}|xTUCr3WJ4XjbIaOsk8MTA5>OjtT9E-hmzV)<)_!4`BE`>YD#8e+y|vj|qsLm{k~Ww+KF=LkKe8d{L|3&}z?PUl+)$ptA+g7Q(>_Y1-Xt<$+! z_7`YjKawcHk4E!LH69N0f@ye5CN0L+G6Q93|9?cuDQhw5c{!F%;fY*`crrAiUGwpz zr4tSJ>$coxwF*03lNU8`VKKpSNVr@t zlcs^QGk!l(4=FlKSIH6?5RfhX?p{T2PnUBd27^$Ec&u9!j>1XoL0nE+4=Y0~Kn#Vb*Vh1zR7lPS+J z>SGvP5FO3p5IR}Jk78z*JFz46AZFT3uG7b1QUymjy*MoUS@;-LwfNl9@=O?0m#{o> zU9Lj9?3kW?sJSNKbj}pQh)0_+u#T)cYWE#D=icl7>{9UZ%ns!rCNGm~ZDGx$ZG5luWXQ2loe2 zn+dv?e;9e^pf)!pIQ%O_&B~wL)qNoh#0i&!<(h$IQtI%}P(H=Kve9zI-g-hKfP`a}8MY@38*83%Z%0dp;hIer~yH|;D|wdYy2{~n1!IRGdDqSy9*)U0Y?nx zOCTt$@Y1a6{2xS+iEE?^hGU4|tAk+e?)hNFpZWgpM&9j($|HKSg5c{=D6VCv0{D}4 z1J#{s4C#)#!H6EQCH%p%WQhEH&RI*h4ginDMk7Y@IiMA|R;MM`aXeOcZ<$++7{IqP zw{XCGCfG1+?>KA=5^B=zOr#?@%z%M<2{02Tz>eIY?MCo#gpO8$2v{nw<baJSV|Kb%jk)pmFrmMTfWaw+08Wl2{YfF9`JahEgatj?h*uyh8b- zZt=*q(Yf2{e32y1e^A8IY>U{Vb3KW1+#-^vkE}3kB7ZtZk*~^-zlq7gct3@Fq$4#X z&&K2kPuB=r72M|b2ExygiS)QuurC%1GJCDuLX|zZJF;IS>DeB#N56JL(nHKP_)+ce ziI`wYX_k+#W7QWD(e;>57({+EqRT;E5`A|@WJXsPc1n?cyq&3;f9oW$!;r&-;1kI} zbn(&d5#ZZ6_9_p{4T0WS+wU+1am`bwG)!OvI8lyCf66VJ3J|&)`QI0mXDeohiBM}i zoM!GI9tJWtFQxG?MmvIsZH3E>{~~pg4ucUBGA0%xz>b$IVwjt_W{o3a36x%zgyHJ& z8VyInLTJ250mG2Rgn$JV#fU!Dn|_>)!>q$GmJ#Q+VYZz0A_ce4h7JQ6lg!XADWE1H zBn*BrB&=@Vwxru(^hbgf%+;*#ps5N|^=O}rn(slX`h7A_R>Orhvl8!B978>uY7Rz_ zxcWy}5TL9Xw|F`Sp*5J4KwF~dYFtGE+JBSqBB1F6mk%hvF=!7f5UzX^T3>;3Bq0o} z7T4RFpB5u95@LH#4SEc9)4LZ}q2{=_TCV1pZAfbQYEE-(<6~-mbjWt#4Ey8{d3jgf zJKkb;VS-wOEEDRb{Ru!FzG4$Sd5;v^8jL6ml2|J{h3%1Qk84ZD0P(3v$ZPBAiK!Ci zyLJ$~!C=BA=!-*lOJ~X{Z2-~Yo7T7lUAiyQ@#1DAd~ej`hWOnH!V?l{l9|Aw>b#Vb zwE-xSn-a5&K{+epFK*t@?3N?6bY&VL%Z-F28Ar1rNgeO$(;;qXiB{4D-!KAJm-1n9}Iy4v=a5_$owI z3u=t2K7u^$%O|+OBC8jso{~=4VBy#vNWv2M0Kt9S9W0(Be5^&EPj$AbY`j@Y>p<58<=d$xbY zc6n&y!SZhqXk()jK-&xH7}IQU3?LG*f>F}~d!ptz4K=!N%p9k@sjW0bDtGeGem%Ps z=Z@+t@$YtOYjBifA{~kNmpYg`X#irQShW>+e@;)cG)$q~(eXBd46AcyA?p8^jbVT2oW|@kOcu4X z&s2`xj2IBt&5IU{ximu=G3o5zVA|jPYgA61?xPlsz76gi;+rtw!5`}O{`{@1#XvAQ zE5sXN*!SiMXTDkZU`3@b7=4Em-l|r^F%u*39MBpJHJd-BH}+Ql?cKY8bEZJi6v!|l z8z_f{7tYB+#8TxDPM|}S<5P{d+9~CPjxx<>cs_)=c}8v-@N5ieG)g5Ju$pHWcF)Ih zp_Sb*ROm2)k0UqTl8aRe-T%+v0gYN7c*nWXAp}ZrgWi}GyL%TP4eTmlclN23q#r^U zL^C7|4GG1==IcIa6dmEN1K1q%(hK}(^zdg<%r7`XP4GmFl)%%+?%sbYfUp)3A;3X| zZNbi&AVrfFHQFl|AqXSRN=O>55|N_1`vySccdNDn0T2;X0>B)&XqXH(&aF@1#yZkj zqQ?b zPhXsfxvyeoFrnLRu2n2Xf~t%J$7aq62{7pPR+3A41sc_3;*gmwh9$SXA|8;)G0H!G z2(`mepMT^rz10TnIbAy@C~Gl2Iu(>f& z(gzY>V#hI8Sq)ZJ>wmn8LmC&Y=8>GofJck(I|C=@F(F(m=P}>qt1{{M8$)yv3HiJJ z$myK&xf@gel@mn5q2lp&M4r(MC>>3SoY#)yS+K=H!Cv%1PEVALJ!Pjn7aDBf@GJY~JRS z-wK4a>(q*rcB}?W0B2@uXI(>j<`Mf(a|DQ4`&byNXY2!m`GGt!B{XEE>iAJ`Wte}| zuc23%p8nHYe60@bNd=OOFX1Qkdy;vgk}wQ>Z^8dz#4JQ)#w})+BULylXI?gyn1dZlD;I}Lg30m%?z#X*lR|J!N@JV6ZH^;H z62y^}F@8;;bgJ?5%%D^Oe*1&5I~fBR?agh$PF>vf)Gm+~**Mea+v!Z0g+aOD+$xsQ z`2A$PJqNf8&P8d0r#wa?)ivNarfc?LT0KyXI*?r`n{UgVrmbC@AHk~9PSUK363$nO z<0$`0n*--kC`~DA(UepX-Oqu61C|(?mGW<4-y5+zWg;74`o<>#nLovxtq*-y(s_+$Kr! zPu+<=ok^I0LKrn1`&eN5u3zt|%B5 zShv{_vTb5w(kV?!UDh^668AHYXyK?dTr&N*H( zzZPV^%Oi-B6Xh0zbe`b@XT%|+8sr?*QZ(IpT=X2<%*MVT52woJd7Nh+j+#>vMm)`(awv8VFMN|s6>L8rNSp~L3B8f_1`Ef0#kYDplB zypp&`a+OtSbXIXf5>*Ax_Cv>NT95}U$jtF_*|#!giBTxQ6NnNm7Q5;2YbQ!>kH`0> zBzJI^<9vdU<28sGd`1v6O-<<#wgILit;`^x&NJ=pt@Do*($A7ess0X4t^|&ry@dwN zU+q=wqDWGDdv!KcF1=VbuJtiWxY4)1F$j#{uVHcm~nP*wtzyHJ1OX zHP+L-IvB@mllla9v|=YP*27+Kn0CQ&Tirq*>x)_xtF}!}I5uif`WY@bSySsx`&xZ& z`;eu<*4ex3RUHB1kbA-}uN@T6&Gv{&s?3JfwF+rN%j--^?9Az~)wZkAZh~M0xL`?O zJE-GAoDO0y`>Clxinha&?+fr%<;@pH{A;H>XP zELI1cp+xol`McjqK=^yw3|bLsDg7-~{4MRzALmGLoZ-_4ioyXJres_;pD+$69w8r? zpnobKfY{_^1K7*0db)=(b@Neel%}nrZKbspaEa|YTYF$vJui=}-}3(%mhxMy$p=q0 zVV0jl$%Yo2K7f%nKV;W?>+mqa^p*K0alE^gksEPj7@d^u(t03>B%I9-B@$Yflp!0c zr{Hw{mZ=1bN|ZPCTD}|`ey49<+O1VSg(H`VPiF&cbre`z@)F@pU$0Dxrj9EqR8Qmr z1tms%Qk;`Yqk*Lpk5Oi}Zc!RrnW0WG0B!tWzUy5qB95KVFD^<7vC|hNMcEk+5dqS8 z39rr0EWAdYi{V|A;soBnMM&rBplf@~&uaMmzSM*8#f(S#YGc^&8HUn^ElFy#!z_@q zb6UzSLO})ytbw4!td%Lt>c${`Hj6f(SKlunvo0P0nw>lmGF!<#ZrLKjF{JM{`j+kA zE3O8n@Lv!OVhf0ieXpC=a!*`O{&BGmTIF|?3{P&fMPqfQ3W2EFiA z<;Hdm>I$ZJ=s}tNGkHtA;)hhmUFEdM2Hcej!`Iw!u9H97fPjX-%z>86N9?MkmJhO= zF_-k7#H2r9EA>}-1*8&pIiRIq5iU~FOcIVbkfy*v-7+|xw~+CJlNit;pE;l#f6Z;j zXL9HlBh^nv0CC<*SQ$yh2%ZUJ%rv51z7$5|Hf<#q)hVE7kwR^dFnzn1)Cr9v5DtYF{waDcrl3UNRI^fK_q5;iLtkNU?Y}wBAKa+ zn)r65pb%db*y4T)tC@{9QIfg*$xh>wiM81AA(5m5WoZJdPCiL&7P06$i$OBPbq?r1 zk<6jSkYadS-29J|kiSKN{OmtPIfj8Va`P0)#>4Zl3eQVIwgkmjlHE-Ww2Bihiw6Dk zqczLfP&>C&e~{!BqMRf?vY5FRriM?OT3BYcy-0g7{b8)k)mUQa{?0(gcd#Wnqil9k zWyw%=3&-eA*(~xFH+vNSLJLB1`HcZkY+N80Y6J-LA(Z^S*`RYkI@EpXwRG6)sc+;r zUdaR7#PdD}TXsy)?omgiEDSOvGup`3y!<@JW(6E9+cKnMm9E>zodG#u9{{GD}9@Z#3;F zXAG4`4I9I`QTiUM78j?&=T|HD*0dNO-PhL&|Hq+$vCg&?212C|3P_!Mp*Kzy%xeWV zkD3-o>g_b)_prCbKM;I>mWtSGqp>XZa{BqF{A))T5vIC>UVMpnVwWVGT{3=t>w~*oDEHMb=WN#fnKA?`Rak#SJoo>jwm4AU^Z1@M@}r zc3@6KBGN5zycH zkB*q=*D6r1rLN*F8I}+-D zCLRnPg~S#MP5R>`hh1|ktHc8Lo1<9F-OaPod!V8X?Qu)2GiaW9F@VsjR`AdnjJCok zRT@L&(^0>JncR=-9N0TiI-Q^EPBO8D6EypjO)IWJD{Nts*kwXlinKDW)@fm)S~Acp1I=VY>3)$&2tVP9 zjRTsw+rT<7OB`Q}R%76nYm!!FAZqzKCRZa?tty&D8mp*yMKn9Y#1WnN2`2jTl@H>r zVyO}5SqWWqPT&cf^|)?Cd{h@l+?H`@H6I!z<*N*|Lu|xiQU|oPF0vY0Bv*4_v(s*2 zVx$YuHm%A)yooO2(nvxV|NP}>h1-M<^R{VK2HFw1!u>}Vw6$EZTKUCjHDE1h5v|HV zyoFq0B4$dictlY|vqMb8Y*vS6YbN?OJ|Dze+4un~J`*$497yS0qbHCqDJ!Z zBF(1k-AW01Tvts%=Id!1RwHX0v5;QRyj3UR?M2B;`ru4)2LFVtkpo$wqBFhaAY_pu zR<9<6$B%zPR5XZ^8K>MH_EZAKE=^p6^O0gamO$wC33hW5Lvq1|!|(mKiVPfci&_&}`7^3>$8cDqZ8 zaVQIE83&IWWkZl_#-MQw-V8;&hAqerw2_ zLOu$Zhd*- zGcIIWxpc&FtHh$ghLi+kY+`aO0j(xql#nwS=G`!eF;^{Zi$}(G zN`nKZw!F4ERUl_7PvkZXhYs+HT|(0&T>{mC@Yo!QklE`fG3cEU>oGKP)2MO`Yu~lZ zp^_#&F{D7T@J$;@!k2>s-O$k|)TnL%7cyK6NuHLl$Rkrd{-JbSg0 zZz=}1FmbA}@-YgmlQ>y8+=_=WiZQu$Rn>$B?EL(lSk#hfk&HK)5wAZEhi4i2R^oqK3?$b>Q`wrXH^29yp>4c&ZQvuLs1IAnru}J-xYF^Z^h78pVlH8D!BnL6z!y5lptC>%8La2d`sYrCXC)e1y%DI;v672 zDY03>J`c6jqa%RC+@7={@>^#kVHb+CiAZ9pe{=l#l(g&^pRgm+Zn4I~ zwdU)p90i26RoW3g!_R_uZ=-@ zBU$ddVBN4RyltgN$o?NT_uNWt?ghg(53ZRoTx|Ks3sQ0n)cl$_A&W*CF=R3!RlubEpJ>%_Wfl| z9w>fc^4d;nFhq77q8c>I zHe$*Y+*Y2cPhsA^kY(RQIX%ZBz{yP9HObaG=&)OhV-CBK&@tZ-|8;{)%vu}`d?om)cg;z0siTe6#=RSA0!>9 z)(7FLi19`kR@PJ1s-*mMw0E=m*Q7GAwGe!sFa>xH&! zn}b72{cS%OUm8Ls=_+6>)qL1?RLgwTeAv8|abjb~WvXh*Vj2qPsFY^G4tr*7%y5Sf z$V?If<0;GOA6&|t0~*~NVI&}YhKBPOiC$P+MYNVAx!*C^jwTabW=tB^>P$@vkkJ{E zx%k~aidm#d$3yt?I{JMne#yURwKaA~g{Rf6ahmR7%8odf7fBuo)8YJ{C6WX%)y zh!-=!&GKj(OCb7upZHakh(Tqr3UzF7po3}CHG1?D1FD-=WeZJ*51?8$U|h8N0tDmD z)ScQMk&@K~(c;`IiXY}lYmNPQ99)k9(BO1z$a&d7&8qK7hjm{1&e307SERbVK4Wl>ouE z!UvmMfzNvB6IYM&poa|oMdD^Y<7l7BgCpAK12t3jS_@^duojfXqSZT$!vNM8UO!%1 zJcE)smaYOxG-F*m7)EUQ1_^r*49E8uP(N$oYKIn{=>non>WBuPa`9HPl884kAeL%* zNxmRMU<_Iy@OALii$5efl$2Pdxe%|*-KCjMS_n0%S~TvDEyxE6zN;a`4TuJsHA+6A z))>MLAq369?E0!Y>aedjNGh*7p$@}mzRUHATi(}I_r}~P^s!%z+5kVNTG3@PCmvnk18MuL|#rZw6P5^ zB={5tw2xz)&5VQ?%Gy>idN@buOtd7@L8W7$g7;=iqeJ)M?o#_pPsmL9y|qQh>I*8u zVyV1=^C@L-we;d9hq8&Y^iJ1tHlet$KJ<#!m@G5EmO^OTyGF@#hbx>tUV=1S9TVCi z!hx)VwJ=JRO0jfk?yCz&ffAU#tUMe}4N)0C@YBVT5_kxjV*j2Eata%Qt`fmL;2Sf8w)m#gG|(VYSTW z_M8Sw@1YWa6)Q+8D70+ns8-uKd@Nx<@HsdGA4g6VFpyMsnF~%UbiT-!9*F>D2t3so zOV%0NUN0>1i`669^Qb>}ZSv$x92N;$a5@RJ$OU_Tfj(^Gqj@;GiI95^quFbYG;TP? z-yqb%&-n33LgE+z`3y6hbIxDmTC}d+1kY>?;r2D^3W6}}IV!c_m{ED3V=r#Agl>cNmHQ(DwVOc_8L+-5&NdgM8-JlR?CaSIyL#zFuWX8oUYHcY{Y$1g850>!@ ziL-`Chq-qCmk>vlnqsPpQU}GKce*VcT8ZS}$E;zCqFqAx18@+spkm$>Hk4d6)47lrktvTmLJ zIk1{%Qm_*0g`9+Wa8QPE){&5tJ@DiBi0}tx$#q2bNVj7jNxvHb(YXN^WV=NpcQK?Z9w@(@yR4a01Zh_M} zr*b9B;2YyWa;m2U)qYUS%XR`CC$p0 zeMGMlQH{g4(*e zOsFH_QLPTA9Y8->_qZwgz`MwcerZmy0q}9ai z__9W80O{=a=-x9DA;_5m;7zJT%vcg^spg-9fDz=NBxssxNjNI47c`_0*nqVDBH0^3 z@~FsHzPdQ0m3$rlL66&U~P(U7oy zeR)936mjw8fm4|Xv4_hu$paSH9fOMyKfgSvKCQSsFb5<=9u4`**Ovz@#Un1hJkUNc zd?x4~QdxpgnjH>T^iu}`)XtyEmjNI__ax+|L1teKS2Q;!f{7hK7Y#<(iY{;k%tjZv zneT8#WpB+1=lbshPvA)BQV}VguA;+B?ELb5Tp?ft*%J-b7<0oobGV|9Ga(u|R3xyl z9wyEt?PE%~B@*Z!D?j}luITPwi-a>hk~+yk0YNVi4TIvI%wxD>t)BzndXh&8fCSx$ z2x=ovD&hv9p=dd4Y_iBR7X2keWCzeM!Fph6pu>svqB46TOga~c$Q*a=v>vJh{2*Wi z`Pr?%W5S)NHRT=a!NL(CE|rMVslP`5&<%eOE2xTj7RF1}a z$-MF>A@XR*S0onwTJ2T14U)(S-MOG14l%D6ED7b7Q-%qbjDqs1jXjzdjcfAYAx)$F z_X_)atryl5u3>ayAKPFVhf(;x=-tpLu0x-!C?a023?EJm6Rte2ao_;U>im(^v^Z>( z*MfYH1-k2Ap<%>Tu|jxE;!lal=X>dRHDjYj>@zHc(a?<4O!9ph&#d@uR!-IekWFraB7cJrv%FTfN>LJ01i#S2C)+w0uJp8gwKaBzAq>sSd^HM=5m-{KsGVrBCx}UbvEJQM1kQ0(m!BRw6JX#yi1Zg z)P#z-KtTCm1#xIN)?T-h9TE+=VZNrj1qkFZ?p@WA6g$;>9WcnoSpM%A>7PsDockPPZu zHe7@>Y8!5qNzH=|;?kQzL&7TF;j2XjJ*J2R>8@P;>1-F65o8a&XA%f3t_%XE1tJG9 zg8cjvfnAmf6Hx^0*EbOuTa}r&nkv@;^TW>k|&)DM8w2=yN*aZOa(d!7(otN4{e3S^aV2^Kh_L&@1n7y zQ^Vlm{v?Nr1oo?25A3k_DDRwq@l2moCFljBam0Fb%kegA>rnzAL4SVhp$6K-MPvui zFTr{QX4^Q{CLN#_C)Ib=No(`gC;Bgm96To9kkp~YU4GKW8khb%*v^CRQT$(x3nxGkBlr-e4 z6{%V?`o+b;p);c+P7#X}UVQS-N_V6lgmHyox28E+0wAgW{E`tE?czvA2xybT$Hmzd z6(jpGioeboP9g{E*>z(KJWg>!?i-W1Y}4a2_EM9*q!YQwcktT9(5kU85+M6s_8yrt zfu-~lC)c|Kjf-OdgC{%@xyK|;i-2FHk0Ick<2cmzGsXznSIvgS)ubjiHAcIU7}Uzd`X_)K?R4 zLO}1JegNn%Nd5Hqn|-p+Mq?^LqOz?ib}YMZ%wU|z;FTB_L0e>$!5OFHW5H*#e8Z7U zx=|x@te~wk=_1U$D+14Tw;to#leOAXs0{&0b!%?3=zrwQq>hcm*`erE3q~LYqtiK| zA3D;=)Q?DnQL7<1R*Kd;kYoG^dblVSmBoUbTJT4cY-i38sq58hGETf`6T>X1`DlXo zBrO(qALK`H#PwgvkHw%4@*~i?pP?VOI;T^^;%`NN)3@);d2*J=RHjf{|OPtTyXw9Fq(G%xNfoP&v` z?M?p|F*>6ChYY0GlYUWSZpx{y45TrJmclO3#;Jxaz4loakz2%g7c~b5h}&Te!5R`D5V1VpP+Uy9s3AHb{_F^|J^mOhg5pNR zJdXK`4Hhw9If}6vJ3COcO|h%Q;l^Ty9>U~>mLjFM-NpoZH}y+L=l0lR6c#Y$s!r!3 zM4WbBVq(pfW~cmnAv5tnHJsh_S2cv^Mp~|*HXfZ&e?j`E#`YA_KhU~LjGko`L4Q?) zd~U=CK)=p13H=U!>^soxuYqws&aqXIK6r2T1x?h!hJ-gLYxLLj>>m%VR37KOnY`HG zj@I{`h{g0?4GOxEyrD5NDsRPXLDqA@|7xsdRR3Ysq`yz8zoth6X-Yex70Y4t!8WW7 zv~{le;e69N_LaAQ2ZoOlp3UtbdLCASL#i$*u9=2|H!-6=bk=pHm!IJF3VV@h9~Q;l+aww!Ks z=sY0M{FcF&L)xy!9NA>?khXi!JK9b;+slu9Nej$4T3CPrNwd9Z5jNbYAk>d){?Uzs z6x-xSpbt?0^zOe0`)?xg%BW@|S$SG&Lt$=0@9uQb8BKlOLN6|z?JBPa3%yA!l9k`X z(XiR9+K*_;(VJCIkN>-R>!0n!m>EnoU(aoFr}G<`|6umUqGRjt#d+{xlSex2%_XlHXs0B9+85%@>nU&?#RK?O6$svo&Tqgdt66@r$oT`eF*E8w3V4+|vzN7Q%WBk}^iJa>ythgt(Ru1}y(unPD7{N3-g zUY^dsr;xrSzg(*51w;TsKX|kUfn4p(L+9S3@f)QlCpVBNE1sVzy+*bJ+5F>xkZ}39g!WVqb8VSa|81*;Pg?P{z6s zA%a*aZ3##d%9@ZYY^~0ZwsnDTqvsLyoWcq$2bWf(>{YU~TEGtU8ah?4AlQajPXv+j z!UwL=(4^gfUb&rjniRkRjk<5&pEoHr z`!Gi2AiYS>{ydrPgZ8Ufgg8O>Bsl_My6>+H!Zq->pDAXb9@Bk)rBECC(ZkUZeB?L9 ziEs2)OLJyMAZWyz{*O~O9DS`+B=}y8tK9I(2q-EqCp!%##)*$SNLdSGRVHjw`2TJ&ZBRtxIJ<`uaZGZv0 z@bJ~pdT{#)jXidv&@k9W3kbG*N6@_Xq>1cnsFwtYsbx9jU9qhl(tU*@=2YLDxMoSrATlD{|}s!K0-jKgm49 zSX%=Kuueo$TaZnWbo-+P;&h85rR%8{k7yvBYgMKk(yU&q&cuQ!|Es)*W{!3{);;qq z2E^U%=SQVXSXmMHe;RJ5jqi!vBh!e=DQvMYrrDa6Y8tOklrLn)QGj!H@@bgZH$m@p`fDj?R-E2M&-rHd+xSWx829PE zOu|j8;c?7{|FNjym5}4dSY{W?bBy2#-bI;RKs&(UUKNhx{wo7t%7D=x%-su^@nnKm zMG`!N?r)E!v>I{V-!NXGPp@H2A#T8;2t2G=zR}@qwea*O*yniPmw*~PdE-?F1M2Ba zfIOMxD-UaKRUvL^J3=#@u3JXr%4}$iTq;SC*96@(#s>J4^|&@ujY(Q2?BQw$&eT$6 zofzy+pUDH?8AiB@v@l|3qJD&jz0vYlM(r3G3%6 zviGSrP)ALu^{l-Ga9q2xuxrMcW2T8QW@cuJnVFfHnVB(WW+!IH?3f)hGqW8t(;M%c zefPO_-u>_Ys$R`#wA5;;rM2X$?(!$CY@YLlgF<3j+~}gGStKz*tCb!XU4?Z>4dJZ9 z?2JZC%|JtOT#EYiB{h`~XJ6(^rIWL8t#sp~an;V+Ps}sb_g#`6U3uyjaW3BdDCN%LW@cZ5^H;s>_yK*&#S1$QkD49#)7)p4T1Ml=4~C z2X}(4s;ULhr8Scq3ft*VQN_G>^BJw?X>Fx();gh|{WMrLCZM8~n23w;x{7S&QUvVx zY$}(_g%Hrqa?`B^i_@3 z+%;L3d^-~eEf(m+%Qy*XMLab`>oQxauEJqaHnhk<6D7L_Flr2uUHdRkv&34codjpM zr;ClQf;i=m+KUY9lWIs*>`p14{jRW=!5^>)>bgF#1;hSUgF-i|`cNcY&G}Yf zppQ_cRAqhtH2M4j-5V?qs$+~yOeuhzh}h|m(wjw2YsONLJ7KlvYqa-TrF6H3JwkM< zBTpkbi^w=}GpaOuU}=T=V_U5O^sC^$Y+lJ2cmjnZimP7s7pLy2!xVRoTS^Jy^zvn% zx}i0`&DxdxZ&#N&LJ*b*(=+kN1LtzPi5TR-2GL=YHaaAXimGKb?UJ52X?V)tKFa09Z{+f*8SW2i}h7e+7+6p2L&f&yptnI+_(g&gwvVe*L}mO*eqR zU0}KaUcW#}tkvve&k$}!0R5aRDKab0!AWc$CF8<26N49KQHMctFK5kXKXtWi?;Ju*?1qfn&%4FlC-X;9nBe#cYB4oc#zgqHo#kZ-FHJt-29HY(L zrdws5#m^c)nCUy&&!`Aa)GtLv5h)^@6<;pTUeRW8ci!NPZHye99E|m?->+;9EZ|rG z^nmxAotd7I>2D$9zl7Y}06HaiJ7WNyoWAK_zrb_2oB&L~Y6SqDg0Z8mvxA|rBY^df zK*-j{30Ut4VE$8+WHa0d6Z8%=uzb(DDd^i7YNzTF6P|?^4 zpb5Mb76n>1c5?z~1L%Zot!y0>?eqF6M^D3Sxpl z3+C@)U~UDh5jHloH8KVOJN?&fbOX&waaB@W#0VX4XKSB8B?6FM<9NUVrntn>A>_j) z{N%twg2-HiAw@r;P^cIwb%Bb75*dnogbxfbn1{VYmh1197Z%xFjk1J2t75rY%Xk@3 zUtipBnqM?M^Pc|!BKI5s(c`2B>c^a?_VM>|w-zeeNbemO7BVQJ8fe$1i3x<*y)f9! zb30FZdZ~DS-R*B$Kcm)V>P|T_$GJD(cp^3-5-1Qjky%!5A2Q5cuq4I0K5WJ}h~evG z3dRryMvOez{7Rzj-Wsdhl0B-{p|n)8oje@%G3++x?5I|`wG7hDf+&bQtu)dN(*iGO z<8~#943OQs&(qWQnO{KYKc?&c+9%7i3TG7gi2XbK5)1?;sh6Y80WtD{!z>%bt=M!Y z0wP?(pcrL7&3yM@(B$!=PVvD2xB>U4^k$gbm+|ZR4>9zZel@l!BA}$|*6C*Pr&V~T zr|NTiW)Gc<>jXZEFFb=DPs$f$C*#=yQQ7M2RwtEo?HC}pBpP4w>pGH^LF%oc_;)ue zWzPU0myjU!YnfLBAyzsGy;~p1;UiB@tQL{6P|IP`FuZFt@*2 z9IX*tknilKFBU5~G`3Hc0z`^?w;y7S97yx2u?yCNth&KaA|S=5b*q% z?7-ImAVGfO$lxj9wBlgq{1CMQybB<30t^eFE`b(yAT$0R6A&^1taeaZ5UO1WcJLqk z#Q5Ptx~1qr*Zj?40?v`q^99P|Fba{<1?P})HDKI@1mxI1g7*o9#~~;1KTMF7Vo?T` z=LyO2ToXIt{D9F8e9VQO05C(nfO-d^q=8NABP@V4`YnFr!h;d(8QBnaL(hc9>{{NG zY{uY4Z0f<>#61V04%Y2SgyA2A0iP6AgTcKJ7K~FP0Y&*_8BbIQRwj}WhggW)5-lEY zzXJ{xnx~(Jh0%|_9jYF+X`rG%XCO&Gn`W4%@&$8`)dZQz{~@5rVD@{l8hIt33oI+j z*Jzqv6}?zJ&2OpS!58#TxbP4Mx`2p>oli5C7NZSIHAEBAd{nF7@lS+JvJ1C1%C(PY z0Z84B=LTM%I^cZqd3&x<^4-#orpq^ z-FvkRD89!m2((Gyk;x=}ln5cgR3M>9ulHAyuprVV;~{JRlsLp`B<6sn&aWd|O_Br9 zB}Gc$lS}$c_EA<6PegE0hC{N2-<{uGs7L{!)MiGlJncu8hHSgAx1=|nSRzyERqFOo zRboZFW%6;VC#h`Ge5z}zE^Q9E?k8W#*W%}9F!^r&ZbkosZz_4BoAo_B!4e`F`N;-h z1{nt727y~Fee$Gj3H}L9sIq1Jhu@}1i$389P8YdPHP=hmb1oS#(c_D;6+TTT9`;=m z9<*P>8j~7p?tb3I+r`1jUuLZ_*LiEBNPeKS zq4Y{7N;XLbO^z-9qHH_+aW<>mx?FaK9(YKkzQv-&(=2U8^mxg*Yxa1SF7w;YbLi2x zAK@O&j}|vSk9dv}aWZf^aIkTdaZ;K4S+Fx~GKMmCGxnJW>diFZ8z3}xHTD~H=#$fb z|57p`t&8NytExUNp_$|@T&>`#GAylAGcAy*8$;7hpw(m(Xwqs@bQP49qEn;Ot&u;= zv=>(ule?wSBB=|u{B?JmuW$m`V4)-TvJBrcDyn4~PwDB$X6H*BNVgC6OQzJtVETFtetl_hY<%{DbA585wg0OY1=Xw-t5uO3}Goj-T9vUIp%7AE@AY zBGn?ZFjg^isZXgDX^yE0)hW~=)vW7}zCHe2>dmm!HuN3{vDh}O8!KMjYp5!tS$F9? z_2$aZ{~e+9+K&ySy3Y<>YOR} z3p|T2%}-QFQb_S)kpoF3iTgO?@H%0sHLCHdHBAY324@y$CA^%x;k?njXdNyc72b6B z`Y&KFE|05M^XL1I^G~u6>=2c(2XKWDq7W+3yzq>$sF2POKf6J@TLTE`rgTZPW-!16 zpM_k4xnWs8Xrn(mV$o7FSCXB$H!HjOxVb{m7>Ec9^|p$~4o(e!h zib;!jilmB2rK(Y@*X@W#X+!~vc`yiZ(LG6A9LnydR;5C>PqyH;f73E)rPQ4)IW+8+ zjf_{v>2|BzQxf13;se>BaI~JX-7nSGx-G7|5c7$xp-&0>{Z#wW7uxs6&+!ncLA61X z!l}L5;f4~`66g}!V6p-Y8_u=5m;O6tNpVR96ij84WuxXn_4oDDf~5Kp->9^ic*xu7 zeli!(3;h^2pZ%?jmd`&K|K+ozx=CwV%@RckG`$hc~E|Jc0gIF7XjSc@;7kBE*$ z8{@9qM6Auo=gNQU&@-z$+v^u>5tQF-ry0{kwCb}$w>n)lsjISBP9@TryfGNrirA7R zT}e1zSNB$ZSp9VyblP5YTlD0*;R55jvU>RAN-f<$U1xEkdPBQv_t$0qWmBu_Nr#HQ zroA4Ld8|=vJ!5HYyJU@p*lfvB;gLU1Y{p*uk!RCs*3!W8eCN8g?=)yMcpo$`!AHC{ z-@!XM%PIluKK+%=%Ae!M`A0uGO?`EJg-^vV;>X2;#fk=cVx4EXX6D&S*^*d$q7(YR zZ`%*pzHHnGd7@Pu$}ttR+(WQ`;<(rvxFSp^-PYG8f)h z$ZGES#dH18Z#oNq44;PY>l@I6zD>`l8#a|1j~_$N{K}Ez>T@aBAAAuRjv30S>csn_ zerGDH`djtdh<|F0&P9tu*>d~SvvO*MX*F=qtap}o`)SBa^c+EJc8czakNIK4nRWZ2 zUENE{c2!IVr1$=#_7mOO?GNwWn71rk@JOh~-?SZ`Z#pYHhu&*1lh9>II()x8*k8C%vv&;FaUS{v-P0HG@Pjct=#_PgNV?&yS=-!eaYGi*xj+c#7iPnoAiB*Y> zM?FNXdVIg`J0FQko!$=aqH#HdQJ-3`ZPUn+cEA8NToV&H&H#*pv zb~QgME{RyH=-|8cf603?ydH=@JdwSZP0j(ntkw(m`O=Nd+GKD2F6ip%KfUn3+|YN| zl#!Kz@o(Sw-N}8I0sUbKAt6D1M`I(vUp}k?K>Kf4R8$7`i2U&*7`R8NJPyY9G!&B^c{dv27vC1{=W(gjKFECnj1Np zy-$>ti31Lp-t+%R^nXd-1BCy}58VHz$3p+E!NSfCU}a?jurM+H$=O)}%nU5RYoJ9I zHUJYN3mg*@6VN)-`!#@r?T`GgF#_AYw`FEw0kE*L0GR24?SO4qIG6zJ?|pJG0Q+b9 z?~m?37sUI`{^xf7xxF&_*1#a=e|!K|`li5V%=kxF5E#M1Y0@y!a{y=4cZ)cgU#@30<64SwCpPc*^MpWh` z=_bHvpx{Kz;sxPpsQ+YrUte&6f27h*EVJ+eeu?-9_ z;#iIDVZ=^DBxq0`{1FCi_>N&V^hQf?-)IUuRZbc-XTYBt%*A$IKY9mLmZI307o}#1C zqc0c<%odpAr|S(lq(Jt(6U##>y=pWb!;q_LDO-8Na|#yHe8NL0HbiG?Z>3Vd;pBW! zC`+Y;W<106t3j55Wj*T>^LL8xV{J*?`(;UrGV(5iurx6#LDMnQu=z}r>B8FiQu&0? zVJG`pgwWx5yl4BTl7810jxx6g6LzcoU4z#}$`RNngG zO#L*Tago>udKJK#k?GheWN(b%jRLY9(uPBM^~m*oHG`SN1(nX;J}nZEtm_*nQj=BR zk-YkcqvEG{O())?qKRlwKjf2NxXW(Ui4*Z>U)g?fff}rPjh5L@6TW1&nt4e$u(B>_LwDF$lbk#y|&UC5MT=U@BY};-h zx^n04Rp9mxT*((kjIZ;V1Q+F1_-3gje9e|m6D=3%(ejksN9cx3QpOKIhFI$NW$I$= z_f)Q(9njmK>@_iv=wqq&3%X6wPQde&3u3*IOk50`YF(tGUSL12k7Y2JG+Yf3P5bX1 zIMU1_*}({XJ7!tnZd^3d9pY3|xw4{rN@xUWsx-`c@kaptv3%Cnxfiq~!s0o5mN{-= ze9GzRKB_x?dN+q>n9Mx1UM$dF`>_1sp8j$9%l*Xta>BX5v$(EbK&P}Q_UA8EG6{rv z(V9@lawgjNRy!c>qTZ<|w@rN<@};r-2-wlLX5?QPSj1-o7;k@L5fA-%aNwLf4JEQN z?n%;_b?X>Ob>7d@Qb^RuzGrD6OS{N6*9xjos&E8?U`1( zxHLtQ-68|ueLh< zqOx9B3`AJ6ye&19ZdPvRuR=^<)k6p<&9t^i%-57=75;kCxoE`4>c5=sW16v&{<6MX zKH#F{ReQ7%+yZMc-ckqTj-~uKP*#qZ{Sb2n%r?|(gBy2r+xQ<;8H1S6utD8fZOrHeY z@D&#OFa^N)?}gf`zm?QdK2yx$Cf$J?1fQq&gzDvGZBq3tTeMPLQ;8#Ap#Vs7929nSKvb3+BA2+b(`6M!-B%0{sXX3I_a6+o^B^^7+X zAU+D2Ai%hxbuW@umXcY7s^$0a4bE}Y2!;>GMUbNn$~TBjf^%zFYXL?#LEbLKY!D(0 zi4V#2Cn-I$WG*t{kx0zbWb#+B$7Es?jya6^Qk^=?n255@gmRtw=wYJJ|H zHSDZVu6E@>p;7vyNel-mYeO>5BqvK6sVphqqd z=_u^+xJO_xST{DJUBo58%tq$*-^bl_ z?E)-ycM~g27 z-)H7OEday#$$Q={e~)l^ffF>ye5tA-DLX0i%C#AH9`A;c<;B{V1?rfKW3CgLbSlVx z&4?GU{!{n73f>N-BN&%NV%LN9;vDkwnEpp)sm4PhhSlq$ac!9@uS-o+d66oec);ka_f}_ka7>*+?viETM zVRC?^@IayU&`;c)+~ivR8CE~lZvxg}=&q0*WkU?oV!TuHjOK|(kMpp{b-Q%9eIdtt zuI!MP;vEGte~L6m?Ux*-i4$x}P)mrmV+A>2OZ21CrAzN6LJz{#W-!N@Y!QnM>7ILq zYLRH~SY8q*$BnByLEuZ9PDnd(d1wAE3O!uD2Cq)h{CT>Yy)AZ??1knlen1|Dw$#V| zUCRo2++sr%KHn?QK>oKvM6wuqrvpnSiVw3_x;j}xPhPdUysqd^wKN!=%yBTa5G@ZF zAE@PvmAqkJ^7=elo}#f6W0+jlXY$OzJLN^MUnapNJt=tQK3H}x7oV$)ly%%o$xkxH z$r33FAvknIj|LrTv$?~Vyz$)MA~@Fp!+u$|TW#wqrar-5G9A5jxTdy3;e7n_(*p;N z*CgJ_E>kjNy`+w`w8&9pQ9NtXXBYBr%YMhdzN2voSV2E0a8N8?yxrx0{89B_zh=54mI}> zL!jio7>L_#mimx>+(v^N{@}G|!$&!bHMXK28Wx1IwoN1HQ&b!j4BaTUP3qg!CW=;3 zlezQeED4{_@kl2+yO=t9cBLxLFU+&U(dyLJ6vH8to6vLD`|~ec8!uWGJd@R&#iXz` zY40VZIHnXj78BC6TH?OWt}AcYKfI=$*+ULimKCL~rJPYXKKz)eSCzgpx9AVUZo*x+ zLubd>ZS^YbQQpAS?JOPin^;XVo(Bq9}8IX|IA?&#g4eWtWyk*2$8_I8)Zt{r~f zaiL+4P@srT~m}qrFxa7 zo4P2|iB^=z2+K|+o*0?DUmh9os(j0s=3!~{RF^iC9UYyg;w~)>od^eA@XDzF z+NqYvQ>`~CPWb%|X?8aL@$teeRa$zRr}{z~T{_v{Zksz69vh961DSUf;r z=(MBz$Y~N14LKoCO`J^Ve9>?IQ(MIOkBE7twmBk=uibpJ3ky!`)vBWr^pm@q)f9Z9< zP*e45arIWX5uqdiu(#=cveVib;+cr?8YKPf6}Z)lqAuERW{e71zf644y;z5L3{DkJ zRikJO&k>(4RGw0n)*VG1C9Ef~PQHyAEB~6uRkm*a4c7(CLrFNtZ3~YyZJG8Qbn80s zGd{UYCVL)fZT2>QNfv!xSOi$b4m%E@7YWsJNg!9wG3taQvOAt*aVF~+VK-xD-AGZu z`PF!EHXO)%6r&E5P)bDK!5s7-anKU<{Ef?+ECFTc zQTEUXBJHS%dlx};UtXERJhhJ4ch{kur)4Do^+iyn_J3`7eFOdBcTP@@#> z^3o>AVUe5Xu*?LmCcIEnd@cP!h;q6LYDK%M7n`hGVbP%J0<1;~5T0upos6K1jtg$2 zEmj$qh73@z{RI2L0=~d$0u_M9l@JHp{6cOSZOW>r8a$*LXe5C|#i^|0tP@MaklrJ zt*ticSVKv9KW`R)ldp@}AvhInb&YoG^~)h4VU4e&tNqyP<@f?MVqWGPtS-^q8ri^) zXwIfZJ*(M6c1z2I7j}kE+0o=TwPSH4c2(FrRM?o{IEzMu^chISnJ7rb7&mO#VVhVF zAD5-Ch4l9rrc$Y+zw|9Hs#ixe;6g@LFG7b)bZqr9X2zBsmq32H3Ok_(n8rS*GhW%Uk1D$U+?9TIft4^g_7QaHgTSHdIEByky3 z*r}4e#U0N2gNt({6=?4wH@SZsHSC8*2yVLu3HN^3ouGG!VbrnO_|pxfW0GQ=Eu_sQ$=>V zSa5Nla_36(&8rx@W5y)dKciTHJSxCw%b1BEE2nWr+n~BD<4zSCw-wt+8rk6)JF@8UFmjreAnCuoM zQn8I{(U)xeCLMm@KpMjJZQSaBjWnf4VkXw=ZVyZ86)3mb$<0K?r$H6}6uW6I9NOyi z>9?=UZ|1V^q3+xFcum$&smg`J*xMdKD^b&b^0wo z#J!~8q!`3VHr_;!J!NL_0$9jZtamec4JRnzNS-5d~-vgnx1yFvH0jM zm0hRTyM5YxGvO+^8kM^Ac7=2-Q-hUaZ-CYrS`&S_QRgFE{aFn<~?Fv<&!PY&=*P(6{aSJrk ziNzxX53#_;!M=2G5|KI0SBPMlc?#E6-u_7&+ft@GlZn1&+?#fbpzf}*`W^#Ac{Umv z?T9+2K)I67lJ|7K!i0{;bnbum96XIUh9Hb+avE5_)QT7q5YU>0M5tCPy;f+-C%s4r zh0dbx-wWitZ5;~TNvn{02S3>+CF2hpHTgJy|1FU^N4q7mZj)qsW5>iD{u4OzOviI`v;+ z##T3+Mpg>AHL<~zj`1UtfHLZ=I?^76{MOl_KY_<5q%u81CA*O&*41T5-XX}{0 z4XwRc_hk0K^9#%b(Ec;7LPeNJY9~$qnSK<)Cm@6kr_x2!uW*E-KMhMtFuF3ak}_vg z0^VWim7GeDVL2uZ#9d=9B6z4h+i{#@wE(*mtA!HpX$!;g3ROKybk{VaC*b2HC)i9{ zdD2EN0Hd)SVniAO9lnxW5 z%A5?FYFXM1@2;1$XD4!A`Kx#1HhdvYa_B(dSb>+H0$cq3CBFH?*LK%v_1Z{mYff!4 z^-;S?oEFs&syTR;QYaO8G57heJE(1GOUR08i*HF!#UW003+4G{#boloLy%E+^IeCI z8Dr_0;G=Mhm%f%WnSX$gC9B?=hB_>uI#%aNbBov(^-={YJ~1 z*;B(&bYNJZOa85lt;bkwS{sSrsJZf|VX)P}(aCcLl?R{1m{q5rzFxr$uJ(WEO7>r&*#Ul-UFG z35(v&YL_RW8}r3Yc%xc(z`jSvnDqf?AiNu0BYa&c*bR0dUYP6`xnVPoVI{h$)uc*a z0*LehGW>RBQ|x|w)FZ|$tF2OuLfZOf7MK7LFs!iy{7@w)D{4Y*0mfk@KIIo| z)S2g{QVQ8m61L9+Qs`TtqUic{`)?u7Y+W_G9V%t9^^&c+?4Nu{h}u$Jt^$%%ncD!H zP#JqcDyFj*&Zsp)UK%9iS2DDbo*7(Y-h;4sB`D{e;sVTO4LDfmMnt~n_nD^FHM6DVDIw&~RAsc{ zr3-$nW@JSo3xu^tLrQeSTlA1#6ycTg$*YV+W3pwZ`1~RmJ2^8E8qbjs$u5TYI6|iN z{RqN{StKt2=h`-rB7%8%t1XQByDh`jQ&T}^!iiZjff}8urF+SN-7*>k?NC7-$0rYN<_ozS`roYn0OG9c)!Z5jqby{aLT%-ijLOlNn<#Xf!;E) zu{Q3SGWgR(4&LW7sie;#J?YCzAjYw=8=sXomAy?f2$i3AHpN&m=hsa$Z?~4j5I<)r zLdAHp%N|}M;@2_6(sAlGxa*DXKx6W~3`bX74?c#Yqvye15B|uH7@i0!6+KaU;i?nw zIf)N@Bg7c;iWqK#$$Co^A)?`985H+=;E5SmxqLAWZ5Jg@x8Nt7?^3x$XpTCY^)XHB zXoyhd!K~gowL+&Wz5{`Y8glx|_dI9yvnv=LTIW9O0qB~&u}pQJkA37|ig2Z-Kx7K!5;n4fHX!mFFsc?rZ^1tm z=C=8GR=SxEn<$~++$2x22ElD|)d|5(7+VhXxM`XTDpA+^5Agc9u(E(SsuS9IX}n=F z4hPm~p97ziq{5fV(F6l1w1kR8tT?|ktN^thtH*h>d~yZA5K34vC80tLFd&7_@7g#& zLlL3ev<>~(Y?tZGRbKk?DUr_sO{r8TtU}dThlJdr!qC9ZRAQzne91C{IBGtZsm=^J zcc?ZdGF$|gadl6y`ewl@yXmX`o>pUhXHme#9zosUAS|_%Bmqxly)k?F+6zMdrMA~1 zwes;@P>mfQ5#%Agyv+Bi##h>oexmrzx|U5BpOt0{q^B7Q_t>@{d{>R)AKOdk}r`jVJtzs4j+ ziRY4I@8kaFvkRLJ>Mym=c6oLSn)F*Xd!{WE2--LPN&AYqh4PC3>Mg$kX4J$j&irll z7aCrlHs&yVKJZ-@pv33g;NaqPl%#YHGSaL*TY*OKpdR!3Xs?N)Eg<-xakUqN;shn8 zfBT^S#1#y>?1iu`<=&A?*BDFbJ^-O_F4OU;*l4G6JQk|4>H8Lfq&HZJa#yj_%NR=0 zyHSl>fE#IFL$!O)wxq{+j$af-_v34hui^502k&BfV3%j zO%gkTw(sC5;)BSiC{@SuH7Eyu9L{Gm>Kt>^wudbsW%O3fs1dAM2YknVkpeCZpMJ@* zp%o=qpLzQRN+=1*BO5-*_P&Ll18Nvhl+ z0|}J6a>b$_H+&qpgOy?B!N0 z@n&vDY8r$|5+pQWL`mH>fqe>OZ%XF)ol>)s50hEz|l>sKo^}BWN&3KqXj$> zeUO2YFzC@uE)&slpHzn8HR?wkk2#vG=4Fgk%`^RY3e3%vyD3?QYQM@NY)eL7SgqWfwtoH$sy? z`g3QP#m#K~HZuXEO!~PTZjyn_pomd4O5Ek|g2SL>6m3z#gXL7=Ct7g{sL>$ zeInzTI!kA9d2OM?VMOKg5h=?X5-UhZ_&Wmh-!}*VVU<5@4?8mp!{3C^J6-=baq$Q1 z`M=<)WCLlNS52f=?*}SKBCWjdS)cC9Po&zN;tpBG3_jmmt?SJ}Y1(GiBiH(u* zuk=?L$h)usdwrMv(SMh|_rT7?1oxLFu=cN3|I+%eR=^hT*Bn4n}xv4*B6bF!E`xhGp$MU|k-^Jnpc9wsmq?noi8zuEV+kdB| z{@x<<_n$1V^M7XkHzoDghM9jTDMm(SR-onojgp%6a`#f2eYoRrP4_e%J<&87H8QD< z3#ueR9)f|@$5cmyFk1fz8T9QlGi7TSeC`tU72}sKdo-2=@#fIC?}>XdSe$pPY`9LLe8$dAE*(&W}sRvlz&g=VC~N-_E0!Q|oole#?jld+yas zC@KyNDO-)07>JdBk@_~8$zj^PrL^b+dWW>?v!8UkU5&r+c91C%xtt@*E5PA2P{=W@ zu)qH6cPhu-SSv<->XYFzx9e;D?UgzHryl$u16yAQ^2vndFkyCsepjeZQz9=vg}dZk z@N%#{m)4qDsxEO2;%6_bga{Ox2noDBp7=_%qmK#WJAWdcHe4;LBvu)r8RW9BmHQTM zhk4*9ywVtX(ej82dnB!7Ncl>}Cax6*8KLog-#V{f$(hQD+1n|@>3;9UK#(mm> znhG>>)|wL}OZYm&b9lUR7U`m?5K5Ks1VDDNlS(1VAu)7t#% za?0FIosxvw@L}O&afeW6c$)FMJnUh3+~vKh2Tx-BV3wcbvvhQMb7S_jqt95SBM|es zKAwU*#5fc!6CX~lElm5EGDX^Il{{K2!8*vPo>GMI*wbB}yT`sNR^FB39cmO!-8hl@ zO_5zjSIM=*O&G&ShOD-|==91YipIb;?YkdEkIHYdL&7(M4a|4Gugzth=2y8W&eEPQH_{^1 z(|a|YE|elN9!qY|xt(648}-Ty zmO9bv7E91VCM70Ixl#&C;c36G19L3FWV5PG6WR;v?Z`L}Bn*+{;wcN_ZGxb%s8hEn z2i(h)1xp>Muw9af(K09Q+@eO%Uxj4xn(XH6t9r9)^rWmZwP)lMej59}BtPDr2ZUg*q&xBv@FIu@<`Mh;BBTfoG|8W3!jk*wztIIXa5 zs{u6=pM5*5$?Ow#N>>ptNfsr1L|?z06^vRlE#oY$**wu0%}RS2L3E6mF}=`q>Z;8) z7M#Xf!?DMUIGoRf8xgxh-L1!-NtC|gc;IUDvrS2B1+>OmVyHyWlEU&D34NU_S209j zw@b<-^Ce%Otl?m@-NQVuX-IB-A zx#!Rs>-sZWRCb}QqvxUR0p_)(M=%-(`2L1N3*?HZ4WVm< zXJSsD?UwJaw~#l{0~&`dnF}dtrAM{&%1Y%mtK-?*_efD1kE9mg~1kj z9zH=TonBWyk5gv%=uG8KT{%K?(l-c9m<9zpntY*id`a8<=|k#A2%eBLDhut7-3gS; zlsY@wv4Q9fGDpW|(ce7P@v{4l_x{gnK71WDS7%qM_f!F&7a&6As6GIWgE@Lwya>27I*mzzK+W`Ug6K!Lm;@V^cpo$&>F?pvg49!;z64Za zWcmYzGF)T^m7y7r+P*T1VcKl4q19YW{!AsV98B??Oq2FxETNPV#yU-M(j+t26-VNw z!7=SEh!4;e#5Z}xDlB}$qkNTn(SxBsn*mDO+3NTf`Dj8c(l0nB)fMS#S@Y2Z(teOX z+HAgjCke}5Z4STAs{eqKc5Q4r!|YBx%Oaf(bIIN>W=SS(jpO2|E$!^Vt`+OOz|w{| zK6uPp0eeG&gjFnB%sY_)NF=XaDYSQ%elC0@vmt6tC2bl)FSBu=d6|0gzR;*P@h%CJ z)~a7GG(Uc1AtmDl*{j_WR>`O*C=wtRY^G_d?Whi~wyJ@s*tVKUQf4z|&POv1&8Uy( zjZZ%|k%e#^I{y*47Nj~mCT*m)Dj?fafN}W)-ZuR z`Hs&Q6~SY{W2i6Mm`9iyu==R^M;Oq>RTXp~&|)+GvPLA?X{EAfgEAF0XW>sDzddcB zW_DB)xTdZ8KwHgfLrP@MwBuIN5aO!1I%+E#I(sPQ{~%}wx!~cbtUe?mY3d*_MR%F` z$YN6MTs|gJ>Zp$xgIV$$Y`++#sZ_TqX4O$xn`FhwVfDwgvsGKsFin!vYP{L{q~HQY z)GkfEJyvxe8LEux695mkK;tH=gcSME5*4!KmCv!cvS6*b7TQ9y*rEBkvv)IsuxB%y zFYISKr-cW8D3$!aPSj7f=QI1}J7>J=$8kIvry1PjHk1bGQoP?+x?6VZK>Z`a-p!1d z44r{ZpHfGJZM)Mqg3u@3flbi2^q=5dN70gqg4WRGcm&V7Tjy~={XAg*Y|b;+oQOz5 zS+)jv9S^>-KY!M2pNGBu+Kk7)5hWGNYa~`ijD4lEcl(93X}5~BW#=s>4b)JZY00*~ zPeDW^pZY3p%CoW{a+>h=!+^jQe`MgzG)4Bn493Ens+>>2_@Va=b%gu{IYstWfI4SN ze)?FA7u`9#M~(ON?bAq!t3itysdqCR0Glvwb48~XREegIzh8e{XGcuGRkt~kgui6D zDR?2p@6j*B52|4wdSivIcaj8TDIm>*`ha8S4qi9zB?FETayN^8pZfN+i-n8XbN0$* zV}af!`v;FuMtn|~We>YA-6iQ%AW4KZV)duo&0Ht|WBWH6=$io^?}arG4S{ib3$EDh z=)sykCY5h502d$dQq)t8$nDBi+9cfg`E7NTMo12RQUD<_``P?AIq=B|=g*Yz{Z z3;?ScBw0H~6cl2qFam6Xav>5ckSq9fb5Sw-i5J!4^em!QinZZA*?1jcI$^yXG%igR*;;^Hsfs zn6Tq7aHr@G9b01CByA>TW^86z4;`F-I_DfpT|*<(ZYJ4(8?rt!UDA7W)QD{9U0FA1 zSTU5Ztg|L<6!lE;{_dUn;y6G*;5DF|37%D=MOw{M-Pyc;4Z$7RPIe~S<}1@l_kyn* zZp&lQ>pu-4%M1EJfXG{szgBBP5F*aX=F^TfG8OT-Zvb~AY9n{!lI$RZmdN12ieYnN znDt$cfz$#j^bHDYz1B;X+m^#RhW;QrNFuu-J74VAiPuWAo-uO*OgF(=_fMLagBqk> zZG3h;d8r}w9XTY*$wR|EzmsB1b!`OoBkBTK1+=-6K|gu)wcL`=&;pWsINzd#u+*V#Ph_|tq5XjnArf~e?Fd$T7s;9m z%YdX{t%BznD2Rk+ZCy^~ukuI}Iw4Xgp5oX@Ez6QUIM>iDgZL~BBk z)G)WyFbDcKoE~uvX5^foto|n0Mh8oQ6l!21q4>on#~Z(|;0&5jXOYA5 z+=TX$LlK=hPTJhKDl*deqX)5>8GrqNAo!hsZsm z^QyRN6a^`18&+eCGVKozusP}pBF6BT=kt~- z56E-z>LouV;mGl$MMFu50o7c`(Ujrll`y|SwvchzHBDyEtLmIO;3mSlDe36rZ5YLi zjt+N|XK@0%_9&lZnZp^O64zF}@t65e8g8wdR*+>=9W@PZD_ix=#6=I~b}T{f)-AS$ z=IM|PsGi_fvqi;A923ToY0HSo@Al)1O=b+ab8b9l82P4`=(NUMSi4tC*3XKWHg|LLw8yl6CikSeoT>z&T|p9tK>DFZ5_ zieS_=@jy{p)QK%JC+dO)weAH3B`iY!t?C#AC+~;PLMk2PUvymE3Rumr`ckK@Q#+hA z9Au<<5*z)v_+Bln8oaN(bDv)_><-;ueigfE;_}Z49K`V8()!%j!-hT4dzqgeRS;BjH3rRNf=BZl;<63!5i2glC}Y-_SY*&8g-oAC z!<0_jVq$}s&3lT7-tuoiwbB;Km~CF0Q+4SkJy1@faT3KxlXupIS@bSZ=1q9SzAS+c zb5Z*}i4)C#Co-c%KnLPRa^{vyvxenHtD}$ZJf%Miydzjf!G=*-xQJ-nxMPEVsLF;h z$upLmbM2nNzrQ*(h7D2KP=mj)9kH6XW!IjnzD6dEEzZ%M_w%M?oMcQ~K1+e06B$Zd zAEh9;6>V9!n^EM454;m%4O|s}(*rbf5tGCj>_uw)z*b2$iM!xSi4hB-^Mdbi>sYfM1 z2K2EwgV{ha6P%01C_}^FNS+gfW7!I(JYn}`;tPrLPOvhlP;4Jg^R}~M(jpK%VSa9& zEn*Cj;M5dnDE4&g2f3H?+D#BP+4aTyHLraZ9k;qx7he|zJCT9f-K=m$7|GpDGMhV zlYE7CNH5U8AoX#kbAfEV5(#RT!0!H10l$+OkNS5WuEv#g-?yY3id0=(v`bzMZ7=70 z3N}oR`nX*f(_{Zi^?_Qghv5rEPJd?nT$gizcd}6HzO2M0e*f?ug8W*-x7!rnaAjJ@GZD1zg80XAl4}lU?{R)gTT~{Yb zC?J2UZfniVt>>V4EylmsS+C0vwDcfO*xh*&RH3qQ++C~*CY_HJaZ1Z;g$x!38UMN;JEEtu$)o45X z6~w;AeusZrw4xs_=XEnlULuOMW1JxCb?AS&ic9>u|C4mu0cMF0i@uWB^3So_jJkR1VAbWfOwT(hNS>VYpagJDN~ZI zZQ^fuFFQ11Qcu37s#SOt5`lK;3#6ch#^f^SZu_tvR;OTZq^U^E^z_s4AvC9;LFeG> zxQMSB^+LnWgkq2gGU;Uhjo`zgvP98)rr*h^9cJ1Onqzj}ikJ5>HnxUIBAW$Tr&5|& z^gdfdI6Ru2HBQ0JHG^IumsR~$@rx^#+m^#WtQFJ<^p0&dpY54s5SHY}8wqXg9tE^; zkfu%Zuti%rHN-%}2Sa0KSq(%;?9!^sI-DP^*nPN#m1Bo!=U#@+zQUC((}B|qDJh!* zv}oNSS|}KEP10}S1;uqiugIEQ#t*i^sBk~go#e%nEx~CzYiAJwUZ;vg1&l=a8ut?$ z#^7$%CXw^Y&+NvZl2rNfyz?&A?aBG+r?#_#!aOBK?2nJz8~A2f5?jfZ>Q}4xe?9(o zwO;JqV_&)Xz`)|Yi~u!qX~ zbqNX=!S!CSJfUe~3)KvnH(l?isjG7+w2;PfN@Tpy=Wa8xz#%AU6S*x15i6h~h&SKG zJS=$98j!6bm8^c5^$oK<_=oIlkTm1R7;J zedE&~kp|S7vn68w_z?)u8U~!$-X|Mk;CI-_V@!)RLh2=7v+hG?cwTa$SQBg!n*DtJ@F)i0^EP;@*hjA2Jy|cR$AS?^{lE$QkdX8jp}gL*-d?2o-N5v( zuT(s;l}-iF{L;1cEa=OePMCWlLR7Jl@((N5KU@q7V$$Rx(Igj`R3QXu>Y^5D;Fe(=)(6rxfYro*497AND_HDp16%OP-yG~`CmgAI% zUOc{Da32xlDM&GZ-oPr8w_P-l6=FGR_*pg*mCqWJaDn`&GDecn-zTo1OVy{q&}lmF z4plH>pqfSIvWLQ=9 z%O=(J?XxEPP?-3M((A>N#M|x3`$Aw&B`z#Ok$9szL+wf3#ENK}`^^xS7xGs>Dxaw_ z$(x~($maxj(c3H4^7gJYDV$UV;#z*rxPxLEE8;3vLRuYXz$CYJO=$U!D?e&z$-c(= zW>jL-^v9qW_HjFai@lj_n1WiIe*H&Fl@g)Dq%LkFL%9mJZV@ui&Lu&3ijh|N!8>L# ztF^*%rj4Oq^IH6%?wY-P+6BOw1S?29q?Pt+<8O0`0Z$n223leC1Ou-$O8ZX)=u%Q` zc*IZSljvE&_N~^Wd3StXJgnSBmAzT^s3+u#3X(jo-}qMnQVOqCaNLKMY~WB zzoqC%2jp0z9}m+swuSy48Zq8#bS2-zbZ6krkPjh~hK zb|0mu?&1W~p#asvQPq$`My1F~g243`#Sph5KaJn_^$!J>``(|!bH|NCi zlcHX&+8Ne(LPz&52LsP7kCCWc=RImYWn2C!O$>wyWy8$zN8s?f6}<9aG1-dYToDbw z_1*pA<_Bpd2lVnz5OdzuK&%olkm!Dr!1>!Awyi3bE@+KF0-bZM;=;~xuV466Sl;^| z=(;3IHF@+vwG(Zlgzg{A4bHwtY^#O|T_#Wl7#FC96*0JSdB5^-`8@M{*{1F-%Pox+ zMQ7&z8b640SkSW1GUFjA-`|n{2U{0IPAdd)kMRytu+!GIFIj*7U1za?LX{XkgHyTQ zz7scamHpc>?@!KFx5I2Z4XGm%(cm!k94&Lpkq?GwB~nhUvSO^v?yv(oIItDpqZie= z7%uivPjYsezjtE^OIBrPVPk*T8l4^1F+|OZLIL-q^w|t#G*-wyQzqRa&3}#~qC0M0 zQFT29GZYeEnFiLiQR(=vN%@H#kso2Msu+IQM)8XC(B>9H+ADhYPQUz&@k!v>a?1-l!&VDv17;?1e_R@Rg0SsjRUE=3jA zCsQ2zxi+xO^b2wtFtC2k?>wGkVzlIc-9McMQ$qR_8ToFJ;_>p|Bp#MC0Mr?8eq7R9 z2?@H&Ybp_`Hsl#T;_y?cE=3QQ(~d*zYNx72sS0b&xrFFcnHh;*PgM7rLNAmQlBW13 z>Ir!98Ue4jTu;(ZKUzqJpu-kRbMiqZH;&8&RRkZ)R&}Ud)66vRc!I9h@mh0X4HBwd zOe$d|rNa*_Y0+|i8Ap&FE)I?uYbn@~ z#5~}zy;?jQt!nNUBPgCFmv^sO4w8ShKVDb07exd_^tF?x&$rn( z9n0=b@e&=L=XkbjE7ZOlgpB^0W_@VH{*C#$r zz3|4!*yY6Nhce3iA6B`zw%8DZP+ES~!1U~r(WbbjIB~f>L5U()F{XUrk)fFaO|t$V zJsGKDW^g}fP{eVQEEKOJ_=5p;_g1?sG&1s0&JV&WNidgs5<9Z0C^@1r?-s9?@w`#j zpQbjsd~e`$!BuQq!EI4=<>h_O-zf+me*}K96hXq+Q;8-98pSVgzo8?=OhCY zA$TzGNmuu@&NY6$&5RO4pme|iA9P9#9tVOt9ZC;ya7(s1WQnBID7K-NbW01Iw&?AO zy{f3WBMnK5gK`yhv*o@KhRW2A)c|U*<^(2A9nLp>rvbgCNs~~M=F@q;>0u935cwI5 zKU@#pwdx1&)(AM^N1c0jJt^g%J_4X)d~nN2MxcA*NCdItv0A~tc^4brfGDQ>*WeW; z^{pO}k#$oN-KYR}0Q`s-v~eTCPI!g(+sN~me7ZH3m2_T*wPY5zYdnRmo4IExEQGN?jt(@p=KP~N`358)NzBd4aHmTAR&u59()?Z3J*SDo-~YVISm z=@x#YG%DyLuTo1Y0%?6oA=EV2h%IW9aZb@vC|?6j*RuP~C%Ky#ne4lE?dcdZQV-tT z)LsdWYgxI-nnp7u;dhmut}>h!TKE!^O{o6+v6PR{kxZd3da;yCNt)5?ObK|cj9P?0M-W+53SbyA`l*E!KE=UA zm3bS`17PAj{ca{en`G~H+UVy8$M3Z}n=%iR_{4kS!Bch~ez+XTS;_Fu|kvC{T}&XWgoDelWi-q{dc~n)+J5*#|gKQM4HNn2}hNxXThpq zs$1RHmY=_W8*S?`dK&HySGnzig?lOb`|MdN{4V#+=D6Gnrpv<#TCEA{JhV;=x>L%p z>?-gBB{wvb(x~*I>HfrUMRpJH*y`da&AoBCNo_L1mHwEcJ5oj4cnqZ~2e_KwDl*Et zM(g_A{JCrpi?b;@RY>kv%`EFo?qi-26C#pS;+M|I*7_Zhn;J9Kx1}1JLOOyqkI+gL z7N+5*L!z*tj?oUR*iPy%`!pbt%eb-x8NcY@+KAGWMJ=#w9#%%xt-xs@KT5oeaN)>)<$Qos}lj1Ce&V z>P3C8z^1Mc%gh7@`^PxB>qN`wL3a-+BPQVj4meKoM0vAHP>8mjM$Yr`y4k);|5Al5 z7r_0+P*!rK*%fEPFsA`SzaB4MQ!THx0~?+jSI2eFxQxlwIKbol+0Xlixn?9)U7-%A z#CWzw1{-d*bq75lybn9bjvJa?!KQE)T=WEt-$tu`$dxp)Gn-<_aIP>(GQKps?tH%3 z-s=jgF*5^tgR({Bz7_eQy+)zsM_hp~9l&AWb_YrFW~123NDQ7VMOF74c4|%BcYD$T zDt?KP(QISF)F0k|nEWzGLAGHK)_oE$t#!X6()eK(5rofT%Hz7R6zX)}d(km|{tee~ za{tOBlP-FJmYn|Cm^YcxY(dbm49(`_KIlWP{9Mi%McMqIVr18XAZk$epBwaEDWR2u zQ6b)SgP2S5L}E?#q03_G-!Nn~4VH$dwGt0r`k)ln4Ayk_H*gZxvWp{z_C)al#|1I? z!>LU$;@VDVG{q3ens7;GsS-2aih!r9Ho=U4<`0SeH&|aC`aFGjw7+K4i5VP(4%EjC zzd=qSg9Ra%si#V&fj(zpxOeR1Q)6sW*wwR{km;~o>9Tdp^E)u)vc}`g#IVoU1wm@N8>?gQu-Fc81@dG1+dH>y*%IFO{IG>9A~T`o30FCc{57kvr8+R&ZP z?|P%V_sAt6>pH41T}-m7L9W=?AU#iP5p_m*wn%Ki=_)6z3Sad~^x zgDs7t-gs5humyGDCb*{ni0=in@} zKl=Z{RzVh8>4FyfRs$p2FRa}2sxvO@`O;!D&iGYbsVXexb&&kiniojLLy{-%c{(a3 zP6ZO34Y~i8T2*^5=%wG-py-L(P+a|Vlv2||MNoC z^`wAG2}EPVJ>HP5d($h2LlNqpXf9(Y-4d`j-B)P2H}z~n_Vf}hGw`784qT778LL-o z`|h=Jg!oZ4Xz%V|<#`7-Q*ZvRZ2n~W5FyW+cWGBhkVos!{uBg{pzs!;VQh?QmVqNc zy=yROL}P)Y9|OP6*X*!3rY$wkBR3zu@7)t7r`l$_M+&)|&zRI?^tQcCsAzvdE*F!~ zL*MII@tBU~N_KgObVP9!_bJW45)8Uu2nCso(=U>}EwpeChEW1!YPqH7Kta6v;tLQ< z#;>{jBNqK$-#`-KG&6;a{Ym_9$a4($+u7{tNsN8G>1xlb=jE~IFs$+-Z>!&~iqr*QfcWdA zRP#|B${LS<5k}B;uClW;5kD~1BwLxZs88T!;~yjeG09%>Q19yiG@{pNiYSWS7&dq< z!r!XD*as+o(gR(Q&OS&fukY;at{`ItjeHpw4u-zjoH_RTgn<70Yk27VUxV53ffYVT z@;W19O>%`d179no>+k9bd~OO-5Mz_w{ZX*(N=MFjuena#vqkc$V0n+_!W0|g`)ihp zaK@688}V1B41BonFlTv@n3@Y`?LtTUR5U)l@FH87>s@~~qlC+M=J%g@WNK77!U;;o zU>LiMYK>Y-qgh9>Kz8{h&XLOwtUQKplD*N}bh0%Yy`ENEqOf5zH^M4xEa6Pwh{<=_{GnTa1D# zzYh<4$wNquZ=o-ttkSe9v<+SPydvKo?#owl4%}Ut#{O7Nn`y&Ihh`4ldN`_d02p+^ zjxzg&B_1p2;{a4XF_T*8dpVBDol zW(dPR8UUjqA_kKw-QPxym_%Rjfdq2nf$2<8nLpXTodAEEODzo2Fqr(M+g;o$iX-3! zD_Q(C`{hoOSZ_T~U>i!~GVZ@Mr5)!GSHt+a=-7+bvH9A732e7Ks!oWdQ{oaW$#|Kh zd&&NC#_6QU4oRITaLGM8ED^B*rLMnfy}3eia`?aU9TP(p{{iVPx?OIRE$i@A`GAoC zayQ(nzD3dn96;Sw)ErS~2>>Uc*9ti=pxTnfo{j#r`kpnqCzHkGXk(PSI^^>!j}8kz zR(k3bO{vH7s7RhU11+^sp|W8nKtwMPV-P=E*0GtQI-660XSbdI&mcDewHEK4Elcbh z6?+g}dXW#A8_S)vP-L^!8I`W7{op35lAuS|o59i7VNAXQDd@3~K-E1Y(uPIMaTDY_ zZ{3*bA@KB91J%58RznitH>*AjPybZ9voNX|CseDwIQ?3!wuXSRBtmG%#HOvB3qd~P zBg%EzvW%H_)Y=9A&-AbZ@}KIxF%|vLHI`DUePP=W-fapYqod$yDWnubWf@T)`JY}- zEtsUtvK*f8{=4ZhoX(`yYY5)&1VK4xs{A>J+5%}_Lby697SU8vz$AUVN|@L~&9hsn zuhg$0o`&BZC@{bFXiGK9NVXs%ql9$slCZITSm}QXk%t370JQsA{043 z$ut_d5U1!+s)n8+<>Dg*#t>L$1a)t?%!xE^qRz`9XAnM)B@v;i?@;kN0#nn4$m6am z*wKB14zdxCi}B|q$268ZCWdpJ3xZbXWictv20t8g{PA0X*H_{M8Q0RCqZyfXxx->S zdm}4{n$S)bqO5UNg)AJ47*$GliE5*%+$N)hp*Wm`ZTaOF-kckj>p*d(y&EweE?HZV z9`A(p;7ZY$)38gnccX_`iZ9cL*O}}oFN*>noEi8-p$|e#>*NhhyV~^HXWS2#x&`lN z5Ii-NslzzK=W`b(0e$5bIz;|W+VR{HH(kOKFRlGqtITv6>9?B`soM>xNiW;vYsXE~ zF6_v#nl|9pEZUm&eGG0*UH-#@$+-e>M#`gSqF9BCqV>k~^4=|fj)5OPt z>DB#b@hILUYvgV`Xn^!xnfd;&z66?uI7AvPu(HtHUz^$U?5kq=@*B+b85p$wNCI|* z$?dsXNCPiIg`K-ew`IhYJiwTNc%~Z4Oeo1gWtd$bNa(FeXsC{_;@-L19@ba8rsvf! z$`IlVTGgVPyJT39)%-4Eq&9Ek*#{q;2NtcV+)K>SHlzXmuIEZP<5+Z&X_-W9t*HIm z*4E`d;~~T@6D^!1eMj;L94WqaCaNCmgzF6diI;GM^fY|KZ~>7b&BLGOm;p87mtz_X)jkgyD7vKQl;!{qJ2Hg`mA!8iwZVx>k)WmVd|-uzWpnW-(CU zm2K4Qd;2p__$*F`WAT4k?U z&mR_edyLG6Yn6~Y=(KnY^}sr*jY`3+;vinq#)e$k-!pUGTl!ba&Dd$HAChv-M=PV|54o(RX+D4Sn)9j}tbvM?j! zt@iF!U9I1qcU1ZvN~gCzA^R<8;)3`uqI{)nmg< z@SFt8M_(K=)+rHRpRJl6KS)?GJ7Cfv;JW%*7jT~*reViysjmqtGFhL>Y(1o(2EcQB z8Ww_MZItvyBUl@r{j^BbYm-ZSEL%vh zb_p`0W+0dD#;437oDVRgFP??=Lfz)~j`~R7&LpIzIO%FWo5K}Qh8AD}lR9cqlxSf( zh;5xeO!Gpwn_NAsJ3M*FtJtuH-f@&w#}2r^-(R;m8a}}`Z1dsqQjwXP5Gi;ZdEBq5 z7#F`r_{vuQln88?F(NA-9K^Rho6E4oaUdjsb<|In#$kZG){M7a`skq`&saalM z!CA(EqhWhL-0!(6@p;Z%l_L9hO?RJ#G*=j|31W$MO-Rp<__iOn~Oh3e|r)YeP_iCu=83vX22;5g=axA9G{CxZwlUf5m#Mz#d;>SRriE z)|M0CyRi+OWd!v-al>WYXVVGIOCFM9~9%SwG z?k$&VsWl-ic4Xlm63kOSLaq*MHQGwAigd`Q+e>dGHQk8}s(|oaGwVW;u)T?8`F74r z?c8NmKr^L&H?e1Vz)zm-Ca_~)>>0WrKra`?qb|ps3pMqh7jYAq(^j{P$D$NLm4Fhh zuN}1#jl#jy*nbgo^^U++Bg~IFllurd&p-c$Ax~%}=C^5ZfJZQLwiltW`>UKo_|Ad{ zY7+|1=hdwm=EgGU_oSsoz(;m4&viFCP2CWzV^XmFBJE7-Dx!J2vqP!O(x<|v?e@2K zOzpcNptk@J1I++Q1D5H43MAm5t7y$%MBj^)RQ5bVx7SGue$A6y2}%7Gh&gN#(5B(Y ze?o;iQL#HeMCa78A$!H2Fb3wIPn$g|kg9>0HLE|unT4!mYG&)wluaKkG-)w(1k>X@ z0M*CngS`M&++>gh|2htn#x2nuU}VcJlmCD1rfxe zB6ZuQ@sSp92@upFq7r$xi?+X0Oeu2#on2(U6=oKbl{s|J_Zlf?T1B+66N*%y|1GWA zw~o1y^*qR?<4J8%Fu^m=vxFLVA2UlpT_z!Th;AtIAP4-FFw-Z3BNp0RYj|z$Nn|U+ zDT4KM)jGWo7Bcrb$905ahS4FeVY42}PSC(r&(Y9$8^tT@;KMwqaKv?lUfh=vwqkCF zyVz%0D>X8Lxr&zXx$6CeQ0{7&a@Uq~nsi2bR}1aVyGKa&Ui&Ox6rm@9v9aQ)aHg_< zN9=Dw=V+#h$?}3x(CjZn;yuEWVF(V5vEKmdMp9L}m62r^r2F)0VFjLHPHz;sfL8kk z+rOH~l^Zs7@8LXY;U(tj_~#17FdEiQVi?khflW;H2CB3Y5-dxsSS)`>uC&eE;5Y%R ze#5g`J`+~9H}N+1uG_uIKCnJw>sB~}DEZY0r6|V`G=bt6X!5tP@8_IRjl$@hEL3Ke z7Hl>-nHeq9OJ+>(TJ(J9CnIEPG-r;q;+9eXYENl#^S(yfKsmKQ$;{1XN6$~8HwtLr zx%@a=7pjzpwV}F|=92L1GE|<$CSb@UJJc&vOq+ltc!akBSoqR32Q^mQZ3FLT^L)i z;>aYk??YNuDd?deT;e)#^s)};%_BmCFcPM*UUSOlWV-~T`#(jpbu<37mi0#c_H%pn zv=P2@LQhVER{SI0H8{J)Ri8?%#f?_QT8}x)n@RVVM2OE-7p8WtaPK~AXO2u8qeagl z8k@u71FE`Df{TH}16dkw?oI6$20Ywi%2!{3x4IC zB-dznW}G(`Rx~;_+4?sd%;^QCvNAZE^Js5F(b-Rgf0uohPEeBq1)Sw63qLBM5+EicX+UW}yQ5sw7veh-Tm$w$>|-(^LL0#w|CSW^pA9mx zGY165ensvJzliFIanZ z734w_x&I?@Mcd`I5CECf1{*fV9yQ0>ocu5$Z+GybbfJ+u_|cEocb6* zb9~1VP>a$0lPqWCWcUtoqqfpFlv5;8F}8Owv$1}sX#jw%01$~5wV;!k*yLRV4dg^( z`qy~JKd`c}l9-VEGxn}D2E8kS|9OJ2GHa717mH*2=C{`#=(I^K_Mb1iu6AK=eqbu literal 0 HcmV?d00001 diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf new file mode 100644 index 0000000000000000000000000000000000000000..94402c2a1e694f40e65561581720c84925f7e7b8 GIT binary patch literal 43288 zcmagFQ?4+NS(l@bqbcFPQ4*7<~uzcMM+-B`P7nla7-owqx z~XRDV9Y3P=tkRdw z@R!A>o$9k?B%nWK>-8+EWmiR|c~tlST}EpQ)n@T(Bj1`$X1Lolc74sIv;KU#3eOc* z)Oa3lyTPfuqE5I3XDsw}eNYE*|OPsS=~ zcgl}_$z|C1kJqu9wV7qq^i>v+bN;orH+&Vk8wn?<0-2;;14BcJqwrbCa^C%gN$8NEj9=ro=2&U7bfmfb>N&jLC^V|YVRULcobKmcoo%VJ*#AH<3(1L5B3 zq{u&y+F@Y|cSP9!#2-@yL&zoZ9DxmE&;<6cIjzVtyv)6;5rX>qee-$S`jd-)DZt6j zzG%~P&29Z2a;{;On~N2mxW=`$x=62`Yd43n%;q({0-I21YB#T3$tn z4}&xL8K507bMGN4HxLrbjfeY@s>UO|>tPL!hr1ns#H!7mN=MJI58(T?t4!Cp61(k`Nm-{O`OC>}D}oRF!?X~b?CCXu zWmjqF*0*7SYjULGTG$Iab(9xwfo{VC@+wW7x06lPF@$+lCf>H@KuDS?(ROwRR<#i=GD3@7NJdPGaW_6Kfi;|ggd8Q-t3;!4M zkS=4e+`;0qj0z#N@G4{TX7*q++ha_7o>+D*3;bSdw{7SBmRYBa+FD?sx*vZjeJRc5-)+ge+`k$|{>LEAl3vF2VF z`|JhXt^k{+SHNo?OfS-Zz*;d+!Rz$wc7JGV_^dBQ2I~ppq|*cW6yQNE8rJcCUw$Di z;``d=Fitw2f)9sBaO?rAvk+SJu9y#2w>L8~gXoAR$YCjxn5N``;8HUudf%ZKDq)JmjqBWz9*m%bCYIKPCidMkLFwOT z^_T zgz9|_h4@Rp6MOe~b8N#b$~Dm19-JU&?)b6VFgjGwp6k3yueyGiPI`Z*%>%2_hVw(!pfov2McmMg@H;d_BzGM{ZBL zdg4$xxZzSf0*RGBGXcml>w`PJJ}&kdrb?giAA^8-W=&i@i9?JF6b)dV#Ad?u>&ciR zD-q-}rjHW)#C4;S_^eSIa!w;i(Di(BUML|--EynOAks8Ct}zlL#J}j7dONRIV|$2Ukp1jb{wO z11ITJ@owVDDZzRaE(Vd2lRQ6*1{iH{+7*=urzT+nA&Ovj3#fXbkXqpULcX^UWRm@h*+Cr(g}-E3=~mF8TtmU9T+n>xj`a;@nr zua6+fHLJq(05p}!ExEiUY~`C(Z!EkoJ;NdqVi>;2HCtNue@ZO$jbVW6?5Cjsl zjSE&BPR|m0d^WI!yEMzvKKsp6$tE=tSteMGil(c5#SkEb`pIC8w{ruzlm(g#|sK7RYBHrY9)Js-%M0xxz)MS>0&s zmrxzo_KjUB^IVINda2_CdeOS}Mdl`aJ=1NHkB2)BTxNy7Qd3TP_L%0^I9hC=v&KUl z!G`-TXcx)z&|IiIU0knBggDo2@~&V6kk_i)K4AxnpEe?$pW`q6u#jHOakAxd7u$bS6IE}tmkBD+#rUhJKhK?xkW~aMHupdH!(9lw{`8s$gYKA5hWxYBYb7<54!gS zuJO^Y!!JH6RuK0?d4sB)l(@oxtbjLCp8Bm-^CqemQO2|_!3TxONJ;u} z#*16KdnEW#aLDhP(OZ~nvUw$bHO+2V9j7H?umKdQ^WgcUK}}{#9f@1-VjL*56Hg7P z5k*-L_J|&SA}fpXcv|I5S0&ig;@)FkJ>QhC>x?OrJi!z*^k>Ym8EUnSs)lV5c$M49 zS=Kcei1C!zbpT93f7Y*Si5x$CCwgvT!h=cX0p*-PAy*Kfp+(LNR+50B6@sVlqNx3B zhT4uE?0AD9Z-#iNOaH^6#KBU)}Zd@NDGq>*+!#Cv#@B7Q`Uq|Jmzb_GK-?pmK!+KMfxQv<^HfluZA z6o-m+wOLA3x=P_}njEaK>WQg3K1i3>S46+(C4j21oGarAalp*@1Tw8#NmW)x+<9zH zLb4S#`?B_u3*tH1q+DW-^K&{(FTpSCX5#~0zg!OCFVYzeU**czd&jLcEu1qo0|HZa z-VkkwAW3cUlBFr~q~Oww_dyBl3vkvE?BwVb|01xE!La~bgx|YVPw;XtUdl7^h2qp; zE9Cfy2~YR}`Jf7>jnl=r;*ZS}?vdBoWXFSX%N}OE1|6`{C@=NxHhh!zBvKjJmCV?q&lh)T*z>3f-Vhh{23$ST&38fsF7bO^gihehYHg$c=It4cpo z*g?s{@*+|i1me+FtqRSE@NPxw&kU?HH0p}du2Ktds0EqVAs#k`*<@)2G0cp?X_@&_@sq_A ziJrQgY7JMek@f@lfbnHmV+vm_|8;44jP<%<8iCx2 zn+RF3;huk%Gn8?^5w0;TpmgrrVfS@T2^JD@S`d)3X$;dh|4alpu1l%Fqg`zzuI*VT z`lQ=s_1M2CZIcs$0Q2u&Zc^ioWcU~=AlrCY-Bi7 z6ikFHxU9j3e$1+G3UE6O_6Q&hLmUqq*BuH-;%(EuT0QA?46Q4iu)Lc5E_u6yjb62_EKHFpzZYvkC&!3Qdw|~N@-#aU(k8^G} zp3YRFAJYRybQSJnP)~+HQ^{UGlbh!`nggq+@~629q=LNN|I5Qre?z`-lM30N!6dk0 zF5WsNHU_3?6VzVzag0c)v4U;j3jZO+OibId0}<|4-8=Eyy#aBqeTA}L&cr}_i*`Go z!w>+3L?+s84ibD#+E2V4h)66fiB7iquDlY9uaab)>Py3{j&N#FKJ+YZw=6%l;|?E$ zBX^gymu>Ul)vBx_R)vO5STvKFlCea}ql_yEJK90d_~aVD?beF0sPiXuK_oa>Kr)u_ z1g@ZOYwW<&?ww3PA}$q8{9GIBiw<=N!A&|8EY1h7%>b(oPbM+iam&u$il58y}2E%Mf zDX)|LvcMgx;i|};@oBlXPdiJTBiCs+#!s+y@=)rMWh@{CKx`6uE*#`gUSr8ll3z?* z;nNU-PE5Eom7?MRXM((ahO}eD!o>Ufzwq}GtQ$yn1O7M?9$Bc zP(0iHB(r1vyNC^nc;5Ort!zbpga%i$IYMcef&1bC^IJu6pe?tXmBuE$YBM!Q{T(W7 z+#p2=O5~jJn&-Hz4kH=xn_Ta2BO-u>K|ow=G8*=rU(q~ewvT_E8U4WJc@<2xN>$=J za}xJf*Hj80Dd*YiRsF0=f+a#JvDUgUoC2-CU>l^bm<4YIA!4A+bD_YRE$OXw579GI zI&(a@oQa-Y%N9?ri<1uX80$?>9RIl$phz;fi+KQaj=E~ocb79e$xkYIHUoxv5Ej~Op8?UXyjW89ESP$Q|CAhOP6 z+b_hPwXBc>nP{0H<#hVq`_d)S;5nj=0&EhA-U!atOXQ3pAXwum5hr=vF-B%tqK*9s z3q(rE$v+1R)uP4>=am_yaWEbaV2asT;(0ORQqpx0hasH>8i=Ev|Pk&WiK%nZNdP6OU7ECN=2-YJsnn}C#%Q)*6C zVs^)lDa~_HX{K1EZUK>i9*Vv^8QyN9vEO6eO+E3cRPX(@PQuqYk~p*>CIU z&{+jm(aEJC0pM_;vFaIniRNy!<0Se;syhZ-@p_J_2er-5 z8RY$rR8qLGTqa2DrsN$EfNQaf+Ulk^yA0S=AyLQNmyrk49gb-w!(I=HCK!%ZEr6Jw zSocYJSvdQ|U6Ln9W)ago+}`zy1FAZj`5jeaDk2LRLb}J+f?GY=j4m>66(MsGSvGwzhDx!<`R*iN%h%hl<&(2yA74gUuRKCYkT= zt3s4a0GLwJ!$sV1Lh+-)tXCqj%eHd3Y-D8UW+Zz(J7qh{wFm`gJ-7>VD6u$^?4$e2 zuvJ+@s0ofVLXve(?@sjDMb2Pa2u{ZpDCXs#RX(Lmcf3)1!y6R$;qGy_I%Y^2=fysK zseIDX3sTt@b|s_k)cNC63c9=U+V;lI-6E}<(bW{ksVco#EKwdw7QQ>|<6&W%hXHeC zD-)w+1YLER=%QE*ho0*g^qTR#DuaspjC+|vF-p@5z@)MOS!y`Omp$|67*2@(uJFAp$wNqH4@J&aClQ z%4$V%Wa{Zfj!nfdU!W&AC{{c0xa^blS7nBpHyWs(4z>bUno&qeH3-3Ft)?W_vOAm0 zQeqBdEO|3>)(IS3TD;sKIcn(5LYC_aa;wy$MsHVS1Jk3@J8(uC(pup zu!KV;FXj-a;mkVIBs1i$AjZLPbl#7;DC?FR7T8`(6H#Gz?iZA})Mdhexb`Tmuie&!`TNklF6y-Lq@LBYzgT>uRu#HN;-iDJ z8n7Z;)2Hy4-Ya2eoCx>&4y&n-SQK0iB|Jq$O!o2qw6TytCm zr95B#O3ZOpmb$VNs{1){ner@p%XKvHD~IvRouq{_)7&3wPBy#*sz#eOOC8N3Wr+DE z;$vi|J%W)kopXSJl|@}jt)^D>chAz-*CiE~tKS{gZWn>inPBgk zYT_|j{di=%Ld_b3oG@BMkZ_W^T$?veDfcr`B%VNzDOazx*j}MbR{2wm{cZHvi?C_w z02Qki8xVfiA^XdW6aAEqmJTu(+_Z!S6C)-l#l7=vF5%2>S40-8(!m?hSCS{zsH6~z zft+z4-p|l`t59z|j!)`;E#YP>p(NQE4(?>(^;*gI^G-Fn70c{X=&hp$m>Wf!6?b=Oardh+kVT zACuTh8O!2gWlOfeFDVgQE*Y91p`%3iS29Q~`DPd&*85(!a z(;L|PvF5Jv`QYqPLun^_UEeb+@4OE5n}0xoZcXf;&7S5QbdzOuJzdX(jrL*J=?)6} z;4Ev+iFkj}-pvuXsM0(0D=yo7j7kT&qcsxSSRt`YE)ari^9V$Y%5`8(tr;ZO8rV2= z{YfW6GBP4~{ZJ{QnKsbro<^sDGix!mPD|7Lvfr)xBNX2;I z6HRljy4P}ki?(J(v)+^=mkmzfuuL6P)<`^P&BuMs_b?PnY^Fr&e@@i1)}{(E}nrvf!BG`g~4GqqqLtSre!RSGrlUzjTk%tYIPkYR5xr?LTCMZ_=}yS-PG&!wP3SMOd~fu4z;DUD zCk-JeSq@yCJ2t{^K9{Z*MVMVgT>M2uo?V?7$X=RbM!3r91uq6$I?4pVh7h8CQIpe9 zljsZeqRY6Qqddgw2b1`k(IyxD5bO0E{$AUVkS{`O_(IJbtq6+4!$Wc&qu}+V7rKr{ zgq2$TMjd?y&Ui>R`8sn{hVQg-9o&o_3JT%|(n_LdZszuD@;+;G8>{cLv3xs@e55V? zIwS9m2qN>ciEDDy$M4sA0&(s}N-@rxg{_?{$Sm+iL98I7Sl(Ubznrf4*P*sW&DAa% zVlSyRg8dz3H`U;@9v?V_<3!xout*WGVd+|K=jjdnnUDp$MH|LF=#Srsi@Daqa@U|bu(15l{rHXKu0(tN1=Q(#$e#< zT!fzTg2G+2I1a{r?u4d;NKP>7`gI3pib7;nSIj&sLfqClRo#wVo48KKLb#(q*9Dwt zsayEq2Tx3^j>C-$#4xIRKXIR4W8d-1AORMsavN`$WDtl0P_)^39Gm1r)?tS*04T0S z+Awj(z08CWD+*mynL284Gw29l1$rs>#gBCOAP$JbA-~QMGqFOX3o5nQl+cc#&V9On z*@C*YwSo*qfyMF>&5MQPnl1!^tO-bFjt1i%{!u z9G|xSCN(X%G-_Fbg222rlC9HQ z$Om>nG^Y^aVFBr8@WEd;ntFY+3If3F#tj|zuQ+)z6P;NvT)FocvdNWq95upCwnU9+ zAUaGeCFsO$GF!f9?l6Hz4q98T0+;ynR3i96o04omEY~;POanYgvJS77`C=eaoYep` z=+Pp*EKrVwdyC~Zt^HBkKfYeL>%V74F?=XjFA+G2f%;u)YT2)w*GKOanVG5Fyh3Sdd6*lXUR>jDg*#Xk{R{K81efYi9d0qx zK?n;lfO%GYH~pPGMZ#0UCa!@G4SU%2y0beu@3#D-OEYcOdwYA_T{u2Fu4Hc$o7*Ue z?FYv^2w%5h#Q&Q>G-Yj;&S!{0wC;>k7eQ?oe%bN@N>_BN9p2#IxueY*YK}vnDi9dX zr&WhSZVED9^8FiAs0CcZbm3>P6nb^bWdYLVeh1I; zK2H!&H3$M1T#(2}+YTFk-#AjD-`kK*p-lW{MFbE7u zVJq{F0`ekq``7d;MV=ZuPfJF1xh9r)e!rDRYfm@@Srh|Z?l%Ifax%8J&qwPH2m6qa z!drhrZB32U#BgNP{i&5VX8KE#y~l5Z07HXy8b*I!2;k@y$uI7o@y8sq#YFJt5W*t} z=Qsk_1eT7YHQ6Wq*v+F02%5?*T;YvX;udUvvpNvYSTk`wXwN#9U-P;;(w`aMBZ>@r z&N}#N3Ck##DdwjDYC?H|nbG|LL$xxxYZZX#vVMO9J`HHypy-;#05o*lPb%$wi3*1xyhhG``s9%pF|i%1IM@%6nhjXBDnW zs9hqO)U>(aNm#*%>r{nas-bc1X$-NbQY9YRL|W3-`+d_F68Q z1IXG1&MzloKUc)q_ve@x*d}e^(ejodMFE}m6X_^!5ei6L`7{aP|X17_AH^+pA+R> z3q6>q@;j8*nGS75Lzvs>u6Giy-LjO{v`H_Z`CZbK%*~X=RZVv7Lh_x19a9tpaLD=J z7U*Hkv*#vhJUbE6u?>#0hWO77x=7n&>$Z0h9*u2$tMRYuY(Yg>Ql5up#T-v=-5(~B zVi&KXNDdo>e2_qfIq5!i777HcuFYJ4>hcLV`dL=OE z$)w>9c=AxPjpFSy9OT7zg3yL?tEg#mnse#|)3qHHsHb`$etvG2b7WIh#&gmy=Yg^c zFe?cHX3XXfM?T%hL+SL)_2>1IR4wTA#LyhHO6qMMp9r#9kXe<@*#X2Kk&!_HNLzog zjEkJ@-4-ky8sh053*a?5BZB@|d8;WAmnX9STrZL{ep4Q8gm)TGt;RQCokSQ1l~bo$ zk;2d=&ZS-WxVEI19j$Y8P+P3hwWj1T@u?$3rBNucX1_N&iF~m8lstJ}6UKPL7A}#7 zV_znbLvDG9{_yA|fyi9Qmh|P<9Ws;{ZPhA_$C&}MTA6sE(e?M`k;7Ssi(0Jxm}BzN z*VqSc#uGhhR|WF}oIh&!S5{Z2V}Yt1k5zXDw-JFGA^18#z7LsTjKO#*JNz53cxv1d z+MqsZ)q+aD1IHr4wj*3mshMdL^;}#rTAQ6n zf#&XtB@z!HONU`DcJ78DH`=n^^xWvV+iGqs!>L4x17qJ_^xQPr$uBdZm?X%1;YtU6 zlVKl|yJ6_2ret{KpP>7re<{3it66Zc(Pg4~ha;yOvSasbp{xOAAtIKA>rZzsj#K1E z^I65B451O!tf{io!3B?J#)b5bViOspJR?|G8n{=ugC6rr5h1OKJ@(d&;%6nE z)r|?*%Ls$o1kQw*->=JZF#4-IDD58fR9NEGD1N?V!fXeNNTIDAH%tT_F+ zn~V}Q0fWl9Z=~=W1VaojPs<1g9Z z;DK*oc@MPH*ThR=SP**sJkeyvN7+XrJ@mKFTpsB*EwMXfRNneB3W1S&{JHu*-MHV_ zMa-0Fd41kjyF)tgeWD(-0dgaTg!S{NY9801KSx+{_pB>e>s zWT-0EBl0o>DRQM@JrV)s+OEpm{&hhA`=UqvCr>5kEilevtYO3=J%nn5V~k0F)=IP` zt>{(D;HnfRB@hVZ(nHCbh(Y(qG2BEkv;i9=UBff%(sqyi>Tnn;- z3fA91rZdYOThR{s_?5GX8Ei_c0E;a{qfNkSW^(l${w@8n#UcRN1DMv45uX+2m5N}S zxH^}U5%CIT^?Maouw@B^*{IX<@O5MEQHi>G z_8>ON&1Cs_kOdmiLz!7?8lywP`r#=rZ-&EhX;HZu>~gPr^TyPX8{PWNn#^K5Lwzrem_CUeaac#wgJApj1;$P?UZ01jsjm^Xu^bw+cjO#sc= z^>N6}mw86zOg`--{)25^*z+da^HR3!&Nnvq@wEjB0())P_d{E6w8rh7vIEJV#(%BG zP%GG~$f45a;ONrEQy@JOlJnyw;UwLBamemXzJT7lZkCwXshkIKBG2ne@`_nE%X{q{ zzwX5twUa>#-F)zNYs?1rSY6w~e+N0U)ihWZA|`Yu>B<-=U?Bn7zQ-X`IFm{@m15J_ z43Twtx%Lwt6H)LjnjfCNoc8x)jC-K>Gl-=cQI*?Y%C9jHd>kUzmvPUhC)5cOFb5*&UcNv4qI5#jJB0`aA30X|8|t zMoB(t7X(^qZQwG}#n$uqS`{IScU#w|!48?>Qkm|Go$b9XIsg1b5AAk>O8WO_HcEq8 z2;GR!j6;Tq7Wkdhb1lRTz;q`#Dv}Xs6=0KlBv5*NAfg?z{meH`D~7;RN38Vgg6i(`!|TKI>Lsz zxxmEjGMaH!bqh1_XSW8p^0W1-T?-lS-h%n#XHuA-BK1PaYCKbmsnMTI2a`ni|ez}3lnw0iZM!oXGh-W`vAdT z;_Vd6Z_Gd@<_RrJJN~J3w&h!cIQ`!Q%l4ZYgKR&1`sJ9pH?s1`CPS(ch1@k|#~~SM zhqqI5k;5}pjlzDUfF}ON>hNUC(XP_O1E`TVe8NBF6yuwMIO=+GJ`f$##|LS}nd@jM z*uFsceo(Eprn=&T^uZ3fcS}N0($Rc%LVo#0C6f?QOO)g~+eKkP9Au&L=`;)|S0k#; zp~ii?TzAe5dcI@>>or83iJT8?{khZ%*Xmz1mW&zMc^0*83M;At&Hczbx5Be+OZz+! zliDJ#g%E+Ia|_D53WB*p9|f=&)!TKsynrYioS0{jBV1(|YVu>HPE|nKuoB6AK4QMR ze;M^S7RRmP!`6v&i;Q}yc=eSMbe5nKiUSE2IEsJ@cQ@~IUs_o;n~iI5h-<}5b5EH; zX_$r1gfWzEO zWT{7?{~rjX0@`&f1Qt-Hnu;3VIa!ZB3URO}=NWFeC-#{sTGlYut~L1)?~CYacIDJp zKBp9Xy-CP!Pq(%txza31drM}8_oL2?S1r}_XmgHC2(=6tZYSGOsMz=kg-OVb!VJm; zBdJQk>TyF7yOlj?@1FTRq=Z@^Yk^BfN>u@R>g3!m;K{ybwERzBpzK3OjQ zv5&3AnI2|vT=3k-WPdJ8ZSNU7>bE?;be?nUl=uLBp6QGl-K(KQj;Vd96O24piB60e zuKNZ#Vw|RD=VNtk>lU)1;pfoFIj;~&P`KHDkvthAaj<0|T}L14i)y3ZUTFBcz5u>t z{y3oXiL;ZM4*CEVD^dda9e@FZy^Ax1XIH^y41cDzn*QXp$XeP9iAJbY@apy;zWTm| zKvkri{Lt$OdUF|84SMg@!AulguMtQsH=>Y(v#Vs;UARVatt4eDCq^{_)hY@M;&~1X z%~}MPMk4V!QHemj3*Z#!+dynLRpoX=`sV}TKMDR#5=RCK#I!7jA+QL`l4w1Ynk*tc z#ci5;h~THJL_$084krhGldM`O3;Uy}LCUEX$C8f+aXuG6uiQU1auSlp7+2}>u{K4c zA&iva{)aH{dR7Xcob_LP2ohid9ONPRDT!l#PBPfRx(hS$x}viN<1}zTZIe!vkVKLJ zhH(W9$?%ab&ygkZs3^{pm}+W0jCrdCJTuOU@BdZPBYM1}tAlL=9ndZ$Lx6cHcw>N1H1) z$<5iC9fW4m*jSVhaxAO)Fecm6XNe9Q{>#W?(D&wcKFif8P&!h654qs&o!=R0g}VQf ziG3a(50N1(q+;?|PePrN)EpU{Z3HyLPH0c4`){C-`4#t(`z0^8U`t*kaMS){N;qyw zD1j2UBjWAXpRDRCzR*iML02}A`9)L)Xef+ zj|u?C-i*J`zAc6DKI*tWAlUUFav%~h+6rR1Us;13=f;p@-f1!Rf<(S z9DGqmMfC%aeC_0uQCj~DmFpIpFH;IQsNRQeZG6`d-*WLu<{CDK--pWnsI8-qi< zuueDHZf1!HC1^OBx^{F9$G=ym>l@$zM7#JSI%vXX!^(jN@5cZ-e@K+LE=gXMtPy|8 z`^(_AfvO|L^E){Z#v$*Wv9_&M=0DoaH~x$UgK}=+Kky06g!*0fztQUUL`V*f;9*ae z@f9w}1`ivFl@Q}P#rt({uso<{)BGMwfAPdnmVZZ09P&yyf__J+|aW1^~N?$53>KgRBb4 zN*IU)Lrkc6icE3I9EGx$21PFgw%rcZ_cBbHH=?rfn9+D&v0j+kwZF#$8aMb^YJC`D znHYA;Wr{DFHLP>(;?$J^0t!E<5ZKpyOwhwn*3}zX-Ti?qPx9iQA1M}WjFk`8k$90b zIvsf-Mcl2!Bn0>8j&Raej5q;Q50&Vzqm7AJ z4I~(9fa?IYI`rA*GV6Fe1yB5t>ae)u3TeCz&XGt#^$C)3@^1p$>&Q@TD}K0-NJ=N0 z;yOOwYLwV0Q)R$7Tn3=M5LmIw0`$U%r;#q#HT^@s_AK6`{{>58VN;PrK@0gD8sLGq zw3fTj2?Oqpifh4OH&Pl!pc-^hXW>mhrrhu}A?l7=z2GLmi}*n$^LjdEqdsxOA&8$< zbHWtCtG%OoD{Cd+8&8*%=wBG2?#sYD$8SQJbj!!wt!cV*8t6t}PfG?h_CpH-^3Wua zi9>tGyhIT-u#{3JCf zg{HA8J;nm6Ncr~{5dvZB6&{T%DNwgqFguIrp*k6K`YRU@w+;hfBJH(0F)`eqr(a-L z7U2pg6IrBA( ze}G9v6DK>|EqT*_z$xDM^ws~7+V+!+PM>G(f<<|8JQRfn3?Ex zp#D*n|7-N$iT{_zly|fWM^&XsAO+oWJ2&Cv?=7o zK*07t{qyqvgEjwI{*SDz%s{~YpA1E(tjtKj@qb(Vzg`UgUBv%2`(N~vf#HAJ(aHY1 zCpv-u_)pt^@FxMCu!*aMk%^+1;Qv1snw{dNthA06I@`_GJ%>a{K=Op;2}!WPC5{TB z5H9go9w;P;)KwT%6a|q&)mXU?P%MzC3Ks?PDWRMMk-ER7k=F6Nx5PWhb-lKsLckbLbll0CCR zZ*Y@tWr+-c<5%*<#kVYJ0D6>ky{S{ueCu#V5fsdu@CP6Oh@>HoE=Rb?cMkI$0QVBJ zqX>|23BwY^)ijIavoX_;`$naALk6m(L7Dw9cjuZgb6Z7}p^<6w)rEi`w&rg+m(yQ4Vfv6mfUF)kVx^6Up7h=s!+{T_{6@VriF#hBH zD!E$%fCo^3rk$)uybx>MgrNf%GU&*wD{CjfTq|0tGq%s%0K19a5RTXM$(v7At`c-U ze;i)MYxYSwW?^lH1!#)m+>G478tlU2I``|Mbg|j5cpm6ifU~Yf) zI9g+AfHPu1O}+Fo-Xt1 z_*$n}M$ezq4}56t2-|&W0Yasd7noRMN0I_+%);Fu>wX}lIB@4YC}w|JdLU|jSe(C1 z_P{#?06~An5r9*GX~lso_(2*3c-H`+1Q^x;T>~xc0hatd=RjlwSna{ILDc$S?V(}* zit$5*^h?nL?)Y0k1l%E@76??tp%uZW3$7qwYeIMk3COdf01pd=#~~!}zt54DVNeEE zVapVkk+!oMx1!Dvf@^YKp+*{~l0mxLjYNPFBU|3dxF? z8BH^!svoPbRhL>1yk>C4g#$O*hk1Zw@7so<&1lO~1JVk=8rAvtau9Z(^xnOTatGx$ z0KVVp&d~c$50oE{AKq>({zxxb9Hb_QRRH{-6ar#vVr2v@h*OB~AjAP?1^+TRXTlH! zk0EVCiu#x}fi4LgQrScli4bCRMPiEdCVyoKOF|t|9@6eViQ}BcVvZOZ{}$tFh;s?_ zNZ=FrwbbXAu-W9CO^hG#yW1CPy`#G*MDPD#4L_VjOvVvePljqpFN!2WUaN({q3Siey6mh z^iC#BHcbXhj;)YZv0Fx2&aSYjkXxd+B(*fKT(|sOrmc*gEuD4Cna$Q?{yq8%y{PL5 z_iX#Hd>*{uxk$vy!0N%m#8SaZWgcO{%&^TE&p6IFWgcxZ*Mx2c(LB~XZPukvPQRH_ zHYI6{_r)=8k%ViRc9ZdGyJ{tN@|)SL09`i2r$b6@Q=805~ z%tqTr)1$toR-(D2#@C?G09ChXx~TgY+!)HR(lPQG4Y52lYMd$AK54Elr`dJwz4qbC z(5c%->y+)BeSv!+{3QKk0OtvH5AB9~U2}FN?+o%;M!Y=RZ=1P~Is_;4BCB7NDBaE0 zn4y?~ohgKdLBB&^qQP*Sci3;kag3)mrj62bx6fZv*-Spvy!@DwS&cq*9kLnPUgKH+ zM;8A-ti1ztXUo>;n{?7i$F^F{+h~v5y9q7MF#*oV+2t5xhtp zE*+)bbWi&4U*26_)^8WD4qp~u_)Kk1KU2&(|?79!S7}px~8hbcEbr^B2{bc+W z1DfPt<3BB&)TbR{C{ZDSBC-2LPN06vxkmTq=U#DqbbKxaQ}J~1xVeAbQ{Ai}iGFAm zl{OO(Svy@Xb1uD5%ZT~>g)&kO|8$Hrxud#CYjWg=^u2gEaT4;9ED#g(Ylxz(R%AYO}1Xc6ZD6eaC5(HNLg@ z^3|y5XoNA=rcLO^oWgH~st!G~+RK9h!Dc~)?RJ_8O<1dbD-^5qb(7k1i`67To#}gn z(Vfs8Ig+*5(@k}6)#vrG2mkZ-{D=Hk*DaTCu50VZEw^eZ2I@M?Qx#j<<@;kdIX8{1 zs%IT4`kMB7aOP1)QFV+(HSLm>7Gm>-CwV77F{4rs+D|+i&ofpAR~I`st$k;KBS8AW zd2x}k0jS=iyk)t7b-(`Fc3JP_Y0gPYr>U>5ukg9}bu;1k|uCN%f5d#HEGG1+_8L&KCts7Cg} z+CoS}9;>;hwCCpWkJ${I2^<=}v`??Q@Q0Mt+7VN!$(RY0^fZnvSDzcffq?6<5VRmp zRVUt-y1kizZwlsn%xcPr1)rfOqeds@^O7_H#HdUyRw`5BZE~>r=zshA-<9yc z)X={aZ&n7ze{|zNO75RDfIciCBqXTsXl#W4mkz6lul-M%KbrVID($}&b;tj$>itKX zb_Qt74ES_X=0=VHUHgx!4`}d@r;~r_f-`~qoR^J+64gL>H)k@zKunz!h?oTbjzieC@4i*M{ z8fJC|d{23WD0%QSntN5SW`+x9&#PBZ{NPk46_ z$J8%4T*ucN*GSmdTqOh%Yht6tE25*TLVri&3p~gu@P7*v-0h+4|EdmY(**QO@&`F^ zXnpWkjZ!^r`{Hu5Al*+cBoen+y)u zkG8K$Au)CEnc?VM_7luGKBC|JgPBMtGug?MUA&~et#dcK`-qlMXX5+oa#>H@Ef?23 zfC=8g$#y#1FFvmG3cfU%N@uW>IA?10R9k^jDjIA2GF%8>zD3I)d8lpbd|y~4bIow& z34W=sC8Is@;C)8l4__sn$V_edXqmJdOLct3xosUe^O>AXVXHs&Sm6)0!^$oyCuu~Y zX$}zgCo*}HArs~-^o{<;K02|>Vg3y0Xb>0m??@5J8HA-LlgtlY45 zA)VM*`MOA&H>nrC@h-PHM>z*HQ5=2+fiT>tO+-&sHWFV-B%aq=E7 zk@orsT9Vz4xBPX!ZRkCMu5)@{ox7BndvZTzzuxxFfo#9v+@CQap4W4}5PScUCJ=a` z4w)j!%j`Q&d|z$^BiovDG=eSq+E+@EKY0#?{8cx*U0nG0lWaUF=!?S@T!4Ew9poxU z3!NMD0`1n-Es$hL-B#(xpda6@99+kjl?={$DlFADY_ADlOFQr188*vkUQ3&@*BL>I zL0^5m?)SIO&LX-g0#2T@u~wVcUw@&n^KoXwB_;&@%KxbC{-xF`tuS#52Xlb@!I>OD zw^IYX#vEG(>iu!w-NBA>wK(wd3wBo9iE21o^HBs09F8FD;sP&TyQ4Z|qq&n;2i?3- z3eoufI%4_0TKr*3kJwX>I`HiXX~ug-W~N>-CZ4K4zX&;YhNRUhVZ26yb%;|vG2i9o zt>fhP0sXp6Nq6GMm~B8c?gH3p=lG_Af?LKCjt*fn{aRv(mVX_IBRK9Zh6PGcYSM z;43m9Q>Q$9Z=qc+NAtqN+iMT1kTA-%`k z=lf*k3??o9`bmvX&f^Vp?r=FM`fwc!ax$5lz>b~9#GB_ldZ~sf$3VieTse06zCit> z!6zZPp9pX;1in(9|Chg$&o|EA$dQN)-O#}h^_2Bz*AB#7k7$I#?*Nl$IBHm>Cdh0^ zc5l~+ds$j`Mm>3LR~*Yd3QU~Wyt~hqOlTOZuJ)EWMKrVf{&rZQ=I0p+n3pN5{ni;L z=CRfVX6ZNB|} zc!^(XvqO}7)fnF}y?O2FkCGAP@=>e=OtwP|E|!|@e6Kws=!%ZrZG-=t4qKk{LRlxtaE3rEhTHRb3=XQ**7>7T7XVnt{~Rr zt-NoO^X`~_erm%B2uC1kM(hae1&cPQHO6ua)gi2j?s@x--G|{5Ya3QSt4U?8Y??~f z759RB$*V#H#9d}3r` z)7b|Khbj(Uw15vo)`k>mdX{eY7s4(ihZ)M!v06*QRY+TS7u%F z53qLj?=9LIVbdHCDeoLc&#zJZwNo#I4p=GDmPf=j2)We}vc#77+Db>1SC+6LuB_Li zPn;EOdD|!sWO>d|>XItAyF(2Va8KHM-r^(WO~z2m1b1XU_DG_KxHDLacki1X;V*G} z?qgx=TSx@WBM~QUP$8AkYJ^;5T)1JPH9aZW&`Y4YEsMJ@95^m%^9HP#cbw(a=E6S@ z$&Zz>aM=lRSva$mg9A@bZ`C-s5m1mSi;{mdXHIioomgrxEj^O%%qgpq+U?ijs*)8?OA{F&$Kjzs?+t&|`H6BP;a*<3o?O%$4;&}D zQt0eQ@Y{;<))CkLkD$huZZ$TQm z+O?$KFln~M4qPNWREXLr=l9zf%%w5cb)baa>N#6;pQf-$ z&W%``XMPKb#n`l;boNao&{|-!!;bGxE1+Y6RTJu%y-9T5oaCWcmKRvN=-BE(X-c4aQrvMdhMC%^F5_)q# zrLb{th1l1w{Mgr))@MrDxlyfcjIcY3(|SYhR2=egZH}{DhJCdeqFO`uWj<5(cU!rB zsceQHM5(ItZg=+Ggz)+A$N9jIlDRhCQ6|a>q>(Wt%Dwvz-I%LZY1PGkXnl45kIxBouM!lihTC)AN>$@%1(-CW4$O9xWVeQPvbjE^)S`@t3$C^O<5G;#sr zpo2Wm#c}U7=9Kpxd+r1i`{u7V3+`QB6*z-MHouaoqQgNAm6{- zF#PKE!T4l`htc+P0L!7+;-S3o^LyEbpd}^TF_54cy_#|g_J#RzFI^-70?gR5V<>QD zL*%0OMYgZrW~ZBjPGS)0o%n|tH;BC{DFg^ka5Q;{_O;9}@eKTr34-R}kJ}&I$oMCe z!py4SOLg(_!1vT*V=HLv{3!5~K0jIz_cZ#dK7GBwqA}}O z3u-x5Y=JVn_r41>C<|}lS||3qDoH7%rxI`3hy5-{nBVGKnBPu}5f)oK0yqT;y5DUFaD>A z@KM+?TQ|?Nym^oT?bi7OP@>ZaHU2caW~(+U1<>m6d;I5A0fu~iamA`23aLe$>g)O>cr&9_?BUFiAAbVQerOPXr(G8=@>>zW3~JyV79k}}Jy z?oE}V16v4u69q3YKIn0zE(4Ve`WHS1sCllqle5M`vEd~t75>Vh@AyD1-nmkEH}}Iz z;!%t0v+cqXW}|LX#HZ-K{g)d=cs##>jZAbMQfTCX;KnCgC~Al`F;xeLNbUv4utVBO z?T#dF!r?sgUz2+f61;GzR(*OJ8U-?{7D!CLpafF_r=s#44$o5x)K=EJZ^p`6>_VFl zxyE!|)0Gb$orC%{6!e^x<+00AP*92UB#c|-uFwe+i#9Amk;=8dIDXAlM=PuWfq1fH zEq(iHIa)T)Od_-`EkP6z1BarH<026!%2MYfsMt}VdK}?SNGHi1y13K`DVv+(T`R^&tilv4R-y;_*7pbX4HZtlMqGo z?CEv}6+`pE8uio)?oSum)!08BUR$G z*&0>zH8t)phV~dG`DFxIBjAZlh!2o3)~Cy|Dn@$tmrRSN2ZqK%YiZ5cL*4DDwh16U zxtw@}*f5Vwp01ep`Y@DF#=I+&b8N1B-q+*%K7W+tqw_PO|Zlb`PzOi2caG6Edyq^5aM!DsQZS zbU+FG*}%|n%$i+kMK9U+TZ$)wVfi*mEL8@i4nu~tID2rZH9K(U0)O5?ffpzV z!ZH^l2hLIOR8-JUFx_k>W+-!RO|N87t3NO;!WpS-3uPDEFg0e7n1U7&+l4kh1w~3D ziPXl;Vf1HW!T#5%iGvA}tfm5T+rF{I;`^MLYO!ZN6HZSlM1F02jNI}Om3hBhe%g?)t z<}K({{aV8Z7qIb~Hm=7_l-erURIRiXjHK%Ji_tM*iO5aaq%8+R;A(y7@rpw+MRY{t z63hn7L^SY)3SuhhPgXunH% z@|}#dBdtzWeos$Dh2@%aT#xZSxT=4&UMW0f^SLWywUGbtQha_|$#*(h=hJEM{>UWr zq2~Qq_hx&*EqzmHc2d%L8?CLR=*$jKqnyUeQKI#_sr9ts>M%Z~GzoGWntPohit52_ z-8%fBRA5VuoQ{;Stdq%Bs&C92ucHY!15dfUV-7kLxd>iYc2k|EoE!;Hs8)V1g=F&# zcXo)uX0VST^e*n7k)nnPu9(pgt-uD_4h1Q=Ia`I=yK&mntw}agZ(|dvRM$SYUOW%D$9Yn?a`uPm_`J;ei+;78 z%}~hUDs;TQ`QhPE8*%PCN^h_d%5;VM$?w0p@{_oyFpNgW>Zf`(O&paIaijcZ4x! zNS2pAkhSVb%9>SA?N@HiAa*L*3yJtBmdN_C9TN;w^+Y=;p&5L`_JG^bZe{{AAudCQ zuKr^X26b}jtP~aWhXXU5n5Nmdj9FRR4Lpm?f`@hs^cR~ClVr~6*mbu?`)GLN=vY}bKsxVLZrBba%a5P!38os_?D&XeQ zVWaCjbxq}u{90@ZxVk}HG=@vlyv7x7i=vh_lGyMYWM}`* z)ZP7UUF9KC_+B!F;x&T}`dP7Y!l~CYb&X$&_%wD&yW1+xdpzk)iJB}h& z3QEL0C9W1>!P7RhIVw`A>NnZMh#kR-vRlqJLI@JIgfZ=@gvoluV^f0b^Z{{_>iz5V zt&GkDdGht)2`Q_*{dNl9E8!7D$*DojLiWv$?!ECSbwA%rqb{3B?dD#uaQEM3Z+6fo z@p7E)uc7wGww+fK8sisHM4p!&Bj7ST&xRIsXAjNKi#JOdU#Rzt829>u#l1WTn`UaJ zM^J`e_%ZtyNk#)$D;Loj+_^U%1;4eYP}E+-Ua0(b3US*#hMgyX0vx8jLw{FTpu^>Bm(OQu26M_w7)jFnS@BsxJLH5MZf{-$$Y8Y z!tv@cI`{J~txAcUiO*0cH+@@Br`s{ig;UK|)5~w3VG8L&>5I4M^2O3`!^gs%PUDa* z>AUaD_paQ(|ISg(^odSU6P+f3kHt|wf$mY9f-Q`Luj3xJ!Q^2F54Wh;Hbqa3LT zfR_$5gnyxj3P_$Be4=CX{{G~#aGB*R^t_w&on2N$hhjUi@h+3|Z9|t5&G+RLsdDNq z`sq2CD}J(}Ty^8SEEQTji0|*U+N2MUy@$2#$#$W`>1I<|`;%mAzc;j^8l{YH!z)0O z_Fwj$VyVRuB92%TwS}X8Eb4hN&W*Vl;QDk(LDhit#GaIdezEgQ0q#PjmK6+Z%-EyN z;7&o7{bIW`unzb=(n#js{?Zl)A0Kqho!PXM*dhdvC3vHc@jcm}>PS2I)n6{NkBttx zO9J#`>A1`2Ro);(G(4E0S;rz8HMI0;+>5J0@=ajI_OKBuoqS;5>Rk5RH9a_lJWS&Q zCo(h#46COO8}0Bp80AYbLVqxda?TGLFsq=SdDA0=WJjQx_Zmc-In7Frvb~lqo;H1b zc$JVKr&Xp+&!ePlEfGGd#WBgHf;t@3e5(FXEsUEPRLLo94=w5yvE4|Fwl+u^tNygp z(WuFOMi6`+@t%Gwu~GK72}X*~w+ePe{+>k39_Boct(y#Fy63g@px&qw`aPoTnNZ(y z#ym+zMBuwDtGca`rDq-^1ktN-fvA&8OP4BKXOD*~Zd@lLDcB5KoHD$qkC>g@w z^bpBU-N>G7Zx-f6ytuXUuG0syUR_Q_J-OcDVK{9CRVQk8Lo7O`vD0dzu_d04L^!XZ zSER%dDur`KHvZE0@Jm=6qLl2_3n+)kr5!qDkpS6`9l$SGTu8F+Gc^4LQU^-IeafM$ z@YDBE-9%jJDF&(xl&o6Jx^j9-*0tUq9ik{i`;|RXmjZTFHx@Y&SCgV3a?5}v9Q&>J#M|B{yGFXgOG=K>W>?-b7mZmM4rY%eL zL1H+6u*+07p&a`vfS_R)KL%A7RUM=;v8dnFP*$ikA3||iwJZng@Kw3 zLRIP=p&pBPa)lyHLrGmlpoJPae_6gs4|4K4=1xF)5Z;Ep^z!dEySo4}oo<(!J3cDk z&s+8VT3^K=iMO2|%MptGt`ymuWpF3@ardB9I&Kj(--vQ;wUMba$@8rzq?XV)>&)ek^a(2B&b3Rsx4KaPhHd54vUJQ4ZOK6V|F$ksPQw_|_p&D;iZdMkL+82~4zILE5 zVK~h-N1%u`AEG40DBwqku!x2_6%;{cFhCFq*6@$eTJOz*gd$QL6K*YwOK`8XGiHvJ zUm#l;RCZBE9aEcqb{jb)@?}+Bc6Mr{gcyb6!=}|P2ybTTupTL8(}t>kD!n0Ne@d@< z3TUgDoS1rv=jRt*SobtYS=>=~ISj;zw^YgBWyWstrR?mQ1DdS%Rn^Ly*hP0I)=voc zGizTG$LdSjL4GAvaKRlY5Q38^>=6F6BPZhQ=3H#)n$QFhXr1g1`Xbv3UvDDvMYvQZ z>5B*LiBu93VWeK)Fj~R5^rOk>w74`lZztM{oin~2Zt@uyP_`Z{IxR3lNmbq!2beVT z&(~&t#jRJO4khCdI1~oote?UHkV%{SKbq-&Q&$AGT{~N5Wx9?t_j{(VHak9O){;fw z5=pjgr26WZ!b8u*hmuI{+i$QPZVys^`C-wB5-GRQSj)~`xqpvU1HK~EP%^E{r+ zg;VFMS#=2t(!tyNylsk2bv11^#YjrUgnk3?@gp32m{=lvTLpRp3}M2rnLRmVLbWYz z;-Dg%;NU8HlHnEK5R=&t$imVNN>WYLxHl1r8PWE0+!0`|3_8$M>Tho5{gVBbhHaq^ zVx5Axl04R!<%URBl6jLv@0ju2DX=8EPI@KC5S6J?#!H0y?-bFTx~rhf)KRAN)wHQg zYFg@u!?HOjn1yzM3d$k_dTHr7E`-eaDO0@kCz`y40(qU^>Q5KHEIotZ*s+&i6)=@+ z4ya^$IQHIKCJcU*y*Zz%WMYXawYBUJvDIhB11e z8L7zfxqO`0KYl7tF|FeMqV==&`_VwWou3L6!!BU*leKy18bP6{=z7Rff5o$cw&($; zCXbfNUos#rbU!InguCXjhff#@nK%u+bqkA0H?o{sg`|g;fiyAA9VcE)w{~HeE5pUQ zZ_9Z$czxG_4`anHAR^KD3X4&$xlce z6JAS=$e@`3yzFcs9TKW!qKSjLsQBBCS-LT+M;qITY&{K%2;wce!d`Olfq6v}nGC8%XSv>V%BBO90h)CWdu(s;F zaoDj_;sb$nk=6vi%kd2KyM@iyPCNVzCr85tOS?i*)5k7qjoWulL2we zcIu-hpX-$-bYKT5LNGraDtVnKegQ*x2(r_=HsZ9U(C8xdm}vLyw#eB`+i9eho-GEQ zerVfJg0rb{WH;q4NGF7)nQF#P5H0gr16`V~1)30MDyV)Dohfk*^-{mj8j@^3&#~N8 z^gF7gRO5rO(2{yJiqcH$68gQD{#Obs>W*th8?r6Z6PS7p4rjgrhcyA#&XQPltyrhg z!8Iy%lVUGuBKpc=b6PL%4O zk=U{Xe1qU`bwi?Mob9lgq;f=Vo8prpHMr|Z%vQ72DnaX;oDo12@%QMS5~nOTbB6Q{ zO1&TOhCHCxZ}g80w|0|k?0IkB4hB_}tJ^3;w|)$r(QqYK0@6C*TgF=^Ipx}Yccnwm zxmuzpF6FJ#pX7{rA6@%EZ5>30V~sG`VBbkImVl=@P%`yaq@3_IM1hi`zwpx~&P0P0 z4$T#h$zel4XP%?WDLn#t#gb1R%Q%}nJ$9(sx4Rd(B&wH0mxIw{=@1*6znl1VGA02I zAalzPE2HiUmzhY!dsj;*Rc2KDguDjNPv+B|e3rUbR#Z-yq%}S~8@qXsntXtf4NLH> zIDS-Wfsj1MUl(s7Jc!3&Pvdq`&+S41PVGYPg*NaJ^8t4h*CSIFf~Py(9r8a!$~HD9 z*n!3OcQIM2=1!A%u_M^IZxxjH5dS_&l)u!9YFzVU_K~7BFtw<~X}`%ryA*H1RL_Ej zUp4IN+Y@<;TB->+Smi^t!fe|%8=DREIC&~hu^qDY=^_1E>)v1j!Mf~qmK>hB&Q>V; zm&A0P=>qJ+hRHNnYjy{-+0@2?2h6&6f*?6V#uvzfSn^v?ge2?rIG#enhOBILSio~P6?Mst zUd}~{EA!#LbT1j?prfEc)!VB=$MZPinAPFsG5geBQr@kJ_(q}M)Z%C?mZKLGWaDxf z@7ny_U1MCDad?i|sml0*pe}iQOzSu%S?y(d{o>4!G#KPMnD?ITF4am5##4$37N4On ztI7^-0WJl^D+Ud#9iMDvau>TDf*b#XEbu$G!X>llPcp*k78Zp-vcM0R%%L@)GQGSh zVF*^;gir%gSnQnygAmV4$3@)RHseXSi}sUkjz^CN&<~fVbEvtONICS4mybP9*9jko z)xlCUzz20`J`gM2?gBzd_cKhNV8cr@pN4yc74VcY%)`xtuC!XvW6rfwH>Hg_b!(2< zJEiCjR&U0aQA^p1qTG3YZjDT}eW<$dxucRV%^RLNzt(XI7HdXzJV(RBp^>yWafHH_ zi)oOR%2ElX%Uf>D(m0mZnHjaIG0l45nBvD@g9egQ|9x1_1W)H&6-J@ zZ1=rUt;~2KSu7-`_rKLWYSecC8E?HJ1Nd>s!)+~UnHHk+lK z9Ybt7bQ$O5Q^-qbeRF<=(}fLBI8hfYcXtFG`}K6=Bl#%8cwdSkiVV|}td57|jUVRw zv@hAPmj&t+zjSKL{vIP@q2wq(VjxBj70-8MW&t1*mD+JDHYiLXVkRCcNycD4Nf4iM zsZhlBUgUgHr6nxEgkWJ8p^7RCSWzZCMeR?;dl^}qiFJPap*2!cLFgj5d==kU{vYsX zX5ZT)kxOm37Xm33A9UPP=Dm)ouDfOUIE0X-oQ$bn5Q%e5lOED||!1Lk^Mh!YbA4MQo1M z&Aq*49@p7swkYF-YXgm&jyPw+r=CPT~pSm-qK#*cp#y0E@dA{k1fJ7~R zl_;|^M$#AO6&BsqqvCj?qM`Ms&{{Vcc6jD~iNSbXd9Yr>f1tw{w4b!Q8$essPVM1q zu_418h_j5B{J=IMDY_8jibhv4!6Sy7VI7KCQ4Tzuaec`Ov!3Kht_T9(9UEiCnKkao z`<80eDF*l`*UU}+o7eQ2;Y`+_gM8>A=@G(j3=@7#$)hrC89^P|L%Cj%I3WfXFiJUtsELwX|DAI@|%8PR#7G92jWh9i80#6mYl~H$=BJ~+JVL>#xfdTr9588Sl}9P_aKFUG5Pd-W9udDy zu&%slbsv=ydywvk8dZ9qck0KfK#Ng%=yMvg3W3yGJmW{%q! zk1uwDE4_S^!^Ehy{0E*cUWr@F_4VHU+U&SDW-q>H?=ckc zL`b0OA|yfMGUB8Pl&kkKIty-9pPSDt=}bHCudK#&H_EUmJr{}qfj&sFU$DVxsP&DvSmzaxf-Ns?7J@` zUv`Dv&OloYk~?(glo|G}P8wKIewuRkBwG>+;<)6}%6*rtF`CJ1HcA>!K#en$U%k(q zemV9KV$ZpK>BYq%Ya%4*9JLiz%awQ(aZ96(G}WlN4rEzB95&S23gozYt2uu%-&Fo|7PadXVL&B94$dxl0fDez0q_>7bT2-A1X zOAY7lr4G*eC|*-egmwi2$(d<}=@5SpX@#EFIo{%R176nKA|=Ctt~>*c^jNvJv61fx z={8cyul(ny&VD<} zOPF!rGgsUk&GqDOkVwOnM)9}U>DH<3o#dJ;=I8cwWSoa;*qj|Kc@HYMAEimG!)^It(EWC8rHx1XrPVa%ysC2dv6uwysgvi^Hke4-bC$5v{=1G zE&#;0_1ICpw0j3Lf?2ZzKGJ=UnETOKK!M|>u4@mMn}zD2O^CUhJHYFefuC%|WICf! z7nnR&kS0RrX&wJO+h5`%jluY z-tz6P`AZ6a{3xLo@oW*2gY|{((;KVA)uHY1Jes&hTuPj{Kv5T#6W2O}YNy-x(Ej#- z0===dsJ)aO_hX5#Slxy#$esv64Le#cbxNFixQQK77O< z@~H**Zl5gpdrgHT8`mH+Q4Nz|e?}QmjsmD@GvQK@7tEf^W_$=WFM`0Pf}HC)6By^g zKu#1|IlG36e1i&A!3sMjZP{a8hQ;{u56Q^6v-Fzh)Fm9aG5}gA2L5S33S^Une*PF| z&SbgUi-vp7ciSu1$w@m6;?%daVK}h%JuRS!kjZpvJE0aA4gy&(cuzU$D1+>;*`d*gI8lO0*w)^!wkC14IF1=Opu4IA4@gW za6H73${X5QWi*ZY_J$cM9iE(zFCIgjVj7%$hkOAW2DU%Lnr!gR z?!a5tX@6gBL`=t~(;=Bt7$Vgz~Yb3wmtgJX2e zvCC~(g@d0Pjae7dZa_W2&>v^DMrnol_Lme?9Y}Y6kalAqn zJ>QI1vzf#FKDc=O;EGm{A8URB4-K({mztvdA;DJwoch}0a9TR@6FcV& zEu_zS2*XU&Ks}>MDr~zDBwyeq;SJNh`66kJ;R9_(;u){|d;Pe@Y7i%}^cFuDSw>b_25R34LnE5Gbti z3$+P%CnCq?&n(@*&@}WQXhJulZB^bf{yYqakWZ?r-B%>mKpYSJ~8SbTRg0v#7S<{zn2l|Qq!9iF(1?%k9uDrGBd=8-Q+QcqMmk{d7~H6kvz z8&#c9PjzG~iu61sXNsZJsQffReBTaQqvBf zUk-zXF!BcK&)&;)No_QD)%G{G57f2fZOR*2)e5Dco&#nyUQODD8M1x`PFp1MA2k9*-;xiMqrl8-i|rKAsgmmYaztCFWc zmR7}8{J6owxop^^HuXFMh+n7~QdmrQ=DXgr09nX8aph0EF;@ArQx{HUSM_zwKxO3y3Ym3t zh}S@!;#V>=CA`pg1=x{?f&@EP($y!Q;rX>xDVn=-XYS<&-jHRsZy)qkNIo&48F?Uu z$;sqWpFKhygYs`c3cpabzZu5h4!MATew7^l){c|95wPH#=KnaAQ$Zd8{Au+F1C($A ztFiRz+w+~!%~VYA>XmK@($uo|P!}O_3(FR}^C{DUN)7KI7p{yACPWYx{m1e#n_^Z$ zPwqUt0vi9V1na$r*Rn>0NZbL82gfA|geL!o_AVW`n%T>KV(vCU-#&A|XAxJlQdM-` zb#E@r?d|GHtS&HwhHoL)0WdxDTwEd~A~0Bc%@wT9(!jJ%X6XKDvUCw%nW~<`&Zd`z z!XQYmF&CvV|m2eUMQHs|~x7P4A0&hzp-Ezc6I=o3;>7$2<%^Xc2<0507c5m3h0rA4M4`S zK>h)YSpfm?jL!k!UH_#2B_p8TpE}F{Ae)7i1)rH7P!EvD!oh^k{-;e21^|@J^uIFc ze2rnh&l}E>m)OLx~yko&Q_!j=+06_{V zYujv^I&^a6Y7t19I(`(h<&lDX{NP#0I-&#w1$*GR?nP#r=qhOFUG|P1=-sE5DDqoW z`1(jr6s@80$Mm%L{%ddF&dW#d$Ab;e2G0q{%zIzii;?|O3jy%)u3yZ)63!sxVt3+R z*y)>1GZ+)4XdqX)sLa$_h!a>kKwC=zVT-ii4rz*ks*A%cN-cZbn<6I_!m9Ao$?7@< z!V=50kS5Y}`wnx%P7lRSa{2n-@<`?_33q!p87{Rn+VfDe-11(p6IbLr4f10H?i9Uf zE!1?d%;iRazb6($^M8x7!)ilnTcbx2CI^E>rx#=n2ZQHRQl}S-_xMuzo_}3o@0jb* z;pdicb8`FYdR`cEVV<4zD{82AUVX>Fs#IlV7V=D$o0bU1LhdSNYxJZHi={3R*Ty`) zcc#rVKe}em zHG8e<8ug>bJKx7*LPM#7B*w|&^T2evKN&V3w1}7j{05tkIDQtwmWa#~KBNVQ2~URh zAW%sThilpNl?u-R4l2yRBW<>85V&iS&NOd2*7`fD4yax@4B>dfGr3_D!Df8k_ud+Y zwHyc?tsOh>k0h0u79H9a!T@uj2NRCYiCaK}TiNmv$UC`-BOtnFg2guk`E(!5C-vi% zpr}s^sde;g1tA!{aMgbJCn@Yax7a)*t+TNqleZtb5UB3vR#4=;1BvcT3yl-@r%+p` zmlEH^>inSLf=={iI_=iW0~i3!Jfh-+wF4@Jz#g#(0%RFqL_AS&j;9d@b%yxX`!!E+ z{ZN|qsU|Lj2Uc`MVq}C&R#2pvKJdnBRpgRu)sg)7Iuy@M>K>y=TU5G9WGzEL^!iiX zrpL2q?BZzRo@t21C`8bYprz>MfSuVa{U^>;mA&}qJp%IERtgIW-_uQU%e9b8&FNlK8wNNXyIdHEslkWbaDyWhE4ih9dO{Ra- zYRV7(npMa6gV$K4uj98`a7}FboPxo~LVRC*9Z~hi?3(M3u1nsA`2*7lH~i0|TE$<5 zpO?GD$7+o8zjHgHqIO2!e);Jbvs&nQ&BZ?luK(G!d>g*(A>rVw?cl58u*T5B9rGb( zsHNR}&7uB7DxVy;+JP2*C9akWpz~X4L-_=t=v)3=1sL#ILCpmqg9H$i)WXDN%9g?~ ziX=kt=rXZL_24u-MXjc0(NVe0&BRVV;_ucSUk@nE;29t99Koye1}WuoZc@IIkT7gn zA-z8HJ{+?;fcp|E@;O{h)S`a2QyDlH(K{#H)aqx__5<*#b z3CVX;7x7n_#|XZAQswGCRXc-*S!^~ZwvBN&;#pV`2MUb`>&~l~3p7HNgeDxv)qP|w z4KcAgB%1cvo~dGF?u!d%$*;S#Q#{Wr29R^yeN*ZgHmi;EwNT_xW{}~ure2rGk4-X= z(JJ3=y>Fn(>a9`g5mt7be(5=|8}Zck?>H&wH}Ew3-1&vD4&Y?JN88!zojbqMexhlx zbJH-UeSOGLIDKMpQFguxe*y3eYHU>SzOY%qo;?A0iZcpn@fCGl(^Rzd>e6cybW~vg zI!Ie4Ry^1AdOA24v)Oupi-fByr{_uPkQ2ijM)(N{bclC?x!K`!jRn$PO5M`QApt1eI0_qu($-fQ~=_=wa< zR26C?v&Khjtz^OY(+Lowx7sqSJOzC+g9t3gaK@$1NqH+eZKEvBI^H?w@O5GsEkx{< z(AR_HGe(AOtJ7!BM5wkn<2`WlH@gbdgtOmdz223odTSqZRWFTQAOMdfO|GbdGJc@F zJKR!VeP{hOT(97jI&lm`0?H77j~npaM5!}|PnseL+_VNb{Hj*5D!Z3umOz1U`O2@4BbL_sW8KYMJ z>7|WVOfhxsL{Qu%0SSstWVBLf`V59MgBJ2Y0cJ<9D&i^`Ni=k4Cjd2+y)S2Zn&2i+ zm~YYf9-&JW%J~_)(Ts7*l;A1JbhsXALRW%-T#A@Cbw63#5{rLw!6J_eA~;&7UTN_q zZ#pV|5JI;fltxCe8vUKloLVKA28a)uv0WN)^g`O=S6sKGtDx>2bz$JqqI|@BfFPdk zcuFPs>n65aI=z>8Z+^^|>lD6;B)A_OdchIIgALn6P|bq2nOZf*+cIPgHl{2XRf!Jwb4{#L{NhM$Zc#I)PwND&`P!B8onwxAkK~zrTICt8amgP z3-pvT*s6q^b|1)=V~!dx$-7>m6yWg__3|5iQuGQajg8uW6EBs7_&r7`Wn2%G>ow{C z%3D{}Nn9w7YQp8fDF_LBExlNBrg?dn(qgnqlxPywKTlWEq(WFOP zu|w;^>ZD`%^jYz$23r@<=yGBiBm)nXo#Tco<*2$;JU|EQ^xs?Yv!_k(*(Oqaw+(Bf zm1R!7Nq#-R-4}hKXuf#s+Iq=129M&2{CcPB)C}np*mXa0X8?3AN0rHA>1?OqI!Apc zmNuWnM@=KbzDQ}&XizxGV38tTH*fbHpuWwYN&{?~d8c)7!=UTkZh2QM{{wtcI~;ss zh(6-}PEcfyXGO;Mt@oB)u$0e@*L zK^sf0Al}P@H6#Md>p78*oa1Lj}bkCr47XS?GkfzpE1bF0QlG$D=*1W>^Hh zyA*sOh3Sm9^lN!O;&JSSL5x zxjlA2e8~V@bH*?p2bD7tIb60PJXBTsSanF{`A46H^Q&`dwmaCy!~ohRVyd zHXaCyz3KGr>FO>P(4eIw=|M1%C)f!GdCR*8u4bhkQh-8FKJmeYg{K?<9;$B&;x@<5 zk&EU9Nz5oxQuh=IEV$Gp*oi)yRG9>Or(f}?vS1(O068{MG?0U-N~0IEPwMJ7mF+X1 z>>ay|4`if&$}l3E{;GsHJr~Bd9QR9GbPwM4Xf46I+hbr*1 zVxtnzuygFqP)!}y4ZgXSRHAgO5yKVq; z?!-QkVRK#Sk9nIh2x37WqSVHYBwsBxz#SeWM(~Eu@7kiyxccR_V9$6iA6@;9FvHHm z@HE?CS(lf?WtT+r#7A#Sdm}niF6RBJ?IT0){5^R%XA1j>-XMigYLp(~rczP*bV{U1_1A9%S3)=&viZk# zOu^6bS^dJ}zk8<-ObD|jnrCSI04hYfQMy_o+KO6%8l4N=C-bB+)}@D`9%WOdYiP&n zZ~S+oKH{4>GEP544WI;j#qv4thB7BNv0{+^8jPZk?VVRhwR#S{7b{loM8&U)hvZmt zF3*<|Ke-C^krzR;%vr11+7P|@DJNLy2O`>)*fnuCo!c>%=vy9qxUfZo2D$gtN(vI% zxcoSv)Rf`!Jcgaoye%OjkNLHhVrP9q@y%o^MXTO3-cJ>-QL{;5myK( zoceDkLu4#nt*gRzG(eng!1JKkvG>+3PnPWmx}m3jpPkvVtR6;FOXIRy9F|jV1oU4P zO~#^^Op;rF|BPJ^U>AePv&+*(L1m$~4^&X3@2DLkW2I0kniynH+(V`xNU7x={IA`J z6=l?^dtl`!gR1=qqKS)Bd$ z7M}q#6s2v`!PV^yG&DNA1^pmT-Z-y5n;@H?w34x;O~o0M8n%O5>fJx@Y31z!e$2Zt zo*oypss0eauGYT4^q?C&-Sfplx57_3An;q&>b0lSJL492S4Nn@QE_3xnR3~J^GSOI ze34OAZu_0|0}9xngBx+3rR{J&|G+V|ko*zzH=nTm;_L4qpPEJ0^Q7;rHfH+q1kF7I zIUEE)C34cNTr~TH7ew}~PvUFv@o)Tu>aLPS#@_Vl%2SZV z>$COK>D)A3PIo0Zg)4Lg7bFd*2qZWlu0@)TeN16y*`kX>XN`%YUL|k-_d>X*m zz`bSp_Bg$cem!1`VpjTk?5BV~rsEg|sk53k2vOU)u-KhyB4!n5NQwaa>)5h_>ZQ5p z!Jkso@1H=4ewKBGQd%*Ht$0e&MfF2^v)PD2YE`(^gq8l5N1)){w&t4hSjMP$er$~+b3~01B-0*_9r8^9lFjXk-+?n!Jd^NgoF7#Y(!Uu7r7?(fM)<%P zCiHZUKy~(f855R}ME(%`oU_h&CoChf{9x+t?qA}u@}>5PP(=L-k*xElYK+Q+NOO0g z{mxgaZr95!8y#bzde?bjj6?j*b}13wBXQfkI-PYvmn)@{AnWkT<+K@yo*d+cRSP5q zv&Sh;AGidUj%$C#xf3nBA^yB9+3li~IxdjRw$|y%v5@zpVB83SmujhjOXGSoP@E69wjEQ@k_Pc@#R^wO zU9WVCa_DikXd?6Lc}|*c8Qi0ixgX>Ui@E>A*M|G26#3_xs!qk_$GGtB`hG?Y z3CghxTTv)-bajm;{!oNmprnp-Ws=UhaXOX#=f&^b|FizeinroO^#`LCrauL<5(Zz~ z4ip}G9xe4ZsYC&txqQ)}p8HA7jXM=}I{%E{PAQ+U0CZ^{?iGh6;f*_8eS5x zclAxH6l&(YJXb@U9-<^m2bp*U+Uj;G5>CrBvp5iS zpS3@+e)8wyyhj1|3j;BpPI)iOXKZU{15rnY8T%#Og6H3gViG3FWBh8QzIsUA8_#{Cq%0IH`LYgj|WeaIN z_W4S0B0QIy3Ma;WJ*XZ?2-Bq`G+&AgBkvyw`aZM={C$ zNFSG1uPBgQ6CS~_vYTG^mO&A#xlXm-AhrU(eY27cHB0H76|7Ory?aulgK1QH=lRp; z=|-|fnVSSpBU1S#`vbO8_sz&YZv*7KJy2)cM|c6X9dau68e1%-Nvwe%SDI(Rj!e%6 z_k)C8giNxGj;2wQ45_Ip8D3#^ltkPHr6wrFC^QX{i8*8|soLTAUXCiR20gme4tK{9 z=Z*SnpR_LHd1Acg-eZTMP5d1BQawHOOoy~FCg<~@LARKb=gk1itz_SD3R^po?&Fc` zy~9Zexm2V2SC32lhsGvLso$wj_6#2&flr&U$|#g|YwJp*^GlH9+!D4JEyd%1!wu&Ro?MW^xYHqKYIjI5dw1 z4gkCO3aEXbf-J@_lC^60%Lasef0D-IK}Q?Mc=_Vtbl+QvSYYkalgr|l1M`4i+#i`R zrk1N~qi6#dS`?Y$Th;|~k8n>6rPhSr@DKY|;k#wYMpzG^5~AhvS(t(~W}E9ltfAKxHJzJ>ex%EgxohtDQuHF2t48Y#{P~x(AOE-3ry3{ zpXe-5+=4~mb(ynd6W@fkMP_~D&vfVpT+mW|>q%OY#;{$Yz#NqfP5{aGCG>x!BXW*s z#rz!GX1;Ja`W`t9U16q2x|C@SHRig%Veo5M6V+yvvqDpWc86JaO2Hv{@P<{)!HQSw zFuBd?%qay0ym)EG#9>?8y;mz4Se`GoVwhalLPX&iNA(%vQ%F`O%2YU!yE*)o(zlM2 zm;|qkQsf#cG;G|YdUAhR#;Q(BjBewo{VIixA+`uVO!p?oE@wr+Q;@0h$#kT}OG$om0# zTb*_UK<}!SsaQgDIOrvrvXK2A{w<{H4RhkFkRa+*AW0yJ@{KGy9Y5wQ{(=`8wm9L9`Of;M;XZWz4B9eV?sLbD z2EF4XXvDAC!t0oL3B@n&rDLA$ScOKU3pC2<4B$DuYv8HnwnNb~!8Jg6OK@K^yez4f z{fHDy6f2ke<=3u8`G{jN8Fsh-XvbH>#cF2Mv1HBT{xT|bJvPo3;^$fIMBnY)o(txz zyc9Rx$z*!^JR?88Z9{HfkKI?(6Q09Ie;SXPo&|`ZYH_tvsQ^qpk&|S&5hRd?-7bh)P|i?BJXR6 z=4-eX-dDtk6a(T5BGXj%1bo)2(rB||FFDh>ybao0`D^>Vgf)XYA}1U1^^z@1$dJAj zhCCekd$wG#z3a&>4!9bMwH?zBws>qY;oL(A3ZjDTRX2^K{LT+aeedo4pBAKZsjgSn z+~2-`I*}{#Ya+2@O+7y1ZlgWHzG_Kat)!b*mFCuN zxWyKSBM8e2FS%_6IqDEh=bKyWN_=}bh1Gfs5tY}-POX!mT#hV_XjZ)lX} z7j;V+ z^|ahOj0Y~maSA}}i+I(Ts|2Mwb5=lsSD22`Xt0(#osp{F1ym(T z4jHNY_1^tP9<{xbJLl?LB$dj~bq_~gRDqd)$o^oz7#b_v_=1HDDRt`tpy~TKUW2cR ze=eMCMD_zST?}&A`%&0N9wnFj&&i}lCvDVE- zGg%^VQdga5lO@^&xQa#F#p9AogeNj85ggKv_Hw{uOWU3{W1jrfj3}Wv<3bmVkn6Cf zCZWSlqZ#rs;_8|`DD2I0{Ush&<8r&*u-(wMY+CZ02YJhs3#%k$K`Sh&LP4rzCCiTo z*;gi^^nSOLn078ws2tzfz8W$Z8&4u!(B)gnr#+PT4LwE44g4a5Lo^Qg1?9*6{X!3a zRQoI~0=VnCVpd=tgq(`jKdVM$I0dUSu{|q<_`WJB3F9sPac8H3Gpus9s|V8AH;>Zp ziue+Vh;~9TQpox?GKL0tiVy;y8Mc$YF5T~T4gV*~( z2yg8Kp<L`QJs`+R_XP{ikjv8IF|?`hTn3N2DrNc9tyaBMWfpBy{2E_EYI{mhl}9nz zjrfcS?+u=PW!t+k+oY$()LDiy3FdJq1uJ9?f1>yab|@bu9nyK?Jl3I%)7il~f6ts9 z8UNbwB#O%N+3StyRk||`pQ5p}K|5qy?C?3kmN?_{TQDgO<8%McaKqZ4=BSYibo5J> zSlaZ+Eh7A)X4UY{UIfj{@dJ%?>Qkd8kHYta!E{~y86|I_yHfw?*VOLFlqiVH1UH<)ER zlPoigdf@70>26`=O3n1a%tBR@TF1)uqpg!8jK;vu$;}R?He!-+w{?Kg8Th3^JUm9s z)aveLu5LcmFpSwhkZ4|R%;+4Fu2yC+tN}HXjjNgSdkvYJlF}074Y{MY%a{m{@-J~ ze-R7*8vp$pygab8zsF!=q5oV9?_W0gf6wCt^YOq8?Ef17QjeP#CUW|>7%%8Q=JJ7g z{$p=^ynn$r{x%o37MNt|-($Ssf3L^I#Sddg{5=nLO+o*%UpH4XmZ2R%HvGqp00!}L1JUW}rB!9n{|~f}51ar1 literal 0 HcmV?d00001 diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf new file mode 100644 index 0000000000000000000000000000000000000000..03034a81084132d0fa355eeb74b3081aff8dff0e GIT binary patch literal 28598 zcmce;by%FsvNsys-6b$M3GVLh?(Pik8l2z~2<{f#CAhmg1b27$@IlsIXYF&(KKHwS z-Dl>Vx4XKktE;DHo~ruQ44J&JC=CNG6D(QJ&gITg!By5oPaiBZfF59LXaUR34WN@Y zu`zQp2e7=66ajRiKr1H`hqt?xfs=`_iIJ_b34oUu*3rqq#K0QX4P-QJDR#LpCg|(} zy}>F-kUk1R|I8_iwV^q`bjPh5Wf%d12iLywQho})?3?b~_$D2kiUvfMuP7VNW)RJ- zv>e;F`1pst%;2u^2eyat2f{YD(#xg(mfBa(mYuSp-|9QopI+Nr$NAa^`Dntcsh6I2 zSE6aO64UI+jx1)CtK6?dcrv#bXnW_&5qt};@C*PlaE>nGm zGgp7%6`Z2du3&nt&1pByNX>g3Bo@#$nrh5E9E`o5Q`T46w%+pDbswZB%3aSbuuX@x zyKSgGZ-&o=r4DV6{@N8voqy=o`E5w!!1_s4q&%nvN7^r}l-R#Eb63uR>X)2u2H`Uo zot)3(U83A2Me@s2d&Brcdz;&1=5M_5ny^hsh2%n2fBjSm&SHTs)aZ$tL_wew$tQqS7nt3)Dn8j+1pcn#I9zJz_&0{k`2vMsy(v4rX6M~Zh zk?K}cOf6)^W$ipm@As=k8@1E-4+CRIw$miF=$M3HG@*#!zm1-*P{8gl1B zy{>Wyi`>*^qZ*A_EH7`x#K&RXtZI3r0Tr7wGZXW})y$-1ZGBIPwBLPKu=GG+KOBUHTPZ3>=IiG#4%(GIkL5GjLy4-hzeUVE^NmpmhyIlvUp%%4tK36$K zK>R4uIrB|%|I2ZQt*>^W)YGtRWM#$TDkq)9Ge-2^4QvusinhJ9l%T8agr z)WYluBafh23|4o&mhv{s_jBJ$kX4(lLv0sL!Szy3Hk@a&q$vK;Bnt;~YDE>G{G~sQ zh`VX$osf#Ya5b;Uv@+ug*VhQ9I`TvlN`*)!;daVT8?qM5Csfk>56fE$pHJ~kf2^oG z?R@CRN@Ev%aN5{I;Nqe|z{@FLE%(#dN_Is?n2CPe$oz8t1ejIr$~?2yRCkSNIbj_P4>G0=L z7U?9Q;diD5KCzLzt=Ng)dh3J7y>xwuUM^K^@_C&e1Q@$q-w z>JDGHz#<=V$S|np2Z~C)7gah?ahWxUpZDTU4En=sXN5J&G_yk~qDGO$^ZQQ(++Al~ za7e{ml%7$R#&L-Ip}0NE*|{;VDj2Gb;6pf&l=7l(yrW=QLZ)~L?B7grpM*VVUftZqb@jbivZACeQK-cgVah^L<{D@Hplq>4*{*Iu-wV4yya+ZP zAT$xghbs@E&7im~YNMfxW|WddXa;;y)uVi>=_o0u!v=dp4 zGM5VmLwu9v!%(T@cNw7{hm2udSnE0AiQdqAsM@ut2wnbrPJrU35~LC$h$}5iW&<`t z2EDQk$vCmeriJv}F#N7hN}qoKXT`JXaj{7rXQ02FGa@@^5XvYsqv)Jo>lQND`8!ef$bi{gjn5|f*(}YQ)^hmh zOPDXwvnwKxSb;xMmZEV*Xe2k@Xcr(7`tzja?7SY1tzhP%Dz1}nLXoNqvS!*;K)|T* zT3Nh?kN8gr)Eur<==?0#mJg(HZlrJ-9=(o6i*`jVCd+V#S?!kGD!Fk4t1I#9)cl+K`cZ|T*pTzhP%}(&&>PN=m z9qMmr6GUyL@43<@c`B*agp5YcLt@esgNlJgMa5Zr?;41fo) z-I&*tZcrin*v}lb$6^n}o#>Ubk_tQ&tkQ6@}M>;umQk;h{ zf>#Jh>8=@ONLR%QI*AT>c##*3NpijhKUdpqn`+bh=hA~Pq_#;x=5+TZax=2P3A2xR z9FIJ1S>4v1;B4;PCqNk5+I14HwWi>sCJ8s-nyZA#K7Vb!8tkCjE+N)K z`Wl?{K$8@C$BcOM!%SOCv`n*)fo|aUVvxvi-smeHT@7lZ5f+{*r4y~#I)4I@9d(z>PTEm``1aryVHkVI>nJ=y!1Y3i# zub0VN4Bs;!;9`!2Ra=MGk7GB{FdBMujI*dFd7Eqo%EnmSITBkFk31^NMiS;xmG7eU zA0_&a^YXK^uvV>$>qTX7rz^Vb- z+0cDEjFkue$R0zHK%+L6WR4S93Cc;QguQyRZC1$ahM_Ca<2o&lI24D*sH z6kD`38j8cqUm0{DJ*lv-`9#NwC+*}>XlQH3_LIc~Zel;W#7^xl05~xA_O#(uu489= zgQKW^EZ$Y3(d$}3>bF9Lg{jTGp+C2bKl=5a zIH6=kYPQdbAwxz8ba`Tpg7(}>+8RgT)=fGa)8bD0_g?B1L^0Pc+{8z{wTA&CF5wf# z@;X+-)|&fJ8goKzN0(ziqT4aXQckd_g&;nwuhVzxN532PyMut%Hs`B`OR|X7iea%m zDK3jH0YBp}Ty{5_FtN$`2c+Ry(#t{8cFau8Btvm;u|#(~6?ChynhOtLQprTWtvaIH z#Fg|@^5BJWAtTYKYbx0Vvc@~*AwT28hWHJFecC)sK=F!$Z$s9e5!Yt*dOXlA)gI1TJV(n&$@Km;i=xaUllT zXh(0@;mHsLZEG>9p4gif#PQFT54QK&A#gXzzeMN_1@@__tR+k}URw%1O`_R%T|qv7 znsTF^l_HnSAigbD!;z;!_EfnP-l5Zj(QCSX`TcwYZ7XUL(HD7&q3vgktoRv zI-5n6H%n?~r*?Wt^paa^P)}zN!IzP74#yp9ebSY?)rV@l4D$7o8~qx`EIi!WyI+u8 zA-_#2tnV|JXJ-_PpL2m#8t{9w!rxs~4bnfGKzWa+4YNs;U-N*-FMbc;7CXv_VU}IJ zInRJJUAmK|#N8(_4T15uH||KgGcyheeI=>^Yo{5XX793qHx7Djz)Plrf*pwv9CH{y z@{i8}kCx!vid&uc4Wh@Lvp+RpZa5s7cjAKRp6vxkz25?pAJ!p>JlPg{xWGHp=Qfk_CE8*ohTo4Y&$4t|+p@wtF1c&1(DYcB*X?eBNB zlD95i%9l*;e6lwi@b-`m&iwgW8PBzlKWX!ww7T z%)`jsUC`lP6kD9g-#1pT-;#51kx3bA9YG?Hcy9Is6{K^m&ud%;nrVY1+h;TxCBzEf zI4~j_h5~{%jDl+1#Uh^ej!2S6*6kJ{x}T&%LqaSB9(KhBdz?i&=faR+E1;;s{{8(Z_0#40d&eH zZcYFl0G*Jnm92x4oq>@F;605kx8f;P{*4 zA6*RZYxrB)zmnk$41f6OWZz7p6L`;v+q|X20d&G9EpF*n)1@Gv9_>RKfhph<~85+LH^kvyvs=)#Fsf+9sTsEQyUd+ zxaSTO8yN&q9i(H!)D&FoUKn)dxs@k5xlp{X=Ju4<*SL9!x?P^!aqiV8hL}x=6!HVC z$Sf)~i|Kn#E2FwgklmT$Hl1|BM5n2$1_1l-vlFnzqJQF<_Bpo;63*$8oy9>1=I zkD|x&t*}iH0U^_{PBxD@DZ@WG(U{vce`sG=Bk)#w;TiCFQn?^M9?uYn$k14`IxeMa z#r$wfs+sz+rY&COL#;I=|IS9K+!^4*CB%o?)wC;uAS>P2o=tcPxbWj+D@Tw_OIoUZ zwx>*g+oAR#j=SXH(UE@r%m6#k)?!{3Q{foR=<$2+kq}Nh%w|JcfLLkw7Cze8fh>m_CwDE- zsuL9T8B!NyFApH99 z(|4hg3m;mnYj|DQ4I>Q-t7Bd<%>!h+w_ zNkLGEEMtiCK#N3Dz9Ht}HAafZ*l&YD24@>2VPp2;Yz1pXY#6E<%o$43&n6iqsY+w* zvYH|@`91jmFr2N*SEnfDbAe$+NsXlGQPq#u*Q!jc0{dlf%!QBG-+{AnvU`uV;isyoUji)>_~f$R(ItXNu@p%^CD-~XOIQ%= zkn@nY5`7=!G!}Ef*5KEbD<{na=#e4C^2x_hlB3H>;)@6_$Z|+F^1Ji93;j?;D72YT zFHUMo*OY4&_LB6X6Z_7Tc$K&{SoXan#xnjW(UVLrZa&d9QI9s0LXXHt@^}7o1E@kL zf2WdPZl!9r=tgZ9&sPbNl$?0O5W^J1P{V*tmR<$2mRP^odQ`b0{)5UXvL8gag3~|T zry6QyYB?887U@5VvE@BYe?REGCfskmhA|;C(b}Qh!Qa8%sTop)?4sAdXHvxZ5%E2u zIU@RsdAD(VYkY~d0;v1i@+tn|v(4u(@x<|_@gVWh#nLLav*@$w#n#1gGxQeZ76ujz z7EiOZC6VI=3Yns+t0y=l}(`@4UZN#-G@Af-*HoL+i-DkRd5rT`&e*NY*GeO zc2f43`)kcL;p)IOcQp6vbm`-hPe+tZ$!fwmvdhX(3TP&I^HxfD%HBK;b+cUAnlUt; zSXwPMfqLzFC09W?DLQpJy$Xf1Y+i-Tvdp}8v34z=p^gc_qJFqQYG+|~Sa!QXhC%Lz z5lL}O$>ir;&0MZNcB7y6VHdnxTI7|X=@PLLk%KZyzh+jI^_#|P$Kfq^&B>WdSepBD zDwj1hN=4_gIJ&t9zT~>6K2X8&gsX?AW3FK8QJ+vN(Hv0|YJAdwP`9o`AfI zG4kpUve+`J8OvYUtt%^{S#xPW@#0F+sa(NqmTextMZ6_`B7b6l;t6&OZbiKN<>W%y z9OyNRal|x_2HRoMexm{3A%@nNHbT$M zE_+60DSk`y=y615K61~c$GT_b7tg{=!xI&f6jF>>cz;~M_dVQkINgxM3bh!uiu%|) z!!wJs0$xtuP~J#hv^JNv5-+-YgBQ>jm&cW>`SZQU`6oGWcJNY|eb_v3QE*i#UN}Y= zR0wDA?oN=-W`9DuDLqo{8B9>YXCarb+%PQgIv9_R*tFElrR2x%4JvNlZm!@oh9bg3 zJVlpD0B8ehWiR#oEHQS;Qnh}6}9!x?!3{O%QhoZZw z6{+B@<4xGDQ(7kNgqq_;hq|4j;qmfsdYu~fp9$~?KdvFahN5*9?MRnj>ovOWfX{z# z4t|Q=>!aF>ywJHfd5!^346F{E6i)2X2{n=^m%xzN0+kb}TX(M3yY$;Giu)Fq`-!P& zvS<_-SbJYPEl6e%R!OD9#6!_a*Ug+uFVr*yoIO=R%i*7lk*0LiFl|nX`YnAU9zmK) z7N7xFV^O_oCT5a3q%eZAd%c~qZ8Q9F@M`eMo5hsH)!D4g<#*6?C(EfNV%gWZa_yx? zSy!FTrj47nqiAb@wfMsMu;_543ErAb*y@ZzmO^ElzIn~rZl7SIpu$Eg&6pOVRj(C> z)yax!O_{|~BC+n|jp6WS*rpuWa_rHXhL_sI%E)ctN$Zc>A5X69F3_&aD+f(i>dA&0 zx(gHK>pEpSBbPas_04L>ZK?)Z_WDS`XyfQw#=`1W$qEax*@DBoLqFW;l-<@t&-#<} z#r~!F_BCstX^=>;UMOAybo>^dfjfE2G6CyegXN9V?(w6X!=`pKA3Y!86Y-0faj~yr zKl;0(ooBgb=Gh9_;#j*PWBaPM?E7tB)^CJ7(Mk^FnQ|NN!P$v8E;jq`3C}X%xfk5S zyowJfUNUcgPH2W{X3s6ph5pQA1$s(*t{wPJr+*y#NW+)<`sF&}HaVqc$V_THW(*@O zl_S&D`%qU4dRxqcU6K_-P_EdU#W%=r`Ut)#sMWaK}QtQ*RN@9swIfo9HSGrg0 zNzhB=96@tNg5I$=@SyI@y7j=W<|SdPEUFE{YwuC#iEj0_$!jO-H60Hu9P;s$w$1ZZ zcbVtFYxQLkst8G!Z^VQB#r(zScq@kjR_=2~F<;rs?ESoHQS0?_*1Z0BO^8`^P=gTN zYy3lnTsOpc(KwZ8t>~dxnb>&5L&S^?v0va1A8Q3_| z2%A^~|8nSsO&pCJfObx{4zLXW3N=J*-YQgpHg9W?F)&iJwKlN%haw7eaC8zfH*f%O zu)LMI82qcl!1!jS8qnCu{M}F%dU{xT01G=ifR&XAz{14%*PWdez|6q%)_xBTm>Az< zp}+1NZ1ezD7B&DY6UQG;dH^#E3xI`{1;9+t0{cF~!tuVg_c@vV`v(5cH|5>re~kLe zNLd5xw?N~6-3BWIv$vgQ{EJucErx>CqG6)v0MIbfvj7+vSl+@ZM)tSpNaF3S0U8O| zm|2;;t%Xj&(da!u<6vcb4`Tl6&@jH8FFGLuJ8=`BnYj~yjfnx4PRYr{TJ>E*{a;iT z<~L;mE@ppr48Zcfc`^oWfAts`-uCQIto2qlWWoqw_$$8q=Lq~E{Cx)gj;;PxY4lh3 zEjnXie^d8=t2_F^33O8y`B`vi(`tNh>g37^cVLr3odOPx3_y-eO$P^s=}e*r=lTYq zks_Aj#}@iEBca|U&W~c|Mhu`GPel}D+w5o(9?L83fQ3xeE7XVvBi zEF`#K|1iGNxSjDe+hYdT_dFPrWsWWIu%nHz9U4#2nGtaqB|M>AnA;^eqhtP0} zDL3DU#~S#O()q9KoYFpV<|-m^a<|18fnhz&-}-@F?>wl1FrB&jUP+gFBmI z5$x*ssto3S=9)vz%)R%yX7ac$M`F?+(VA1K|Ljb{XWlV{3f^?)-wc*F9=uN1-DX%CZu?Qn6&GDtyu+yn?gftKgvzOj=(g=V@*$Fw5&QZ|?8; z$K6ZXhbbbi7QHBy^SsRpf3@hAmFX7V)|K9cdxrc@x?!!ED?PcMv}#m5Lrj1Ge?w*lLNMhMILMeoP3$NmceDAhn%{B+zjoc`jMV22+=J0wj*9AWE2 z6k$MI>C|-FD=?x?nx#^j#-G~LwZ*u6NdZ{fTFIFVS_?~`bJ|G&m&c?eQhmOjHYqJEyKpR6KgiK2P3KZk4>yCPQFd#Oe)y=1%K^s7aErW1(J zd9J-n5v2)Vb}6AShttO$JN1ZzOQ5%Pp{U78ijNu zIW9GBq|;EWt_IF3hUl`ONx`)x`;7v!8UB}54+K{sfqEycy!j+C;fiXlcnsHaShq@H zNU4G6Byl|>zGa6+dbSPW%Eo7OB_dHb4Ah10L;-mIbFeS8Rg9IN??2rJL#%>j?FHz; z68eVTLejzPDYr;nQ-z?wF~Duv4Q^pPvtgZN;|XCOe=N)*Ud15Z{X7#@AN(q{FBBV= z0hamprO+9D(a70=rW-)c2<0Ng@{GCkEI#TUE2y|`bq>8;{xdBbH7yvT64vsgF@7dk za~9%n@QUswQk<(Ht6v}VVh8Pl%*A1Zp$LPSIzJEie%=QmL|HH=MYtj9Fg%B%>q5uh zW4g|4Wmki20Y8i34!qQ-Jr+b%wI>WAPt)r0_}H)kl-?qb;jRuVC8E2OVc%UF5Wm!# zvA~SF;(#GxAVo3BI!6n*BVmET4nf;c6%SC@uT9mb;|^F$Bco-Enu+DvCXIxd1LqTI zB<{AI)N97K^v+Q*3gyaBh-AQGm5eR#`ZfN@iws)B{2(st&YKg%Q zs1Za_Fg7}Bo?_pT&yQ0;S3s`zNzVjvN2>PQb)&LF!{P_?W#LzJd`K4I&11PnKiO$V zi-IK#HJccW7F@PSf{GNs(#*3$$`=~%P;i6z?XQ;}EVr)JAiAjxj3OqR5vJNVEe~uC zI9hMp9*3n9R{=`|i-t4`@bq6{&%ks-hB4i)hJGj%)mR0&DeN^>2h-X>$=d_h{#run zzbuL?&UHXcF-T^Sr%>y!HKwz^(#4)FUWn0;_=PY~U)D}{A~L~BpggG4PMXe!a4d1|?877UD z8yeKE_N6AL8nBp>y!?y#pN^rx2PO`5clLE+3o_TGyBg zO=4qWZp569;~SDvHvWSKv}DRRmmloH;$2ABYbE)XUG#)VC+91Ae4L7~n(f$ak z7BH*f>_stb2dv1&FtwZbaF?#rB4fy8iDLQ9y1Zm+ilc#wm}w}5+wtP(M+K;3Z@dTI z0tfF$TB%1(iq_i zHfP~_p?VUgz!YV&#viV=7o_RZq(Hta@`nCR<~+jR!Y-4zzlj&M7%v^+xFKf!bnk;) z9VHq(GK4*Uv$PP~h#fuoE|Z^ zd+PKHEvUD@x9?X@8Kz_0pm|6HG}81@Ev^K*TNJ7|l>Njqln@1^G2iB?Nef zw&wipkutq(Ta<{mIy^2bfuBfryD|-ih@>Al$r;$IiMV)16}BBHE3lgIc^M0;AvdhQ zmTLx26?E-<9-%0Tw<~2&e=Euz_*poSja;8A{>esTMt7T5T8?DJL&vWqOCd1>a)pz} z-y<;2P|cC;(-_fb{_t>eXQhcil4!}~l1lMU&Km{n@QLcmYgx+5bLbd-9O-ojK-f>} z>nRl{bhUN7*|AVorE55I;&b=%N3|aw99b-JkeJ0e0pKaEhvylRoN2u&=%2R z*rqT%&6^6^?_q_NFtm>FL#E2cC!6$26q;$wlmd3=P+gVlMm;^r$Q@Vuq~9 zLn_>5?iBx-QF9y=N>Oo3(R@?*)*=o11dcqX^rD!Ik_t((Z^b4laQTy9PUnvKE2eyJ zc^jRLO%b6L3L3&1=#g=d-AWR=A|E(aQPEIYJtN*XSb3nhMwI5bw49|veq#y(q6u!k zv%Eq8;jAp~S6m0q)z!4rdlhH$Oogvw zU7m-k**`CX^t2=0+H=r=%vTC@r4Fq;FSO=hCOm*)5I5X@HhpS0o>}aAl#SmVN;8)5fxDquuu`q$oFo2zEg6}KGP*iNzXBnV z?l)`;BtOMbnCft95UoRaAs?3q{+|LbsRoo5)Gg4B3p?s+T8yGxk+Md_ZAr)#gW5}V ztRUXt$+ki^B>^_4(5T%4IrwAQ*r8DgFnMwP5;Y!aEUufh_M#O6AMF8LMov!o(~6@i zCu{A5-zAeUT6);7;OIA_{dcO#hW@lBrZF^gaLBi8FgHNh zI^eo&`Kp2BV&v~KJhFR8(fs?s=`3KmP5YT-02y0ma$r_n zdEU*q$C!csHa7MLB^8ODi9+0Y>F1;UMW(%Wu1rTYo!vl~#^LHyV56tPYJRBUA%E{s zyx;375;$n^W}lon@$Du z8l}w76jyqLKS!a96!?ul?ik!gLUp`pUOsOD!Li_*7O2mtMnavb^Y;S%>3cS{a?+wT z!&Oy0CL6}G```G9n%uinZLR%B=f6@m9~*n*_={2|U8&}b0*~-^E2|9>KBDkm7M5a2Z3Ta{8{^_J#-Tj5nq_RemuP6L z0u!CRPyS&XLGDa>v_WFN%Z|o`k;~+llNI`%1}rA?d<>4q&*wzj_2Iy^9AUY<#7b}B z=9wP@Xj|IQ%E4d;7s^$e2<-{_;s=$wn2!u^pR*86*tXddnbX{VyBDJwW zfWV-nhfd)?7BtLq7&0bi?T|FN20b>D{blk32hBxEc#j<=RHUBUAjmbwP-=2&Qm)qQE8E6(R|lU*B?L+!<5~UOuV%hPVvqfWtB{i7 zGUUD@_~@y!kYj6X%1F85_lsAE)gy74*C-*TtfLU-&_l_ZJTz3DB?U${S72MGJM;o_ z`XTq^zEkWrl6Q6Sb~A@?OfG{@>?hhvP{(Y6-1qsNA4aJ^scn*)7|v((%xt+^mTDR! zzpd$z?fk4#N-0l=#&2y18z$FLVjBBxHm@{0lJbll$BW`EpX+~&fk#Ss@Qo%86}=r7 zuCRv%86>@ZPOR^mJyj?nb96iMlyv(s`K)~k>yiX~g@rSs!;|zY-P1)OpeOh}F~bF!h))}}V?1lcBYAEc61zybJ$0RF|`BB~l#T$FxXg=PXys>Qpt!SNIWH+V`4+;Y?ptymJY;}oNs229QV)0oE^1| zny6*RwGIBWkqCd8M!(g|3wJz+hiW!FRx>x#XOF?$aowi~0xh=L69UTY{<(z4BPdD1 zsZG}+%P-Yw9VlRruEZ%fO_hjLm$^J2W=2TOaEyu3BUBK75s#^!971vmfmb_!#!*PY zpkuZA!h4>Uh6jceY=|QC(AQlJ#Yxzs3JbEGJsxyR~YxD5D!c;cOT! zX@0!e-lSJeiE03>rZ2sA-6{$t{`X$#-tBCFzjw0D73A5Hbk?4^-LO|@#@x-3)|pN_ zZ+?I<{+F~s+a=8V`7J20HK%(%kInOktKiK05DhbZj8~SaPQz7c4rlt^WK=6r%*&7g zK?31G|BTLGzT^%@JxWZm*+~VEJozKpSy(1c#hNVy*Aabs@E(M6HfdZt1>dmL$z|SxHVdC%j53Ha?MFxl^WxGSHH3)QiU>K{ z4q)0#=-^bpEMIyj_!#HHA(m0c|1#flrZIgyjS3%6mY?Q8QE^i%!M>+$a*49wdGb$e zAdK-hPWnutu9~VBcAG?6jZ3i0E%Ep(YT%`Bm4N4F^$}Q-EFiLLxhG-;}LwK zdb%gPduT1$9h1RU)%Bv~JIidPYg*0J;`THA78d=PyA4pw#+aH1Jj)rKRt}) z+7daHO3)(w`f2^lfxezg`ix|@TE#MC=AO8gco%N2<{WjlrV`oSU;O@vm;>a!E+%d`(M*lpBI@Q|l16m2WYhrbV3HT0o$@?rbr6$kdf$Vo9Sa@u_a*Be( z-}71)JmxoY(@X(!IvKTl{YE2bH0|q;JC-N-+p!}IvAd&!vyQ2ELaUe`0*hPWZZ^wFBiw5!3dowPkG9T zOHXvE=eUZNHUCSNO-DX#T~wo>w)j3!SJ)*_>A9#?YSt?;Uz@t5uFM{F^Yo=?XuKFQIEVO6FUc@K)a^YL;h8prZM>+u8ZO-* zn4}8EkNGRfM`fX}%x~zSH7N0Ho_-Dj-LLPwdUq#cl0IfeKu1|Qm7T=+{{db5 zyTXn24G8%g=6Wv>`x~(M3!D1?09t7Z$tj9UQ;C~cxtKTsjST+6ME*f`U>W{Z2=@;- z_5TJ}y~8{I1y?aLzV-eKu3}_ke}k+3{lUb<1bDyR0jGEP<*)X?bNthOlm2&ESk^Z< z=>1}Q)A4@&>A!(OY=8Lv;(n*Gu&@Hy-*o)pdmDSF(f_*y?BCLF;MkwBe{{afzOC); z_0L8BPy2oTzv=H*zKy(raqq2nU4M_f-~Y1eZ4~fdGH>la{O>Yvt^e@&f9%tL;P_8c z|B3eh+f47a{5Q%CI2+*S8U||5zFtanffpyI9h#@1R z4($Je7P7rzo&SawGQIWw4J~ANgA)Hj3pv>T3oe9ZVSGab-^t={z}~+Bhs@0X4LE$a z^xuKQzbiNY{usVZ{*UE<1BZX=IRE~`f;R)$82Xup5Ks^i5F>TbFS-8Ij3Iz<|1i9gBywYbFFib1z{n9LXIR+q ze#kJjyYsG4I>_U#>!C}?L=E@vq`jAny`xjd64}`kPq}nX=bugW&Yi8J;sJ{&0e!MI zDQswinL}03RpyV&`R)?S_(IUODXVjbQk%;q&EWKQwqLJ= z5;ap&O-Y|7RVbyjvWP;ryyZ=6N33Oc!wGBn|Rn)oQr4}P%D1xK_&HOq-j zknpq-r_qdBv67*BK{FM8xnon|rA)iV^}IaS&Z37YDn11l|Owv=xxk)j`z zLE}Cco2?lOowz}1LREhB6{xu&F=y!b>`U?CSKFdZ2jZ3(rv`%uIGrdWH)FA3YkktO zZmQyrR|(#gj-2OOy_XC6l0XA*@2xC?;&NWk(gl+_cY#m9 zv+%ms~dI z@2%;pjQg3`D(+HH_pM~7;j7Pf^&ueR&e6A~7{B{cL3{$Izt~OcE1x+xvbjMCCid%~ z5kT9apLJ>}V6HCW;2-dSs$ie`p~QfsHeJXe!jDtKyKIq7PHiNo5_flZ4w&20Gcw}` zJ@%-5HR-bMk`avgMq><1)Jew<@!cHRIY(7FTv7r3tIe~b*^flNu)7>d{oz$GVmgPF zdlPfx=uIz&tu`TIDpZuv{L>x^ol2LOBD}McsYzUA#7l?Bv%V)p($rg#R_d}Uhv29Q zPW<%{XiON~P{aFz6>d!%U`eeCF67a0km_rRQ0}thNzzLNAM0_HH7-jw?azK4LaLnj zz~q(eX~O+K!E+?DESPW#!{!G2Du4h11OOj_xh-nGu|v z_a@12)gI8)T3JMm!ZEA$?}8V5JuPj&{%{)4Te0>#d@(m2rEz{T6HP7QTFS9z@ykDa z1e-3#WL~zbQ1QCcl)sTXRS}{4>APx~^4uQh$ue;Yyw*kd^PJ9^-u9gAoFB=Mcb20P zFQ_oRrPQN#6twDT7<6XPQ}S|VTLp=ny(8{P=jS}d9@<`v2^baA4MaTy)6V+gWVs1< z3O^;`h~{H3WT1+q1W+`A;Pmqj`4Z7%*UJOpoPG5x;8LE8`q_pV zw>1bYlxP!Og1OdoKLaaXP21iy1ciu2el(tJ~JqREJFy#{DTMfM0d~`C=+fG16=1GdGhgQc~0lZ17H(@t1^B!!^iCeH8dkAGy1!UTm_P%LE z14uvQDMkd@v=LUpgNUR`ra#SWUMvm_9i(gSyA@Dg1SF!w1*^^`aTIdhRR^Bw;?1c} z?7#`YCk)D8?08bVFpYj_PUfpmWc=_6Vwz4jVY1(RB4|g`9KPCDQRIjzjcNJ*jEo^Y zY;_vWO*r^?Nw$%weZHHa}yX%i918;2ZV8p1lFI+B#&-vN8<(;i>n3II! z1Z7rZGMX6+udJb^1v_B|P5LJ5W_<{6pWwLP6k-^-$aR%&s0b zl((vi8S6okp7R>4dJ^J@II%+KMv(obJJ%TZNHhSzOlH>u(hs%W;s-V z0%vv~0gho0eE8son;Bnm!O@6pv7@sVsu|)o*~1X@=}N_-AgDDLUkXKpAigj+uu&-L zYk|fD*0tTXpas|)mvT?3#f*PtRFj;$8Q;BJxi3gAC_mwf7Jv5>9#d>@0 zzkkOUI&mapkG_it#t(Aq2HORXAJ(o`;xvp}O&@dEd{HAu5g2BJFZ`C-GcEOraURc9 zJZP)mm5#(JIQ;uD0fBMVqlvhq0L^~kt8H*K;Z$s22D|!HEcMUWzu8glT__)uvUVzv z?@YSOKzD5yyUy+^uh#B)8@R_w$3I;Bvh^25*3Tk@?`-Ut7P{~pt``Qvn|@SQTUSm^ z_C^%;z?nOMG#FKj3UK4|$ra^A0fpq(v6<-oV|!?RO83TCZ1j_7^y?NB6L*T)*QicuD-02 zdGyou-bxe@+_;)!Hwof;kNEsL_w2u=22w7rpw&~B+x`Ra?~PO+Os4VYrpJ`MG%(fb zCZDMM>T`)ukjwm?5YT8Ws0S&JeLstNlM-eClozoNYjtxC+e33VZ;H2XmuD#x8#n>! z#lt0u=2o>KRb`DXwQhpiIDK!Q@*C?suxh^z*h6PaJIRWveOKrHl3qhX z(6c^)zYvFIdi>Y4qp6FK=IiG|n_J`ax{TElvCkmvAIl3)0U|E0dF12ZA00%SDlJzk z^bB1KT?-Tzm4DH0egs15L|a4S_ZM#NdwP-GduZHH*I0GWPAa$-7r~*&yK~1qpP<(Z zH%?)^omzov4r2-ZvIx9J_8HSlZO-m>e{C!obq&A`Fxx&Rb_OT#G7f%%-!zt_=~NU9 zjwvT4MO=+>u5oE<;HDJ$^$YxOY`iCt-C+Yx296e{7Th|iE*xUj2n#yV)+2MobT;Bv zhx-EZk?yiDiS7XYWEDA=jiiy+oKbKkFPp9{jR_8SmH~N9ANS7#U(qAR{oh3(zR-fw zZdJVj5e$L*E@NAwuRcU1zHgsi1tSXm+yF`3us>8Y`ZZq3LEw8>jbG|U&(^4jtKi5ccm-r%A zZey4GuMv#7Uc}?PfbtLKiK9y)zu-PLu;kOJY@pcZI%6gw8nF)yr@$J9dLusKA3?{b z-429N@3{VwG{$XG~#$aLkw{)3l z5n&noEvMNIY$IH-^*`g=zu6#~U=1qr5@u}?I*5cu4C-TzwR~JDe(}$o5{)buI==0BSX1Ku4vj6+7;5i6){nKP zZkB#86+c2Ch_Hyah|!$4u>M?>)JSC=@C*A8?ZNs%>)j3-n81FB3f)MncOWN39}IBi zqRbe~*{Q zjey^tdd(`00{lJ`#VaJk((yUXK11^yS^+`y45+9O4A0If8&=?@6Au>X%fUk$!RUO+ zWkejqO0DPf^axB$dHxgaZT$668@NchIHB81|6ynF@yyqEu_()VKaJ2y<$DRiK^>f+ z)JFo17;iHUfZ)ji2L%tC78huU&-AD8cU4MzpX!*sinCGR}$(^Okb-_t_mdUh@N1MXyI)#2L=<9kV zzNXx3X^c)EMsBHY!AG0G_x{Kg=d5E?Bh}SJj^iYS?H_&0J{UTerGaIOYJ0S|7Xgct z{(d6D2%rB98q6Ma-a(mtWeDoz38KJbx2@O#3Q|dvke^+p@Auf3ddc}w%QngZFBJWU z2h2xZBKJQ8okUxHRo(H+%5u%Ek%;E@?z-3c>qtcxfSBct>d{$fu|+5&ENe%Y=`HSX z*VA?}5oXSvVCj6gNhh7 zz3^D?=8cf{cY*O-PIsQ4LV~$yt@>5TP9c%Tm`J3W#-^+wh(C2t<+H)hOXaBhzM;Dr zOGAp3(egVni-$e(7QNsF!X*Yg7u3{+j_N z?>X%njDvSEBn+xveIN7=zGZ9TIhoN-^eV&%CNIP}3PT;U=n^^$aE7WMtDAnSfy-mH z{^E^Y?wD&k(lRjO)$`cY^771(?1~4+O%8n%pBSPg6j%Bigy?G2UOW2& z*tw=(XWS*3%NT{crV>}1!Pi4IPD3<~dUrB@{ygoC&bKQ*hHz8LO8U@!*WB?3kATLt zQTw4Ro}rk|lMqjY=CKh)^o)pg@6HU;1B-dUBntS#^F)E1ZV~&4SV%H|{q#ogI5)*i z#9sOA+<3QE&BqK!1wU{LEWvC$jW-L-CjTO-8p=U!fWdVz=N60#0kKNrVSZIznU|!U zgh8C?#Wlw152CKFs}|Er%FY&;4M!w!3W==BgM&MisbR+g03LC#;LdKt#J#7HUSN)U zpRZ}2OZYd_x}L`uNjKO>TxaXL#J#zL3}2p#DLl2QZ(H5A4pxX~31_Wkll{BeU+QXd z_?m9G@2K=$PjBmEyO3Pih<7CbO*WeI?PA@dBGD3GXwU{0>7&D>BE&K0SN_Cby)-Ldua)sYlrQ&>I>W4qAfUD9@Q@ zFcN_}@YmEqJ@dJ?U1jcOtrUNryJdcM$WcR|vI&oyRINbXwoGve6CaNhHCwc5 zuR971*!clFD&}|i0f{YJLmT=9yWWn*J*?74#1abPdOZSCKOXT<_-QGtDT40_fB`r>O@Tu!JQ%pb5cE_XkLd#I@Hh<9LAt?w7t zf&WvOVzZP&DC8*hFzVJiUiUA-Ml8tV61=k)>TJq#3o+c|JJ=~OA+T+Nw6%VHWTKob zSgZw?9AA?Z_8Q>;zq4DYvQOtDC5i-I;I&SX1EwkA0W}A3jTe!WS`~UBW!7y z++*mKaO~}!F$Yis+rg2HDw?CY-KX8y1QL~ymKlhCLPA_nqLOJL1qqnMK#X}b_kLlK zhiTFF5d{H|1~~K62D8WbK~c6;HmMTb54h=PlHU zoL(Y|8;bE&)XZUlYum#yhHkGLo~bL>+_+BZhuvkg+2h%)S@Xl`wDbkHgbd4;=GmK6C>nd^Jt7g=D|&5L{#xztcg{4S0Zp$GKv@* zKD8AL>=rMxRUmbI7zg*H3u=ehE>eXVzCtP-%SP{O=xem4X9SQTgdXyRKs!?Xv9ZoC z_A~}aSY8Q}R0Yu%#n&4j!2aR2yr#B(i0##6r^#*eR6teYZW5-gsvU?k;n;IUXe6^# z1Czyc09aK@UaVz(!Lt6c?R=LU8OXHwM39wBx0WmF0vUr#D|9POO%L=b<-snFN;*IxxOxQ8lOk=b6 z))Zn+;Zm}A0Ej{{7o>yMhj;$kG0rB{P-<{^A$@?;5dF3aF>b`c2CxNm1kiR^b$`?Z z3a?7qEG56BVQV9Q3-z0QCV2b1U{2DA+GzRokTO)KcVV9?szF}43YtuH-e}zG+mUD7 zmE?luo9bUo#S=J59iv0K6ua(;=ug5QU#He@ZO!)i8@~tDfMJV3Xhx2Y!`2t1M+b*P z>E~)81YhcUJ9F4iR%SCeoHxS&&ksfnpdI5D&;Syq7@|4nxqy`p2Jl_`vYb@eKnGF- zoS*QHc7!hap^h9veZ~Chvs*26-YYjHY z`A1rjqz~~RqM+b9w|)#48``SFEL>qoRX5Hl&3Pp}7e&72l$5uquq4E(NW)v&`p%~wV;vQREs4 z=^V&e0m(qoiJ(%11&-L=(*S9!5$ohvNDb5mx5N8_frZ^j0+b|-<}jnDDcz-wMIm+q zHGH;dc*s?f)Ct#Q|K5-|AgGtC{1}`X+?}S2v zjokXJfW_)BkV(OK#jb?nLwK-pu^_w4M)!pj&Bc3oOvjL1q;k3(A|-s<5<^$jUodLv zXi1>Eu*Eg8p-&QiB8KNiEWppxHo#39Qt>D8;i-3eS5LvB>{P?}^|p&^-1|ydJW-f< zU&zmr)h5fs@Xh6mF4<#kHy%pH1fVOzdOVz6c3RFxqK}7qAH)^?O$P@Y7qR(vDuQU@ zqCPL3iFjO**MBWgr?B17lFl>g>a4YD4^XO!fCXKF z;|oQ>KujnfV)B%)VMmK`R|kp>`_;7}WimSQ9`HVoVqjN9h>TmAxrOn(VqMCS%O%r( zdC8%_)s9OoRAZT^w9iWg_k+_pp8m9TP)LWE8{hKMET>qf!h;ty2+?Q|kXvRdE9jcB zK=sTP9i#0U=pvZSomGkB#XLd%RTuh#=M(Bb$MQ34me=Om)zY8W0gX%l{ymsCQ`2)b zY$+vW{AXvE^BTcnnXv3@gFAz`%|%$T>90qiD}vu9oQ6&=4XtUu%2 zqg>97TV}Ui|H_npiO^7e9R(kwoG`-kKNo-|d=f>plYqttD8|7=K4Hx(Cjlg$`_fGbiX=RbXO8s=gJk!)jCxAN&s%c$1 zmgtZE@DS?ug{JlH3*hVT%;fUGo5-6U%nb89T64+%&=Vs$J%*zI!tSc`MGBIyzhrE` zCXk7@m}==Y|6J~JJYt{7tgtNVu1xFdAG_g+qYJ6~8{)2ma(#X^zb2_gDp_OpwyF+_7D(@e|}^(kL@7&AE`=m*I+t$6C- zKb^RQ$~_0mF&ua*2*Mqq1**mbvkpWCl?*_(TCum)*6Hp=7C+de3kL)Ri$q4%f!-}P zbjJkTtlp7|Jhw0ZWbO~7WnrCEc7BFY&DScpVsoa06`xQrv1Dyn;;NU?k0Q0uFxCi| z%_SsjauVU_t#9s+3NA%AqJ<*ReuYRJ`AzXSET`qTGC(L4FT=Q#XjNIjnqOSUv_ zERg$XB)T;X!CT99s1$;h?1aI7D*ur^8&?;AD5>^YZPfr42)cepTwluH8|0pXy6!GH z`|BQnqR;qh)DrI)smb<2%ra0Vn4ILaDPOhA0!8dpir#+Swl+hFC8Q$L%di3-r~PFY zz@imLug~{IdcCIdXLrlrx3D1`rV&cbotFu*bfTY@o`HQUHZKF@4q-EsRA*JC7h;Y{ zgy{D2#Du;J(<|{vvrOrbxzBe|Gfc(iIDl7j3!AH|cIY6GSonk8cPfDGoK{L78jX@B z8)^1u%;}wnls?5=AK)Ye*d5H>zs(!Voo`qpSmwYyq->C)ANjG@z9cq3n!ARkCW78% zdbqxCl<|+FlJpYEK(V#I*;OE+<3$EpaZ3CHrm#y(mA|&W)4|PhfC?!Z{-rKn=no#W zkSvEGj_PB?Hh|bekCDEDM%xBFO)pa;`luLR3{HNsfAMpicDvEUpPQ^FCP~*4Y^5H9 zkjbNnuTPC@HEV-@uTOPNK2@SWq;tPu+KklGK6LwU4w*`G)%csbOhpnWWEga4?u_Mz zong7!wW?FcPGz1~RP{$VMmc&Evro8ie!`9fA^Zf{j&XZ#rV_FUS(6=^_MYyr z+5j1s92?-KFn{+lB1l@8MD{Kou2ReBE9n0{GPk#dG%J44-4{@~l?iaaFmc?K56P4B z%McnB%ZO6&*nCMKCBR~p5v@{N76 z^1V6RG|+aDS6v3NUDXZyKcw{M8`HF=TnI13r+631Cu1>y=c{!e7B1FFNqt?f=AY}8 zLP}E7T~F{vSomkAtX`D(-%xH;r**6plFyo${pp^~;?pg?SvcR6Dwyqh6w(if9n6<( zKKHU`^})Ud7k9n~P*jxup<^iQU3U+c#Yyl_gh4Hz97CTaaEHic&eZ~dylCCu=2Ubz z$VaSuM2&aEIZC!(X?W3W90(GrGG<^@W#uGUu@T{fbn8T>0xbSaGqt=HCm}E_{`48y z*t*|8&`Yb^vd878{(E{I;W4@XauQL2)8$Hl8vvW2iMDHGh)Br{5rovx?@FN-NW!&t0!G z+jHpf7VFLP`>E08b_;uWxk$RR+UExR2+t(sz>V{v57F?1GswsHRP}TB~3H&F9{n#tP3%^0H zGoPTE%1;O%Ew3%#nMIG>v(M%s`!|3RS7r>0Op3@TV zYj2=t(1%QwvL9+25-WVR2sQfIPuxe++(#dpI97>f9LEJc7so_IoB_HN&-9WXF43r) z~N** zNC)|Y>%dIZefRyMMlN8WY16cZ-gUR@(ZiXi2h_gv%lUJ>@9F&Y|!&g zubifAzE6aZrxE81#QIi0Po{EBueL^=a1v*&V|EhCp`IbxPnhZf8PSasGT}rD=cv+> zZ`4xs$a*dh`1LG{5yPFDojgaqm^|zf_>RQII4D2B+8Lvz-HVK^Bf}zPOHWPzvO&Z= zO{o8{?DzSBR-pvdXyJ@FqKWPQl^{hEvoe$u-7738*3CGXGw5n_4?YehV+|O z#kN1QH@|i~=abVqT%CrcNIl{w7ij9OQEskbid@pG_-f(B=uQ_;gpZBOXF{%J1bGe@(6T zRt#-6xKOaT!ffZ~?Ac=SJdiMy@JtfyiZ+#!BRJ#rm?F&oi`5T*^viRWdC$jkbw(>O z4n)m3E9^>>bTq;<>{D#X!f?Dvq{H#hf zn;M@>-zN?F2>6^!$rhp~mbMZN0(a!x2I#l**p5%IE`%6(ZKid<5#^Ky8I*3K42nAt z;pn4Jbr|{KNx6>4yiS*FFPB!0PdJ)1f+RCkmCr>mR*@MY48l=agG~Fl$P6w06FU2F zg(r#@tY-LLM4`##=D@JvJ*cswe>H3iHBDoSnGmXg{Q#SQSz~JAm>Rk{d|6M%@wGC4 zER$?a7+YoVeuMX%&X;;ZM3R`3QKp4wxcAsiCqNmy?1M>*AkK`<+Cb`to;cKLGtEb# zrw80wg^pL$GS@31r=Htw_Sy9d>wzD-ZSo|W^ibmU6k`=ilm?{2z%Wz+@pmUU2E+BF zTwc3GnnoI0Y2RRrURcg^!S(${-5y?z`sfHuxfa+}y0*I7??OI7DHSGN)csCk<+T=Tt%=_Nh8DIBpcRZ@V^vy}fSVo4l>GecqLB zgWm_bDONiN7+o@>^iwRD=zMS9x$higG!!`Yz26HKTl!RaD?2_Ed{(E=vXVZiyU4_HFffHANCHXG5npY6X(1 zsmc_M^aau1-!fxlMzDS2#^QsUzR-O~$Jt?bb|A=vJG@Wig1bf<7fY0QeA34s^isBy zn{iEKz%WN|Va2FHb!9oc31edXeH`yayjVy&3 zFQy@9)XpEW&il?$CXRFrCDZNUHEDFp12y`N0B_0r$KyHpu8iX8Jcla zDN--zx$nQX_JS*BZ%Jjf>bB&&uuJ|daok)~Wc#qN;T8#p_~4}(4lQf-A}6TjmMXS= zLF=3TQ7b=9FuBZh+L){`^Ly6Tt64{8=Z8ZYy@7lmmvs1h0fPWa!AX*lyONyh6VPGJ z-mL)+%{CD^SLingWC{+^ZZRC)^vf`1LgYjHe7oWlz9Jdc&UTS+A7Fi6NgOE;ocQU_ zJCLADBQ=sVT}>f}_o^~TEr=yKrQ%=+JzF?Hui64AfV#QC4~Zx48;entz1SE!$zxSFTR^jb;2j&03Wh4M}% z#v@!Xxfy(<+As&z7aq(J@UT1_zzpIzq--J2;aq)X6k?~qW4;03fXDH0a?)Ap1lUbA zt?In%Eb5w=rZZ;4n8m02Xd8+?hki0 zx(Q#F*m9a^9}ef|i6Lju3U?eMA|`C?l`NN-4WzHlr+(&25n~xnK>8a>2*rphx>`%~ zPi87SPq?Q+|E7W$zyz%LJh{Y1>-)2?!olVldj9DQPwcN#ey{SxH-i*ji>=Y}<+FgN z8+=&OK2sW8g-ka+MWAQbp$-Mf(&gC7C{svucw7`{vYv4Lo|iL0*!%SP&+ZwS{I7vU ziC#tEAIOVkrobLD)9KKc88TbC2DT2(vd@8S;oJ%zMk+Z=SPjD=T1>SRC|w4Q*Qwj9 zd+JogH3u_;EMr3<-fk|lrQAT!7nNIw#okUI@qMwl;a%eQT>+P-3C=0SpScEXfp{9} zl^T)Yi4P^*BY(RDFYvEeD=i-cX66ri9Of#h3n0+#{3;R1?G3IqpTw2`OVvFsix`fZ(WC^1A`sKi##;%~}8gj^MV7hSq{rNn0 z)V{{nTzZ~PX;>qq;ST#m&Vgn{Qo_?Ua=wb+uGWB{Vj0QvdWMcJ^ z(!_DFNZ+u>P6UW`LLaajV`Td~zcd1o1#mhb+Jy%&5n6KOH;)+oG>>)<=cWF$z(&1I_g9_1 zI83+Y^N?=@p^UYh@H*5^DnbJ!-~4pju0q{dPADCXb*np&iR3Pi)Jw~tvbE20b!fq=WyGg6tXi*1&oGs?pch7q(HrVj3aK3&C-+>L^@R^<)HG-oGSj;I1eKbE z5vtpDe||iWMurx|pyo zb}bL1FWuLSs^%XE>=ri1gAP3tA5hSDSYvkaej;Cd zRBxQWZzX<4KW-&ox;pyek5@5=S?BdizI%1QdHM5blK#3Z>`GXGT>>-J%@PZ)dyM3% z#t07C3lyex%))c?G}kXWA7cW|Q(V{6WJd7{ zby3hOokG}ECYOgXII#jgXZAP&*wgN~SBj!h__jlY|E42+$Bx|Q5h1`)Ywi{_+}|GB zTo7gTzFhp~++zdFvp&HF769{n?nJ}kH8QyqkuXKF;cKbqdVF%Ic!}@AO|${LgNG76 zaKk*)i&l@V^8nyh-Zk ztaNvibazWwzP>0ziVy{Yqf)QscW8s!`ByAo z4-|iC6s6794nvat?@*4F&aT%}EMG&EE;58}8U#nh-ZRDCRkd?Ty1Q+{3x@RVS6t+i zfZYF!KmQ-U??3KmR#sMS?*DQI{~yNSkJfH)cq3YAx;F%si=&yl>D$g_wD!iP%IajA z<}R+*jt*~REM`_tW)3oaS}}KPyEjr6uOu56mp&bts=Kj^n>X1TQubeTT2D7R6c+Kf zP0rrvQe?E2F2+t)9@gGuY|I>N%P2+{Uxyk=yTweA!2Gzf7Z{u>j)ja&$^}WHU z{;`e!hsOC&J;J|fZ*%%K!~dnZxfs8}t6cuI7oNJcxB1)sAY)N;bbQ01{Zo^m_Quk3 zuyA~1qy5iwi8pAN0ILOroy~%k*8*Z;&TeMLYQn~C%3)z@%=`9=pPiSTT?pm>AM!t) Z{;qCs%kW`__. For details on how to develop for an SDSS product refer to the `coding style guide `__. All contributions to ``sdssdb`` need to be done as pull requests against the master branch. -Contributing a new database ---------------------------- +Contributing a new database or schema +------------------------------------- In addition to improvements to the code, you can contribute database connections and model classes for your databases. To do so, first remember the directory structure of ``sdssdb`` @@ -59,7 +59,7 @@ For an example of how to implement a database with Peewee you can look at the `s The first two lines simply import the base classes for the database connection and base model class. We then subclass `~sdssdb.connection.PeeweeDatabaseConnection` to create the connection for ``awesomedb``, overriding the `~sdssdb.connection.PeeweeDatabaseConnection.dbname` attribute. We then instantiate the database connection as ``database``. Note the ``autoconnect=True`` parameter which tells the database connection to try to use the best available profile to connect when the class gets instantiated. Finally, we subclass `~sdssdb.peewee.BaseModel` and we bind the database connection to it. -Next we need to creates the model classes themselves. At its simplest, a model class represents a table in a given schema and contains a list of the columns in the table, each one as a class attribute. Model classes must subclass from a base class (``AwesomedbModel`` in our example) that has been linked to the database connection. Differently from SQLAlchemy, Peewee requires that all the columns be explicitly described, as opposed to autoloaded. To help with this task you can use the `pwiz `__ model generator. For example, to create a file with the list of model classes for ``stupendous`` you would run, from a terminal :: +Next we need to creates the model classes themselves. At its simplest, a model class represents a table in a given schema and contains a list of the columns in the table, each one as a class attribute. Model classes must subclass from a base class (``AwesomedbModel`` in our example) that has been linked to the database connection. The default mode in Peewee is to explicitely define all columns, as opposed to autoloaded. To help with this task you can use the `pwiz `__ model generator. For example, to create a file with the list of model classes for ``stupendous`` you would run, from a terminal :: python -m pwiz -e postgresql -s stupendous awesomedb > stupendous.py @@ -72,6 +72,30 @@ Once the file has been generated you will need to do some changes. On the top of The first line conveniently allows for access to the database connection from the schema submodule. The second one renames ``AwesomedbModel`` to ``BaseModel`` so that all the model classes in the file inherit from it. You'll probably need to make some other changes to the file, especially to the foreign keys to make sure they match your naming requirements. +Using reflection with Peewee +'''''''''''''''''''''''''''' + +Peewee provides a :ref:`reflection ` utility (internally used by ``pwiz``). Base on this tool we developed a `reflection metaclass <.ReflectMeta>` that can be used to expedite the creating of models by only requiring to define foreign keys. Note that this is not an official component of Peewee and it comes with certain caveats. Before using the reflection metaclass, make sure to read the `API documentation <.ReflectMeta>`. + +To define a base class with reflection with do :: + + import peewee + from sdssdb.peewee import ReflectMeta + + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + class Meta: + primary_key = False # To make sure Peewee doesn't add its own PK. + use_reflection = False # We'll enable reflection manually for certain models. + database = database + + class AwesomedbModel(ReflectBaseModel): + class Meta: + use_reflection = True + schema = 'stupendous' + table_name = 'stupendous_table' + +When the connection is created this model will be reflected and autocompleted with all the columns that exist in the table. The reflection does not include `foreign keys `, which must be created manually (along with their referenced columns). You can check the `catalogdb `__ models for an implementation of this type. + SQLAlchemy ^^^^^^^^^^ diff --git a/docs/sphinx/intro.rst b/docs/sphinx/intro.rst index 0d261207..6adad910 100644 --- a/docs/sphinx/intro.rst +++ b/docs/sphinx/intro.rst @@ -21,6 +21,8 @@ This will returns the first 10 results from Gaia DR2 with g magnitude in the ran >>> session = database.Session() >>> targets = session.query(GaiaDR2Source).filter((GaiaDR2Source.phot_g_mean_mag > 15) & (GaiaDR2Source.phot_g_mean_mag < 16)).limit(10).all() +.. warning:: Note that the implementation of ``catalogdb`` in SQLALchemy is very limited and should not be used in general. + Available databases ------------------- @@ -111,15 +113,15 @@ Note that the level of readiness is not necessarily identical in both Peewee and - - + + - + @@ -185,8 +187,7 @@ Now imagine the case in which you are running ``sdssdb`` from your local compute >>> database -There are two database connections, ``SQLADatabaseConnection`` and ``PeeWeeDatabaseConnection``, one for each mapping library. Each database connection has two keyword arguments: a user/machine profile, a database name. The connection will automatically attempt to connect to the specified database with the profile unless the ``autoconnect`` keyword is set to `False`. -:: +There are two database connections, ``SQLADatabaseConnection`` and ``PeeWeeDatabaseConnection``, one for each mapping library. Each database connection has two keyword arguments: a user/machine profile, a database name. The connection will automatically attempt to connect to the specified database with the profile unless the ``autoconnect`` keyword is set to `False`. :: # load a database connection with the Utah manga machine profile and connect to the manga database. To create a Peewee conenction replace with PeeweeDatabaseConnection. from sdssdb.connection import SQLADatabaseConnection diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index b11e03ce..edbb0376 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -69,13 +69,13 @@ class that will affect to all subclasses, except if it's overridden there. preferable to define the foreign keys explicitely. In that case it's possible to disable the reflection of foreign keys by doing :: - class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): - class Meta: - primary_key = False - use_reflection = False - reflection_options = {'skip_foreign_keys': True} - database = database + class Meta: + primary_key = False + use_reflection = False + reflection_options = {'skip_foreign_keys': True} + database = database Foreign keys explicitely defined need to reference existing fields, so the referenced columns need to be added manually. In practice, this @@ -85,21 +85,21 @@ class Meta: Caveats: - Many-to-many relationships need to be defined explicitely since - it's not possible to set the through model based on the reflected - information. + it's not possible to set the through model based on the reflected + information. - When the primary key of a model is also a foreign key and - ``reflection_options = {'skip_foreign_keys': True}``, both the primary key - and the foreign key need to be defined explicitely. Otherwise neither - will be added. + ``reflection_options = {'skip_foreign_keys': True}``, both the primary + key and the foreign key need to be defined explicitely. Otherwise neither + will be added. - Reflection will fail if a table is locked with ``AccessExclusiveLock``. - In that case reflection will be skipped and a warning issued. Note that if - the table is locked with an exclusive lock you won't be able to access the - data in any case. + In that case reflection will be skipped and a warning issued. Note that + if the table is locked with an exclusive lock you won't be able to access + the data in any case. - In this version, indexes discovered by reflection are not propagated to - the model class. This should not have any impact in performance. + the model class. This should not have any impact in performance. """ From 67bfe5025a28430b9fbfff8cccf0c99b82f48206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Wed, 8 Apr 2020 12:36:08 -0700 Subject: [PATCH 197/197] Fix import error if astropy not installed --- python/sdssdb/utils/ingest.py | 3 ++- readthedocs.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sdssdb/utils/ingest.py b/python/sdssdb/utils/ingest.py index 822054a1..903f2d4d 100644 --- a/python/sdssdb/utils/ingest.py +++ b/python/sdssdb/utils/ingest.py @@ -16,7 +16,6 @@ import re import warnings -import astropy.table import numpy import peewee from playhouse.postgres_ext import ArrayField @@ -398,6 +397,8 @@ def file_to_db(input_, connection, table_name, schema=None, lowercase=False, """ + import astropy.table + # If we drop we need to re-create but there is no need to truncate. if drop: create = True diff --git a/readthedocs.yml b/readthedocs.yml index e331a5b3..fa3c76b4 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -9,5 +9,6 @@ python: - method: pip path: . extra_requirements: + - all - dev - docs
' - f'{table_name}
sdss5db catalogdb
targetdb
archive