diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index a548404e..00000000 --- a/.coveragerc +++ /dev/null @@ -1,8 +0,0 @@ -[run] -branch = true - -[report] -omit = - python/sdssdb/etc/* - python/sdssdb/misc/* - python/sdssdb/tests/* diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15fbcef3..55fd45dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,12 @@ This document records the main changes to the ``sdssdb`` code. * Adds ability to test against real or fake databases * Write tests either for `peewee` or `sqlalchemy` databases * :feature:`-` New framework for writing tests against databases +* Many changes to the ``catalogdb`` schema files and PeeWee implementation to match the contents to SDSS-V v0 target selection. +* A new `.ReflectMeta` metaclass that provides reflection for PeeWee models (with some caveats). +* Reimplementation of most catalogdb PeeWee model classes for catalogdb using reflection. +* Changes to the schema display tools. +* New tools for table `ingestion <.ingest>`. +* New tools for database `maintenance/internals <.internals>`. * :release:`0.3.2 <2020-03-10>` * Change ``operations-test`` profile to ``operations`` using the new machine hostname. diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf new file mode 100644 index 00000000..312cc1ea Binary files /dev/null and b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb.pdf differ diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf new file mode 100644 index 00000000..94402c2a Binary files /dev/null and b/docs/sphinx/_static/schema_graphs/sdss5db.catalogdb_lite.pdf differ diff --git a/docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf b/docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf new file mode 100644 index 00000000..03034a81 Binary files /dev/null and b/docs/sphinx/_static/schema_graphs/sdss5db.targetdb.pdf differ diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index dd781224..3910c0d3 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -7,13 +7,16 @@ Database connections .. automodule:: sdssdb.connection :members: DatabaseConnection, PeeweeDatabaseConnection, SQLADatabaseConnection - :undoc-members: :show-inheritance: Peewee ------ +.. autoclass:: sdssdb.peewee.ReflectMeta + :members: + :show-inheritance: + .. autoclass:: sdssdb.peewee.BaseModel :members: :show-inheritance: @@ -32,7 +35,11 @@ SQLAlchemy Utils ----- -.. automodule:: sdssdb.utils.database +.. automodule:: sdssdb.utils.ingest + :members: + :show-inheritance: + +.. automodule:: sdssdb.utils.internals :members: :show-inheritance: diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index b8ecf614..c942b926 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -49,7 +49,7 @@ # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.autosummary', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.mathjax', - 'sphinx.ext.intersphinx', 'releases'] + 'sphinx.ext.intersphinx', 'sdsstools.releases', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -133,7 +133,8 @@ napoleon_use_rtype = False napoleon_use_ivar = True -rst_epilog = """ +rst_epilog = f""" +.. |sdssdb_version| replace:: {__version__} .. |numpy_array| replace:: Numpy array """ @@ -292,11 +293,9 @@ def generate_schema_graphs(): UserWarning) return - schemas = ['sdss5db.targetdb', 'sdss5db.catalogdb', 'operationsdb.platedb', - 'operationsdb.mangadb', 'operationsdb.apogeeqldb'] + schemas = ['operationsdb.platedb', 'operationsdb.mangadb', 'operationsdb.apogeeqldb'] - output_dir = os.path.join(os.path.dirname(__file__), - '_static/schema_graphs/auto') + output_dir = os.path.join(os.path.dirname(__file__), '_static/schema_graphs/auto') if os.path.exists(output_dir): shutil.rmtree(output_dir) diff --git a/docs/sphinx/contributing.rst b/docs/sphinx/contributing.rst index 1be8ad42..ae567d02 100644 --- a/docs/sphinx/contributing.rst +++ b/docs/sphinx/contributing.rst @@ -7,8 +7,8 @@ Contributing to sdssdb Contributions to ``sdssdb`` are most welcome. Product development happens on its `GitHub repository `__. For details on how to develop for an SDSS product refer to the `coding style guide `__. All contributions to ``sdssdb`` need to be done as pull requests against the master branch. -Contributing a new database ---------------------------- +Contributing a new database or schema +------------------------------------- In addition to improvements to the code, you can contribute database connections and model classes for your databases. To do so, first remember the directory structure of ``sdssdb`` @@ -63,7 +63,7 @@ For an example of how to implement a database with Peewee you can look at the `s The first two lines simply import the base classes for the database connection and base model class. We then subclass `~sdssdb.connection.PeeweeDatabaseConnection` to create the connection for ``awesomedb``, overriding the `~sdssdb.connection.PeeweeDatabaseConnection.dbname` attribute. We then instantiate the database connection as ``database``. Note the ``autoconnect=True`` parameter which tells the database connection to try to use the best available profile to connect when the class gets instantiated. Finally, we subclass `~sdssdb.peewee.BaseModel` and we bind the database connection to it. -Next we need to creates the model classes themselves. At its simplest, a model class represents a table in a given schema and contains a list of the columns in the table, each one as a class attribute. Model classes must subclass from a base class (``AwesomedbModel`` in our example) that has been linked to the database connection. Differently from SQLAlchemy, Peewee requires that all the columns be explicitly described, as opposed to autoloaded. To help with this task you can use the `pwiz `__ model generator. For example, to create a file with the list of model classes for ``stupendous`` you would run, from a terminal :: +Next we need to creates the model classes themselves. At its simplest, a model class represents a table in a given schema and contains a list of the columns in the table, each one as a class attribute. Model classes must subclass from a base class (``AwesomedbModel`` in our example) that has been linked to the database connection. The default mode in Peewee is to explicitely define all columns, as opposed to autoloaded. To help with this task you can use the `pwiz `__ model generator. For example, to create a file with the list of model classes for ``stupendous`` you would run, from a terminal :: python -m pwiz -e postgresql -s stupendous awesomedb > stupendous.py @@ -76,6 +76,30 @@ Once the file has been generated you will need to do some changes. On the top of The first line conveniently allows for access to the database connection from the schema submodule. The second one renames ``AwesomedbModel`` to ``BaseModel`` so that all the model classes in the file inherit from it. You'll probably need to make some other changes to the file, especially to the foreign keys to make sure they match your naming requirements. +Using reflection with Peewee +'''''''''''''''''''''''''''' + +Peewee provides a :ref:`reflection ` utility (internally used by ``pwiz``). Based on this tool we developed a `reflection metaclass <.ReflectMeta>` that can be used to expedite the creating of models by only requiring to define foreign keys. Note that this is not an official component of Peewee and it comes with certain caveats. Before using the reflection metaclass, make sure to read the `API documentation <.ReflectMeta>`. + +To define a base class with reflection with do :: + + import peewee + from sdssdb.peewee import ReflectMeta + + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + class Meta: + primary_key = False # To make sure Peewee doesn't add its own PK. + use_reflection = False # We'll enable reflection manually for certain models. + database = database + + class AwesomedbModel(ReflectBaseModel): + class Meta: + use_reflection = True + schema = 'stupendous' + table_name = 'stupendous_table' + +When the connection is created this model will be reflected and autocompleted with all the columns that exist in the table. The reflection does not include `foreign keys `, which must be created manually (along with their referenced columns). You can check the `catalogdb `__ models for an implementation of this type. + SQLAlchemy ^^^^^^^^^^ diff --git a/docs/sphinx/installation.rst b/docs/sphinx/installation.rst index 9613097e..f1bdde0b 100644 --- a/docs/sphinx/installation.rst +++ b/docs/sphinx/installation.rst @@ -10,11 +10,15 @@ Installation pip install sdssdb -then you can check that it works by running a python terminal and doing :: +then you can check that it works by running a python terminal and doing + +.. parsed-literal:: >>> import sdssdb >>> sdssdb.__version__ - 0.3.3dev + |sdssdb_version| + +This should provide all the libraries that you'll need for basic use of ``sdssdb``. Some functions require libraries such as `astropy `__ or `pandas `__. To install all the necessary dependencies you can do ``pip install sdssdb[all]``. If you want to also install the development and documentation tools, do ``pip install sdssdb[all,dev,docs]``. If you are working from Utah, ``sdssdb`` is installed as a module and you should be able to do :: diff --git a/docs/sphinx/intro.rst b/docs/sphinx/intro.rst index 0d261207..6adad910 100644 --- a/docs/sphinx/intro.rst +++ b/docs/sphinx/intro.rst @@ -21,6 +21,8 @@ This will returns the first 10 results from Gaia DR2 with g magnitude in the ran >>> session = database.Session() >>> targets = session.query(GaiaDR2Source).filter((GaiaDR2Source.phot_g_mean_mag > 15) & (GaiaDR2Source.phot_g_mean_mag < 16)).limit(10).all() +.. warning:: Note that the implementation of ``catalogdb`` in SQLALchemy is very limited and should not be used in general. + Available databases ------------------- @@ -111,15 +113,15 @@ Note that the level of readiness is not necessarily identical in both Peewee and sdss5db catalogdb - - + + targetdb - + archive @@ -185,8 +187,7 @@ Now imagine the case in which you are running ``sdssdb`` from your local compute >>> database -There are two database connections, ``SQLADatabaseConnection`` and ``PeeWeeDatabaseConnection``, one for each mapping library. Each database connection has two keyword arguments: a user/machine profile, a database name. The connection will automatically attempt to connect to the specified database with the profile unless the ``autoconnect`` keyword is set to `False`. -:: +There are two database connections, ``SQLADatabaseConnection`` and ``PeeWeeDatabaseConnection``, one for each mapping library. Each database connection has two keyword arguments: a user/machine profile, a database name. The connection will automatically attempt to connect to the specified database with the profile unless the ``autoconnect`` keyword is set to `False`. :: # load a database connection with the Utah manga machine profile and connect to the manga database. To create a Peewee conenction replace with PeeweeDatabaseConnection. from sdssdb.connection import SQLADatabaseConnection diff --git a/python/sdssdb/__init__.py b/python/sdssdb/__init__.py index 2c4d267f..a180f81f 100644 --- a/python/sdssdb/__init__.py +++ b/python/sdssdb/__init__.py @@ -1,68 +1,34 @@ # encoding: utf-8 -# flake8: noqa from __future__ import absolute_import, division, print_function, unicode_literals -import os import warnings -from pkg_resources import parse_version -import yaml +from sdsstools import get_config, get_logger, get_package_version -# Inits the logging system. Only shell logging, and exception and warning catching. -# File logging can be started by calling log.start_file_logger(name). -from .misc import log # noqa - - -def merge(user, default): - """Merges a user configuration with the default one.""" - - if isinstance(user, dict) and isinstance(default, dict): - for kk, vv in default.items(): - if kk not in user: - user[kk] = vv - else: - user[kk] = merge(user[kk], vv) - - return user +warnings.filterwarnings( + 'ignore', '.*Skipped unsupported reflection of expression-based index .*q3c.*') NAME = 'sdssdb' +__version__ = get_package_version(path='./', package_name=NAME) -yaml_kwds = dict() -if parse_version(yaml.__version__) >= parse_version('5.1'): - yaml_kwds.update(Loader=yaml.FullLoader) - +log = get_logger(NAME) # Loads config -config_file = os.path.dirname(__file__) + '/etc/{0}.yml'.format(NAME) -config = yaml.load(open(config_file), **yaml_kwds) - -# If there is a custom configuration file, updates the defaults using it. -custom_config_fn = os.path.expanduser('~/.{0}/{0}.yml'.format(NAME)) -if os.path.exists(custom_config_fn): - config = merge(yaml.load(open(custom_config_fn), **yaml_kwds), config) - -# from sdssdb.sqlalchemy import db -# config['db'] = db - -warnings.filterwarnings( - 'ignore', '.*Skipped unsupported reflection of expression-based index .*q3c.*') - - -__version__ = '0.3.3dev' +config = get_config(NAME, user_path='~/.sdssdb/sdssdb.yml') try: - import peewee + import peewee # noqa _peewee = True except ImportError: _peewee = False try: - import sqlalchemy + import sqlalchemy # noqa _sqla = True except ImportError: _sqla = False @@ -71,9 +37,7 @@ def merge(user, default): 'Install at least one of them to use sdssdb.') -from .connection import DatabaseConnection - if _peewee: - from .connection import PeeweeDatabaseConnection + from .connection import PeeweeDatabaseConnection # noqa if _sqla: - from .connection import SQLADatabaseConnection + from .connection import SQLADatabaseConnection # noqa diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 0b2ffa6c..20d4b322 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -14,12 +14,13 @@ import abc import importlib +import re import socket import six from pgpasslib import getpass -from sdssdb import config, log, _peewee, _sqla +from sdssdb import _peewee, _sqla, config, log if _peewee: @@ -124,7 +125,7 @@ def set_profile(self, profile=None, connect=True): # Tries to find a profile whose domain matches the hostname for profile in config: if 'domain' in config[profile] and config[profile]['domain'] is not None: - if hostname.endswith(config[profile]['domain']): + if re.match(config[profile]['domain'], hostname): self.profile = profile break @@ -179,19 +180,10 @@ def connect(self, dbname=None, silent_on_fail=False, **connection_params): # Gets the necessary configuration values from the profile db_configuration = {} for item in ['user', 'host', 'port']: - if item in connection_params and connection_params[item] is not None: + if item in connection_params: db_configuration[item] = connection_params[item] else: profile_value = config[self.profile].get(item, None) - - # If the hostname is the same as the current domain, - # do not specify the host. This helps with the configuration - # of the PSQL security at Utah. - if item == 'host': - domain = socket.getfqdn() - if profile_value == domain: - continue - db_configuration[item] = profile_value dbname = dbname or self.dbname @@ -326,7 +318,17 @@ def change_version(self, dbversion=None): if _peewee: class PeeweeDatabaseConnection(DatabaseConnection, PostgresqlDatabase): - """Peewee database connection implementation.""" + """Peewee database connection implementation. + + Attributes + ---------- + models : list + Models bound to this database. Only models that are bound using + `~sdssdb.peewee.BaseModel` are handled. + + """ + + models = dict() def __init__(self, *args, **kwargs): @@ -355,10 +357,16 @@ def _conn(self, dbname, silent_on_fail=False, **params): self.dbname = dbname except OperationalError: if not silent_on_fail: - log.warning(f'failed to connect to database {self.database!r}.', UserWarning) + log.warning(f'failed to connect to database {self.database!r}.') PostgresqlDatabase.init(self, None) self.connected = False + if self.is_connection_usable(): + for model in self.models.values(): + if getattr(model._meta, 'use_reflection', False): + if hasattr(model, 'reflect'): + model.reflect() + return self.connected diff --git a/python/sdssdb/etc/sdssdb.yml b/python/sdssdb/etc/sdssdb.yml index 18f7f75c..ea21dae8 100644 --- a/python/sdssdb/etc/sdssdb.yml +++ b/python/sdssdb/etc/sdssdb.yml @@ -36,9 +36,9 @@ sdssadmin: operations: user: sdss - host: operations.sdss.org + host: null port: 5432 - domain: operations.sdss.org + domain: operations.sdss.* manga: user: sdss diff --git a/python/sdssdb/peewee/__init__.py b/python/sdssdb/peewee/__init__.py index 6ff14b2b..b9970256 100644 --- a/python/sdssdb/peewee/__init__.py +++ b/python/sdssdb/peewee/__init__.py @@ -1,40 +1,223 @@ +# isort:skip_file # flake8: noqa +import re +import warnings + from sdssdb import _peewee if _peewee is False: raise ImportError('Peewee must be installed to use this module.') +import peewee +from peewee import Model, ModelBase, fn +from playhouse.hybrid import hybrid_method +from playhouse.reflection import generate_models + +from sdssdb import log +from sdssdb.core.exceptions import SdssdbUserWarning +from sdssdb.utils import is_table_locked -import re -from peewee import Model, fn -from playhouse.hybrid import hybrid_method +class ReflectMeta(ModelBase): + """A metaclass that supports model reflection on demand. + + This metaclass expands PeeWee's ``ModelBase`` to provide a hook for + declaring/expanding fields and indexes using the + :ref:`introspection system `. The feature is enabled + by a new attribute in :class:`peewee:Metadata` called ``use_reflection`` + (which is set to `False` by default). When set to `True` the metaclass + extends the model using the fields and indexes discovered + using reflection. It is possible to mix explicitely defined fields + with discovered ones; the latter never override the former. + + Normally `.ReflectMeta` is implemented by creating a base model that + is then used to defined the table models :: + + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + + class Meta: + primary_key = False + use_reflection = False + database = database + + class Tycho2(ReflectBaseModel): + + class Meta: + use_reflection = True + schema = 'catalogdb' + table_name = 'tycho2' + + Note that ``use_reflection`` is inheritable so if set to `True` in the base + class that will affect to all subclasses, except if it's overridden there. + It's also a good idea to set ``primary_key=False`` to prevent Peewee from + creating an ``id`` column automatically. + + If the database connection changes it's possible to call `.reflect` to + rediscover the reflected fields for the new connection. This will remove + all reflected fields (but not those explicitely added) and add the newly + discovered ones. + + If the database class is `.PeeweeDatabaseConnection`, the database will + call `.reflect` for each model bound to the database each time it connects. + This ensures that if the connection changes the reflected fields are + updated. Note that this will not work if using Peewee's + :class:`peewee:PostgresqlDatabase`. + + By default, `.ReflectMeta` will add all the fields from the reflected + models, including foreign keys. Sometimes that is not desirable and it's + preferable to define the foreign keys explicitely. In that case it's + possible to disable the reflection of foreign keys by doing :: + + class ReflectBaseModel(peewee.Model, metaclass=ReflectMeta): + + class Meta: + primary_key = False + use_reflection = False + reflection_options = {'skip_foreign_keys': True} + database = database + + Foreign keys explicitely defined need to reference existing fields, + so the referenced columns need to be added manually. In practice, this + means that if you add a `peewee:ForeignKeyField`, the referenced field + (usually the primary key) needs to be defined explicitely. + + Caveats: + + - Many-to-many relationships need to be defined explicitely since + it's not possible to set the through model based on the reflected + information. + + - When the primary key of a model is also a foreign key and + ``reflection_options = {'skip_foreign_keys': True}``, both the primary + key and the foreign key need to be defined explicitely. Otherwise neither + will be added. + + - Reflection will fail if a table is locked with ``AccessExclusiveLock``. + In that case reflection will be skipped and a warning issued. Note that + if the table is locked with an exclusive lock you won't be able to access + the data in any case. + + - In this version, indexes discovered by reflection are not propagated to + the model class. This should not have any impact in performance. + """ + + def __new__(cls, name, bases, attrs): + + Model = super(ReflectMeta, cls).__new__(cls, name, bases, attrs) + + Model._meta.use_reflection = getattr(Model._meta, 'use_reflection', False) + + database = Model._meta.database + if database and hasattr(database, 'models') and Model not in database.models: + database.models[Model._meta.table_name] = Model + + if Model._meta.use_reflection and database and database.connected: + cls.reflect(Model) + + return Model + + def reflect(self): + """Adds fields and indexes to the model using reflection.""" + + if not self.table_exists(): + return + + # Don't do anything if this model doesn't want reflection. + if not hasattr(self._meta, 'use_reflection') or not self._meta.use_reflection: + return + + for index in self._meta.indexes: + if hasattr(index, 'reflected') and index.reflected: + self._meta.indexes.remove(index) + + database = self._meta.database + if not database.is_connection_usable(): + raise peewee.DatabaseError('database not connected.') -class BaseModel(Model): + skip_fks = (hasattr(self._meta, 'reflection_options') and + self._meta.reflection_options.get('skip_foreign_keys', False)) + + table_name = self._meta.table_name + schema = self._meta.schema + + try: + locks = is_table_locked(database, table_name) + if locks and 'AccessExclusiveLock' in locks: + warnings.warn(f'table {schema}.{table_name} is locked and ' + 'will not be reflected.', SdssdbUserWarning) + return + ReflectedModel = generate_models(database, schema=schema, + table_names=[table_name])[table_name] + except KeyError as ee: + warnings.warn(f'reflection failed for {table_name}: table or column {ee} not found.', + SdssdbUserWarning) + return + except Exception as ee: + warnings.warn(f'reflection failed for {table_name}: {ee}', SdssdbUserWarning) + return + + for field_name, field in ReflectedModel._meta.fields.items(): + + if field_name in self._meta.fields: + meta_field = self._meta.fields[field_name] + if not hasattr(meta_field, 'reflected') or not meta_field.reflected: + continue + + if isinstance(field, peewee.ForeignKeyField) and skip_fks: + continue + + if field.primary_key: + self._meta.set_primary_key(field_name, field) + else: + self._meta.add_field(field_name, field) + + self._meta.fields[field_name].reflected = True + + # Composite keys are not a normal column so if the pk has not been + # set already, check if it exists in the reflected model. We avoid + # adding pks that are + if not self._meta.primary_key and ReflectedModel._meta.primary_key: + if isinstance(ReflectedModel._meta.primary_key, peewee.ForeignKeyField) and skip_fks: + pass + else: + pk = ReflectedModel._meta.primary_key + self._meta.set_primary_key(pk.name, pk) + + +class BaseModel(Model, metaclass=ReflectMeta): """A custom peewee `.Model` with enhanced representation and methods. By default it always prints ``pk``, ``name``, and ``label``, if found. - Models can define they own `.print_fields` as a list of field to be output - in the representation. + Models can define they own ``print_fields`` in ``Meta`` as a list of field + names to be output in the representation. """ - #: A list of fields (as strings) to be included in the ``__repr__`` - print_fields = [] + class Meta: + primary_key = False + use_reflection = False + print_fields = [] def __str__(self): """A custom str for the model repr.""" - pk_field = self._meta.primary_key.name - fields = ['{0}={1!r}'.format(pk_field, self.get_id())] + if self._meta.primary_key: + if self._meta.composite_key: + pk_field = '(' + ', '.join(self._meta.primary_key.field_names) + ')' + else: + pk_field = self._meta.primary_key.name + fields = ['{0}={1!r}'.format(pk_field, self.get_id())] + else: + pk_field = None + fields = [] for extra_field in ['label', 'name']: - if extra_field not in self.print_fields: - self.print_fields.append(extra_field) + if extra_field not in self._meta.print_fields: + self._meta.print_fields.append(extra_field) - for ff in self.print_fields: + for ff in self._meta.print_fields: if ff == pk_field: continue if hasattr(self, ff): @@ -60,7 +243,7 @@ def cone_search(self, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): return fn.q3c_ellipse_query(ra_attr, dec_attr, ra, dec, a, ratio, pa) @cone_search.expression - def cone_search(cls, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): + def cone_search(cls, ra, dec, a, b=None, pa=None, ra_col='ra', dec_col='dec'): # noqa """Returns a query with the rows inside a region on the sky. Defines a sky ellipse and returns the targets within. By default it diff --git a/python/sdssdb/peewee/operationsdb/platedb.py b/python/sdssdb/peewee/operationsdb/platedb.py index 24f65dc5..be47cbc9 100644 --- a/python/sdssdb/peewee/operationsdb/platedb.py +++ b/python/sdssdb/peewee/operationsdb/platedb.py @@ -141,8 +141,6 @@ class Meta: class Plate(OperationsDBModel): - print_fields = ['plate_id'] - chunk = TextField(null=True) comment = TextField(null=True) current_survey_mode = ForeignKeyField(column_name='current_survey_mode_pk', @@ -198,6 +196,7 @@ def mangadb_plate(self): class Meta: db_table = 'plate' schema = 'platedb' + print_fields = ['plate_id'] class PluggingStatus(OperationsDBModel): diff --git a/python/sdssdb/peewee/sdss5db/__init__.py b/python/sdssdb/peewee/sdss5db/__init__.py index 207a319a..8c2d058d 100644 --- a/python/sdssdb/peewee/sdss5db/__init__.py +++ b/python/sdssdb/peewee/sdss5db/__init__.py @@ -9,10 +9,3 @@ class SDSS5dbDatabaseConnection(PeeweeDatabaseConnection): database = SDSS5dbDatabaseConnection(autoconnect=True) - - -# Create a new base model class for the observatory and bind the database -class SDSS5dbModel(BaseModel): - - class Meta: - database = database diff --git a/python/sdssdb/peewee/sdss5db/catalogdb.py b/python/sdssdb/peewee/sdss5db/catalogdb.py index ca654fc2..88c38b34 100644 --- a/python/sdssdb/peewee/sdss5db/catalogdb.py +++ b/python/sdssdb/peewee/sdss5db/catalogdb.py @@ -9,1942 +9,819 @@ # @Last modified by: José Sánchez-Gallego (gallegoj@uw.edu) # @Last modified time: 2019-09-23 00:54:42 +import warnings -from peewee import (AutoField, BigAutoField, BigIntegerField, BooleanField, CharField, - DateField, DecimalField, DeferredThroughModel, DoubleField, - FloatField, ForeignKeyField, IntegerField, ManyToManyField, TextField) -from playhouse.postgres_ext import ArrayField - -from . import SDSS5dbModel, database # noqa - - -class AllWise(SDSS5dbModel): - - best_use_cntr = BigIntegerField(null=True) - cc_flags = CharField(null=True) - cntr = BigIntegerField(null=True) - coadd_id = CharField(null=True) - dec = DecimalField(index=True, null=True) - dec_pm = DecimalField(null=True) - designation = CharField(null=True) - det_bit = IntegerField(null=True) - elat = DecimalField(null=True) - elon = DecimalField(null=True) - ext_flg = IntegerField(null=True) - glat = DecimalField(index=True, null=True) - glon = DecimalField(index=True, null=True) - h_m_2mass = DecimalField(null=True) - h_msig_2mass = DecimalField(null=True) - htm20 = BigIntegerField(null=True) - j_m_2mass = DecimalField(null=True) - j_msig_2mass = DecimalField(null=True) - k_m_2mass = DecimalField(null=True) - k_msig_2mass = DecimalField(null=True) - moon_lev = CharField(null=True) - n_2mass = IntegerField(null=True) - na = IntegerField(null=True) - nb = IntegerField(null=True) - ngrp = IntegerField(null=True) - pa_2mass = DecimalField(null=True) - ph_qual = CharField(null=True) - pmcode = CharField(null=True) - pmdec = IntegerField(null=True) - pmra = IntegerField(null=True) - q12 = IntegerField(null=True) - q23 = IntegerField(null=True) - q34 = IntegerField(null=True) - r_2mass = DecimalField(null=True) - ra = DecimalField(index=True, null=True) - ra_pm = DecimalField(null=True) - rchi2 = FloatField(null=True) - rchi2_pm = FloatField(null=True) - rel = CharField(null=True) - rho12 = IntegerField(null=True) - rho23 = IntegerField(null=True) - rho34 = IntegerField(null=True) - satnum = CharField(null=True) - sigdec = DecimalField(null=True) - sigdec_pm = DecimalField(null=True) - sigpmdec = IntegerField(null=True) - sigpmra = IntegerField(null=True) - sigra = DecimalField(null=True) - sigra_pm = DecimalField(null=True) - sigradec = DecimalField(null=True) - sigradec_pm = DecimalField(null=True) - source_id = CharField(null=True) - spt_ind = IntegerField(null=True) - src = IntegerField(null=True) - tmass_key = IntegerField(null=True) - var_flg = CharField(null=True) - w1ba = DecimalField(null=True) - w1cc_map = IntegerField(null=True) - w1cc_map_str = CharField(null=True) - w1conf = DecimalField(null=True) - w1cov = DecimalField(null=True) - w1flg = IntegerField(null=True) - w1flg_1 = IntegerField(null=True) - w1flg_2 = IntegerField(null=True) - w1flg_3 = IntegerField(null=True) - w1flg_4 = IntegerField(null=True) - w1flg_5 = IntegerField(null=True) - w1flg_6 = IntegerField(null=True) - w1flg_7 = IntegerField(null=True) - w1flg_8 = IntegerField(null=True) - w1flux = FloatField(null=True) - w1gerr = DecimalField(null=True) - w1gflg = IntegerField(null=True) - w1gmag = DecimalField(null=True) - w1k = DecimalField(null=True) - w1m = IntegerField(null=True) - w1mag = DecimalField(null=True) - w1mag_1 = DecimalField(null=True) - w1mag_2 = DecimalField(null=True) - w1mag_3 = DecimalField(null=True) - w1mag_4 = DecimalField(null=True) - w1mag_5 = DecimalField(null=True) - w1mag_6 = DecimalField(null=True) - w1mag_7 = DecimalField(null=True) - w1mag_8 = DecimalField(null=True) - w1magp = DecimalField(null=True) - w1mcor = DecimalField(null=True) - w1mjdmax = DecimalField(null=True) - w1mjdmean = DecimalField(null=True) - w1mjdmin = DecimalField(null=True) - w1mlq = DecimalField(null=True) - w1mpro = DecimalField(null=True) - w1ndf = IntegerField(null=True) - w1nm = IntegerField(null=True) - w1pa = DecimalField(null=True) - w1rchi2 = FloatField(null=True) - w1rchi2_pm = FloatField(null=True) - w1rsemi = DecimalField(null=True) - w1sat = DecimalField(null=True) - w1sigflux = FloatField(null=True) - w1sigm = DecimalField(null=True) - w1sigm_1 = DecimalField(null=True) - w1sigm_2 = DecimalField(null=True) - w1sigm_3 = DecimalField(null=True) - w1sigm_4 = DecimalField(null=True) - w1sigm_5 = DecimalField(null=True) - w1sigm_6 = DecimalField(null=True) - w1sigm_7 = DecimalField(null=True) - w1sigm_8 = DecimalField(null=True) - w1sigmpro = DecimalField(null=True) - w1sigp1 = DecimalField(null=True) - w1sigp2 = DecimalField(null=True) - w1sigsk = DecimalField(null=True) - w1sky = DecimalField(null=True) - w1snr = DecimalField(null=True) - w2ba = DecimalField(null=True) - w2cc_map = IntegerField(null=True) - w2cc_map_str = CharField(null=True) - w2conf = DecimalField(null=True) - w2cov = DecimalField(null=True) - w2flg = IntegerField(null=True) - w2flg_1 = IntegerField(null=True) - w2flg_2 = IntegerField(null=True) - w2flg_3 = IntegerField(null=True) - w2flg_4 = IntegerField(null=True) - w2flg_5 = IntegerField(null=True) - w2flg_6 = IntegerField(null=True) - w2flg_7 = IntegerField(null=True) - w2flg_8 = IntegerField(null=True) - w2flux = FloatField(null=True) - w2gerr = DecimalField(null=True) - w2gflg = IntegerField(null=True) - w2gmag = DecimalField(null=True) - w2k = DecimalField(null=True) - w2m = IntegerField(null=True) - w2mag = DecimalField(null=True) - w2mag_1 = DecimalField(null=True) - w2mag_2 = DecimalField(null=True) - w2mag_3 = DecimalField(null=True) - w2mag_4 = DecimalField(null=True) - w2mag_5 = DecimalField(null=True) - w2mag_6 = DecimalField(null=True) - w2mag_7 = DecimalField(null=True) - w2mag_8 = DecimalField(null=True) - w2magp = DecimalField(null=True) - w2mcor = DecimalField(null=True) - w2mjdmax = DecimalField(null=True) - w2mjdmean = DecimalField(null=True) - w2mjdmin = DecimalField(null=True) - w2mlq = DecimalField(null=True) - w2mpro = DecimalField(null=True) - w2ndf = IntegerField(null=True) - w2nm = IntegerField(null=True) - w2pa = DecimalField(null=True) - w2rchi2 = FloatField(null=True) - w2rchi2_pm = FloatField(null=True) - w2rsemi = DecimalField(null=True) - w2sat = DecimalField(null=True) - w2sigflux = FloatField(null=True) - w2sigm = DecimalField(null=True) - w2sigm_1 = DecimalField(null=True) - w2sigm_2 = DecimalField(null=True) - w2sigm_3 = DecimalField(null=True) - w2sigm_4 = DecimalField(null=True) - w2sigm_5 = DecimalField(null=True) - w2sigm_6 = DecimalField(null=True) - w2sigm_7 = DecimalField(null=True) - w2sigm_8 = DecimalField(null=True) - w2sigmpro = DecimalField(null=True) - w2sigp1 = DecimalField(null=True) - w2sigp2 = DecimalField(null=True) - w2sigsk = DecimalField(null=True) - w2sky = DecimalField(null=True) - w2snr = DecimalField(null=True) - w3ba = DecimalField(null=True) - w3cc_map = IntegerField(null=True) - w3cc_map_str = CharField(null=True) - w3conf = DecimalField(null=True) - w3cov = DecimalField(null=True) - w3flg = IntegerField(null=True) - w3flg_1 = IntegerField(null=True) - w3flg_2 = IntegerField(null=True) - w3flg_3 = IntegerField(null=True) - w3flg_4 = IntegerField(null=True) - w3flg_5 = IntegerField(null=True) - w3flg_6 = IntegerField(null=True) - w3flg_7 = IntegerField(null=True) - w3flg_8 = IntegerField(null=True) - w3flux = FloatField(null=True) - w3gerr = DecimalField(null=True) - w3gflg = IntegerField(null=True) - w3gmag = DecimalField(null=True) - w3k = DecimalField(null=True) - w3m = IntegerField(null=True) - w3mag = DecimalField(null=True) - w3mag_1 = DecimalField(null=True) - w3mag_2 = DecimalField(null=True) - w3mag_3 = DecimalField(null=True) - w3mag_4 = DecimalField(null=True) - w3mag_5 = DecimalField(null=True) - w3mag_6 = DecimalField(null=True) - w3mag_7 = DecimalField(null=True) - w3mag_8 = DecimalField(null=True) - w3magp = DecimalField(null=True) - w3mcor = DecimalField(null=True) - w3mjdmax = DecimalField(null=True) - w3mjdmean = DecimalField(null=True) - w3mjdmin = DecimalField(null=True) - w3mlq = DecimalField(null=True) - w3mpro = DecimalField(null=True) - w3ndf = IntegerField(null=True) - w3nm = IntegerField(null=True) - w3pa = DecimalField(null=True) - w3rchi2 = FloatField(null=True) - w3rchi2_pm = FloatField(null=True) - w3rsemi = DecimalField(null=True) - w3sat = DecimalField(null=True) - w3sigflux = FloatField(null=True) - w3sigm = DecimalField(null=True) - w3sigm_1 = DecimalField(null=True) - w3sigm_2 = DecimalField(null=True) - w3sigm_3 = DecimalField(null=True) - w3sigm_4 = DecimalField(null=True) - w3sigm_5 = DecimalField(null=True) - w3sigm_6 = DecimalField(null=True) - w3sigm_7 = DecimalField(null=True) - w3sigm_8 = DecimalField(null=True) - w3sigmpro = DecimalField(null=True) - w3sigp1 = DecimalField(null=True) - w3sigp2 = DecimalField(null=True) - w3sigsk = DecimalField(null=True) - w3sky = DecimalField(null=True) - w3snr = DecimalField(null=True) - w4ba = DecimalField(null=True) - w4cc_map = IntegerField(null=True) - w4cc_map_str = CharField(null=True) - w4conf = DecimalField(null=True) - w4cov = DecimalField(null=True) - w4flg = IntegerField(null=True) - w4flg_1 = IntegerField(null=True) - w4flg_2 = IntegerField(null=True) - w4flg_3 = IntegerField(null=True) - w4flg_4 = IntegerField(null=True) - w4flg_5 = IntegerField(null=True) - w4flg_6 = IntegerField(null=True) - w4flg_7 = IntegerField(null=True) - w4flg_8 = IntegerField(null=True) - w4flux = FloatField(null=True) - w4gerr = DecimalField(null=True) - w4gflg = IntegerField(null=True) - w4gmag = DecimalField(null=True) - w4k = DecimalField(null=True) - w4m = IntegerField(null=True) - w4mag = DecimalField(null=True) - w4mag_1 = DecimalField(null=True) - w4mag_2 = DecimalField(null=True) - w4mag_3 = DecimalField(null=True) - w4mag_4 = DecimalField(null=True) - w4mag_5 = DecimalField(null=True) - w4mag_6 = DecimalField(null=True) - w4mag_7 = DecimalField(null=True) - w4mag_8 = DecimalField(null=True) - w4magp = DecimalField(null=True) - w4mcor = DecimalField(null=True) - w4mjdmax = DecimalField(null=True) - w4mjdmean = DecimalField(null=True) - w4mjdmin = DecimalField(null=True) - w4mlq = DecimalField(null=True) - w4mpro = DecimalField(null=True) - w4ndf = IntegerField(null=True) - w4nm = IntegerField(null=True) - w4pa = DecimalField(null=True) - w4rchi2 = FloatField(null=True) - w4rchi2_pm = FloatField(null=True) - w4rsemi = DecimalField(null=True) - w4sat = DecimalField(null=True) - w4sigflux = FloatField(null=True) - w4sigm = DecimalField(null=True) - w4sigm_1 = DecimalField(null=True) - w4sigm_2 = DecimalField(null=True) - w4sigm_3 = DecimalField(null=True) - w4sigm_4 = DecimalField(null=True) - w4sigm_5 = DecimalField(null=True) - w4sigm_6 = DecimalField(null=True) - w4sigm_7 = DecimalField(null=True) - w4sigm_8 = DecimalField(null=True) - w4sigmpro = DecimalField(null=True) - w4sigp1 = DecimalField(null=True) - w4sigp2 = DecimalField(null=True) - w4sigsk = DecimalField(null=True) - w4sky = DecimalField(null=True) - w4snr = DecimalField(null=True) - wx = DecimalField(null=True) - wy = DecimalField(null=True) - x = DecimalField(null=True) - xscprox = DecimalField(null=True) - y = DecimalField(null=True) - z = DecimalField(null=True) +from peewee import (BigAutoField, BigIntegerField, CharField, + DeferredThroughModel, DoubleField, FloatField, + ForeignKeyField, IntegerField, ManyToManyField, TextField) + +from sdssdb.core.exceptions import SdssdbUserWarning + +from . import BaseModel, database # noqa + + +class CatalogdbModel(BaseModel): class Meta: - table_name = 'allwise' + database = database schema = 'catalogdb' + use_reflection = True + reflection_options = {'skip_foreign_keys': True} primary_key = False -class ErositaAGNMock(SDSS5dbModel): +_Gaia_DR2_TwoMass_Deferred = DeferredThroughModel() +_APOGEE_Star_Visit_Deferred = DeferredThroughModel() - allwise_designation = TextField(null=True) - ero_dec = DoubleField(index=True, null=True) - ero_det_like_0 = FloatField(null=True) - ero_detuid = TextField(null=True) - ero_flux = FloatField(index=True, null=True) - ero_flux_err = FloatField(null=True) - ero_ra = DoubleField(index=True, null=True) - ero_radec_err = FloatField(null=True) - ero_souuid = TextField(null=True) - gaia_dr2_source_id = BigIntegerField(null=True) - nsc_dr1_id = TextField(null=True) - pk = BigAutoField() - sdss_dr14_objid = BigIntegerField(null=True) - t_exp = DoubleField(null=True) - target_dec = DoubleField(index=True, null=True) - target_mag_r = FloatField(null=True) - target_magerr_r = FloatField(null=True) - target_match_prob = FloatField(null=True) - target_origin = IntegerField(null=True) - target_pmdec = FloatField(null=True) - target_pmra = FloatField(null=True) - target_priority = IntegerField(index=True, null=True) - target_ra = DoubleField(index=True, null=True) - class Meta: - table_name = 'erosita_agn_mock' - schema = 'catalogdb' +class Catalog(CatalogdbModel): + pass -class ErositaClustersMock(SDSS5dbModel): - - allwise_designation = TextField(null=True) - ero_dec = DoubleField(index=True, null=True) - ero_det_like_0 = FloatField(null=True) - ero_detuid = TextField(null=True) - ero_ext = FloatField(null=True) - ero_ext_err = FloatField(null=True) - ero_ext_like = FloatField(null=True) - ero_flux = FloatField(index=True, null=True) - ero_flux_err = FloatField(null=True) - ero_ra = DoubleField(index=True, null=True) - ero_radec_err = FloatField(null=True) - ero_souuid = TextField(null=True) - gaia_dr2_source_id = BigIntegerField(null=True) - nsc_dr1_id = TextField(null=True) - pk = BigAutoField() - sdss_dr14_objid = BigIntegerField(null=True) - t_exp = DoubleField(null=True) - target_dec = DoubleField(index=True, null=True) - target_mag_r = FloatField(null=True) - target_magerr_r = FloatField(null=True) - target_match_prob = FloatField(null=True) - target_origin = IntegerField(null=True) - target_pmdec = FloatField(null=True) - target_pmra = FloatField(null=True) - target_priority = IntegerField(index=True, null=True) - target_ra = DoubleField(index=True, null=True) +class AllWise(CatalogdbModel): + + cntr = BigIntegerField(primary_key=True) + designation = TextField() class Meta: - table_name = 'erosita_clusters_mock' - schema = 'catalogdb' + table_name = 'allwise' -class TwoMassPsc(SDSS5dbModel): - - a = CharField(null=True) - b_m_opt = FloatField(null=True) - bl_flg = CharField(null=True) - cc_flg = CharField(index=True, null=True) - coadd = IntegerField(null=True) - coadd_key = IntegerField(null=True) - date = DateField(null=True) - decl = DoubleField(index=True, null=True) - designation = CharField(index=True, null=True) - dist_edge_ew = IntegerField(null=True) - dist_edge_flg = CharField(null=True) - dist_edge_ns = IntegerField(null=True) - dist_opt = FloatField(null=True) - dup_src = IntegerField(null=True) - err_ang = IntegerField(null=True) - err_maj = FloatField(null=True) - err_min = FloatField(null=True) - ext_key = IntegerField(null=True) - gal_contam = IntegerField(index=True, null=True) - glat = FloatField(null=True) - glon = FloatField(null=True) - h_cmsig = FloatField(null=True) - h_m = FloatField(index=True, null=True) - h_m_stdap = FloatField(null=True) - h_msig_stdap = FloatField(null=True) - h_msigcom = FloatField(null=True) - h_psfchi = FloatField(null=True) - h_snr = FloatField(null=True) - hemis = CharField(null=True) - j_cmsig = FloatField(null=True) - j_m = FloatField(index=True, null=True) - j_m_stdap = FloatField(null=True) - j_msig_stdap = FloatField(null=True) - j_msigcom = FloatField(null=True) - j_psfchi = FloatField(null=True) - j_snr = FloatField(null=True) - jdate = DoubleField(null=True) - k_cmsig = FloatField(null=True) - k_m = FloatField(index=True, null=True) - k_m_stdap = FloatField(null=True) - k_msig_stdap = FloatField(null=True) - k_msigcom = FloatField(null=True) - k_psfchi = FloatField(null=True) - k_snr = FloatField(null=True) - mp_flg = IntegerField(null=True) - ndet = CharField(null=True) - nopt_mchs = IntegerField(null=True) - ph_qual = CharField(index=True, null=True) - phi_opt = IntegerField(null=True) - prox = FloatField(null=True) - pts_key = AutoField() - pxcntr = IntegerField(null=True) - pxpa = IntegerField(null=True) - ra = DoubleField(index=True, null=True) - rd_flg = CharField(index=True, null=True) - scan = IntegerField(null=True) - scan_key = IntegerField(null=True) - use_src = IntegerField(null=True) - vr_m_opt = FloatField(null=True) - x_scan = FloatField(null=True) +class TwoMassPSC(CatalogdbModel): + + pts_key = IntegerField(primary_key=True) + designation = TextField() class Meta: table_name = 'twomass_psc' - schema = 'catalogdb' -_GaiaDR2TmassBestNeighbourDeferred = DeferredThroughModel() - - -class GaiaDR2Source(SDSS5dbModel): - - a_g_percentile_lower = FloatField(null=True) - a_g_percentile_upper = FloatField(null=True) - a_g_val = FloatField(null=True) - astrometric_chi2_al = FloatField(index=True, null=True) - astrometric_excess_noise = DoubleField(null=True) - astrometric_excess_noise_sig = DoubleField(null=True) - astrometric_gof_al = FloatField(null=True) - astrometric_matched_observations = IntegerField(null=True) - astrometric_n_bad_obs_al = IntegerField(null=True) - astrometric_n_good_obs_al = IntegerField(index=True, null=True) - astrometric_n_obs_ac = IntegerField(null=True) - astrometric_n_obs_al = IntegerField(null=True) - astrometric_params_solved = IntegerField(null=True) - astrometric_primary_flag = BooleanField(null=True) - astrometric_pseudo_colour = DoubleField(null=True) - astrometric_pseudo_colour_error = DoubleField(null=True) - astrometric_sigma5d_max = FloatField(null=True) - astrometric_weight_al = FloatField(null=True) - b = DoubleField(index=True, null=True) - bp_g = FloatField(null=True) - bp_rp = FloatField(null=True) - dec = DoubleField(index=True, null=True) - dec_error = DoubleField(null=True) - dec_parallax_corr = FloatField(null=True) - dec_pmdec_corr = FloatField(null=True) - dec_pmra_corr = FloatField(null=True) - designation = TextField(null=True) - duplicated_source = BooleanField(null=True) - e_bp_min_rp_percentile_lower = FloatField(null=True) - e_bp_min_rp_percentile_upper = FloatField(null=True) - e_bp_min_rp_val = FloatField(null=True) - ecl_lat = DoubleField(index=True, null=True) - ecl_lon = DoubleField(index=True, null=True) - flame_flags = BigIntegerField(null=True) - frame_rotator_object_type = IntegerField(null=True) - g_rp = FloatField(null=True) - l = DoubleField(index=True, null=True) - lum_percentile_lower = FloatField(null=True) - lum_percentile_upper = FloatField(null=True) - lum_val = FloatField(null=True) - matched_observations = IntegerField(null=True) - mean_varpi_factor_al = FloatField(null=True) - parallax = DoubleField(index=True, null=True) - parallax_error = DoubleField(null=True) - parallax_over_error = FloatField(index=True, null=True) - parallax_pmdec_corr = FloatField(null=True) - parallax_pmra_corr = FloatField(null=True) - phot_bp_mean_flux = DoubleField(null=True) - phot_bp_mean_flux_error = DoubleField(null=True) - phot_bp_mean_flux_over_error = FloatField(index=True, null=True) - phot_bp_mean_mag = FloatField(index=True, null=True) - phot_bp_n_obs = IntegerField(null=True) - phot_bp_rp_excess_factor = FloatField(index=True, null=True) - phot_g_mean_flux = DoubleField(index=True, null=True) - phot_g_mean_flux_error = DoubleField(null=True) - phot_g_mean_flux_over_error = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - phot_g_n_obs = IntegerField(null=True) - phot_proc_mode = IntegerField(null=True) - phot_rp_mean_flux = DoubleField(null=True) - phot_rp_mean_flux_error = DoubleField(null=True) - phot_rp_mean_flux_over_error = FloatField(index=True, null=True) - phot_rp_mean_mag = FloatField(index=True, null=True) - phot_rp_n_obs = IntegerField(null=True) - phot_variable_flag = TextField(null=True) - pmdec = DoubleField(null=True) - pmdec_error = DoubleField(null=True) - pmra = DoubleField(null=True) - pmra_error = DoubleField(null=True) - pmra_pmdec_corr = FloatField(null=True) - priam_flags = BigIntegerField(null=True) - ra = DoubleField(index=True, null=True) - ra_dec_corr = FloatField(null=True) - ra_error = DoubleField(null=True) - ra_parallax_corr = FloatField(null=True) - ra_pmdec_corr = FloatField(null=True) - ra_pmra_corr = FloatField(null=True) - radial_velocity = DoubleField(null=True) - radial_velocity_error = DoubleField(null=True) - radius_percentile_lower = FloatField(null=True) - radius_percentile_upper = FloatField(null=True) - radius_val = FloatField(null=True) - random_index = BigIntegerField(null=True) - ref_epoch = DoubleField(null=True) - rv_nb_transits = IntegerField(null=True) - rv_template_fe_h = FloatField(null=True) - rv_template_logg = FloatField(null=True) - rv_template_teff = FloatField(null=True) - solution_id = BigIntegerField(index=True, null=True) - source_id = BigAutoField() - teff_percentile_lower = FloatField(null=True) - teff_percentile_upper = FloatField(null=True) - teff_val = FloatField(null=True) - visibility_periods_used = IntegerField(index=True, null=True) - - tmass_best_sources = ManyToManyField(TwoMassPsc, - through_model=_GaiaDR2TmassBestNeighbourDeferred, - backref='gaia_best_sources') +class Gaia_DR2(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) + + tmass_best = ManyToManyField(TwoMassPSC, + through_model=_Gaia_DR2_TwoMass_Deferred, + backref='gaia_best') class Meta: table_name = 'gaia_dr2_source' - schema = 'catalogdb' -class GaiaDR2Clean(SDSS5dbModel): +class Gaia_DR2_Clean(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) - source_id = BigAutoField(column_name='source_id') - source = ForeignKeyField(GaiaDR2Source, column_name='source_id', backref='gaia_clean') + source = ForeignKeyField(Gaia_DR2, + field='source_id', + column_name='source_id', + object_id_name='source_id', + backref='+') class Meta: table_name = 'gaia_dr2_clean' - schema = 'catalogdb' -class GaiaDR2WDCandidatesV1(SDSS5dbModel): - - ag = DoubleField(null=True) - astrometric_excess_noise = DoubleField(null=True) - astrometric_sigma5d_max = DoubleField(null=True) - b = DoubleField(index=True, null=True) - chi2 = DoubleField(null=True) - chisq_he = DoubleField(null=True) - dec = DoubleField(index=True, null=True) - dec_error = DoubleField(null=True) - density = DoubleField(null=True) - designation = TextField(null=True) - e_gmag = DoubleField(null=True) - e_imag = DoubleField(null=True) - e_rmag = DoubleField(null=True) - e_umag = DoubleField(null=True) - e_zmag = DoubleField(null=True) - elog_g = DoubleField(null=True) - elog_g_he = DoubleField(null=True) - emass = DoubleField(null=True) - emass_he = DoubleField(null=True) - eteff = DoubleField(null=True) - eteff_he = DoubleField(null=True) - gmag = DoubleField(null=True) - imag = DoubleField(null=True) - l = DoubleField(index=True, null=True) - log_g = DoubleField(null=True) - log_g_he = DoubleField(null=True) - mass = DoubleField(null=True) - mass_he = DoubleField(null=True) - parallax = DoubleField(null=True) - parallax_error = DoubleField(null=True) - phot_bp_mean_flux = DoubleField(null=True) - phot_bp_mean_flux_error = DoubleField(null=True) - phot_bp_mean_mag = DoubleField(null=True) - phot_bp_rp_excess_factor = DoubleField(null=True) - phot_g_mean_flux = DoubleField(index=True, null=True) - phot_g_mean_flux_error = DoubleField(null=True) - phot_g_mean_mag = DoubleField(index=True, null=True) - phot_rp_mean_flux = DoubleField(null=True) - phot_rp_mean_flux_error = DoubleField(null=True) - phot_rp_mean_mag = DoubleField(null=True) - pmdec = DoubleField(null=True) - pmdec_error = DoubleField(null=True) - pmra = DoubleField(null=True) - pmra_error = DoubleField(null=True) - pwd = FloatField(null=True) - pwd_correction = FloatField(null=True) - ra = DoubleField(index=True, null=True) - ra_error = DoubleField(null=True) - rmag = DoubleField(null=True) - sdss_name = TextField(null=True) - teff = DoubleField(null=True) - teff_he = DoubleField(null=True) - umag = DoubleField(null=True) - white_dwarf_name = TextField(null=True) - zmag = DoubleField(null=True) - - def gaia_source(self): - return GaiaDR2Source.select().get_id(self.source_id) +class GalacticGenesis(CatalogdbModel): class Meta: - table_name = 'gaia_dr2_wd_candidates_v1' - schema = 'catalogdb' + table_name = 'galactic_genesis' -class GaiaDR2SDSSDR9BestNeighbour(SDSS5dbModel): +class GalacticGenesisBig(CatalogdbModel): - angular_distance = DoubleField(null=True) - best_neighbour_multiplicity = IntegerField(null=True) - gaia_astrometric_params = IntegerField(null=True) - number_of_mates = IntegerField(null=True) - number_of_neighbours = IntegerField(null=True) - original_ext_source_id = TextField(null=True) - sdssdr9_oid = BigIntegerField(index=True, null=True) - source_id = BigAutoField() + class Meta: + table_name = 'galactic_genesis_big' + + +class GUVCat(CatalogdbModel): class Meta: - table_name = 'gaiadr2_sdssdr9_best_neighbour' - schema = 'catalogdb' + table_name = 'guvcat' -class GalacticGenesis(SDSS5dbModel): +class KeplerInput_DR10(CatalogdbModel): - g_dec = DoubleField(null=True) - g_ra = DoubleField(null=True) - gaiaid = BigAutoField() - h_m = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - t_dec = DoubleField(null=True) - t_ra = DoubleField(null=True) - twomass_desig = CharField(null=True, unique=True) + kic_kepler_id = BigIntegerField(primary_key=True) class Meta: - table_name = 'galactic_genesis' - schema = 'catalogdb' + table_name = 'kepler_input_10' -class GalacticGenesisBig(SDSS5dbModel): +class SDSS_DR13_PhotoObj(CatalogdbModel): - g_dec = DoubleField(null=True) - g_ra = DoubleField(null=True) - gaiaid = BigAutoField() - h_m = FloatField(index=True, null=True) - phot_g_mean_mag = FloatField(index=True, null=True) - t_dec = DoubleField(null=True) - t_ra = DoubleField(null=True) - twomass_desig = CharField(null=True) + objid = BigIntegerField(primary_key=True) class Meta: - table_name = 'galactic_genesis_big' - schema = 'catalogdb' + table_name = 'sdss_dr13_photoobj' -class GUVCat(SDSS5dbModel): - - avaspdec = DoubleField(null=True) - avaspra = DoubleField(null=True) - band = IntegerField(null=True) - chkobj_type = IntegerField(null=True) - corv = TextField(null=True) - dec = DoubleField(index=True, null=True) - difffuv = FloatField(null=True) - difffuvdist = FloatField(null=True) - diffnuv = FloatField(null=True) - diffnuvdist = FloatField(null=True) - e_bv = FloatField(null=True) - fexptime = FloatField(null=True) - fov_radius = FloatField(null=True) - fuv_a_world = FloatField(null=True) - fuv_artifact = IntegerField(null=True) - fuv_b_world = FloatField(null=True) - fuv_class_star = FloatField(null=True) - fuv_ellipticity = FloatField(null=True) - fuv_errtheta_j2000 = FloatField(null=True) - fuv_flags = IntegerField(null=True) - fuv_flux = FloatField(null=True) - fuv_fluxerr = FloatField(null=True) - fuv_fwhm_image = FloatField(null=True) - fuv_fwhm_world = FloatField(null=True) - fuv_kron_radius = FloatField(null=True) - fuv_mag = FloatField(null=True) - fuv_mag_aper_4 = FloatField(null=True) - fuv_mag_aper_6 = FloatField(null=True) - fuv_mag_auto = FloatField(null=True) - fuv_magerr = FloatField(null=True) - fuv_magerr_aper_4 = FloatField(null=True) - fuv_magerr_aper_6 = FloatField(null=True) - fuv_magerr_auto = FloatField(null=True) - fuv_ncat_flux_radius_3 = FloatField(null=True) - fuv_ncat_fwhm_image = FloatField(null=True) - fuv_poserr = FloatField(null=True) - fuv_pperr = FloatField(null=True) - fuv_theta_j2000 = FloatField(null=True) - fuv_weight = FloatField(null=True) - fuv_x_image = FloatField(null=True) - fuv_y_image = FloatField(null=True) - glat = DoubleField(null=True) - glon = DoubleField(null=True) - grank = IntegerField(null=True) - grankdist = IntegerField(null=True) - groupgid = TextField(null=True) - groupgiddist = TextField(null=True) - groupgidtot = TextField(null=True) - ib_poserr = FloatField(null=True) - img = IntegerField(null=True) - inlargeobj = TextField(null=True) - istherespectrum = IntegerField(null=True) - largeobjsize = FloatField(null=True) - mpstype = TextField(null=True) - nexptime = FloatField(null=True) - ngrank = IntegerField(null=True) - ngrankdist = BigIntegerField(null=True) - nuv_a_world = FloatField(null=True) - nuv_artifact = IntegerField(null=True) - nuv_b_world = FloatField(null=True) - nuv_class_star = FloatField(null=True) - nuv_ellipticity = FloatField(null=True) - nuv_errtheta_j2000 = FloatField(null=True) - nuv_flags = IntegerField(null=True) - nuv_flux = FloatField(null=True) - nuv_fluxerr = FloatField(null=True) - nuv_fwhm_image = FloatField(null=True) - nuv_fwhm_world = FloatField(null=True) - nuv_kron_radius = FloatField(null=True) - nuv_mag = FloatField(null=True) - nuv_mag_aper_4 = FloatField(null=True) - nuv_mag_aper_6 = FloatField(null=True) - nuv_mag_auto = FloatField(null=True) - nuv_magerr = FloatField(null=True) - nuv_magerr_aper_4 = FloatField(null=True) - nuv_magerr_aper_6 = FloatField(null=True) - nuv_magerr_auto = FloatField(null=True) - nuv_poserr = FloatField(null=True) - nuv_pperr = FloatField(null=True) - nuv_theta_j2000 = FloatField(null=True) - nuv_weight = FloatField(null=True) - nuv_x_image = FloatField(null=True) - nuv_y_image = FloatField(null=True) - objid = BigAutoField() - photoextractid = BigIntegerField(null=True) - primgid = BigIntegerField(null=True) - primgiddist = BigIntegerField(null=True) - prob = FloatField(null=True) - ra = DoubleField(index=True, null=True) - sep = FloatField(null=True) - sepas = FloatField(null=True) - sepasdist = FloatField(null=True) - subvisit = IntegerField(null=True) - tilenum = IntegerField(null=True) - type = IntegerField(null=True) +class SDSS_DR14_APOGEE_Visit(CatalogdbModel): + + visit_id = TextField(primary_key=True) class Meta: - table_name = 'guvcat' - schema = 'catalogdb' + table_name = 'sdss_dr14_apogeevisit' + +class SDSS_DR14_APOGEE_Star(CatalogdbModel): -class KeplerInput10(SDSS5dbModel): - - kic_altid = BigIntegerField(null=True) - kic_altsource = BigIntegerField(null=True) - kic_aq = IntegerField(null=True) - kic_av = FloatField(null=True) - kic_blend = IntegerField(null=True) - kic_catkey = BigIntegerField(null=True) - kic_cq = TextField(null=True) - kic_d51mag = DoubleField(null=True) - kic_dec = DoubleField(index=True, null=True) - kic_degree_ra = DoubleField(null=True) - kic_ebminusv = FloatField(null=True) - kic_feh = FloatField(null=True) - kic_fov_flag = IntegerField(null=True) - kic_galaxy = IntegerField(null=True) - kic_gkcolor = DoubleField(null=True) - kic_glat = DoubleField(index=True, null=True) - kic_glon = DoubleField(index=True, null=True) - kic_gmag = DoubleField(index=True, null=True) - kic_grcolor = DoubleField(null=True) - kic_gredmag = DoubleField(null=True) - kic_hmag = DoubleField(index=True, null=True) - kic_imag = DoubleField(index=True, null=True) - kic_jkcolor = DoubleField(null=True) - kic_jmag = DoubleField(index=True, null=True) - kic_kepler_id = BigAutoField() - kic_kepmag = DoubleField(index=True, null=True) - kic_kmag = DoubleField(index=True, null=True) - kic_logg = FloatField(null=True) - kic_parallax = FloatField(null=True) - kic_pmdec = DoubleField(null=True) - kic_pmra = DoubleField(null=True) - kic_pmtotal = FloatField(null=True) - kic_pq = TextField(null=True) - kic_ra = DoubleField(index=True, null=True) - kic_radius = FloatField(null=True) - kic_rmag = DoubleField(index=True, null=True) - kic_scpid = BigIntegerField(null=True) - kic_scpkey = BigIntegerField(null=True) - kic_teff = IntegerField(null=True) - kic_tm_designation = TextField(null=True) - kic_tmid = BigIntegerField(null=True) - kic_umag = DoubleField(index=True, null=True) - kic_variable = IntegerField(null=True) - kic_zmag = DoubleField(index=True, null=True) + apstar_id = TextField(primary_key=True) + apogee_id = TextField() + + visits = ManyToManyField(SDSS_DR14_APOGEE_Visit, + through_model=_APOGEE_Star_Visit_Deferred, + backref='stars') class Meta: - table_name = 'kepler_input_10' - schema = 'catalogdb' + table_name = 'sdss_dr14_apogeestar' + + +class SDSS_DR14_APOGEE_Star_Visit(CatalogdbModel): + apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, + backref='+', + lazy_load=False) -class SDSSDR13Photoobj(SDSS5dbModel): - - aperflux7_g = FloatField(null=True) - aperflux7_i = FloatField(null=True) - aperflux7_r = FloatField(null=True) - aperflux7_u = FloatField(null=True) - aperflux7_z = FloatField(null=True) - b = DoubleField(index=True, null=True) - balkanid = IntegerField(null=True) - calibstatus_g = IntegerField(null=True) - calibstatus_i = IntegerField(null=True) - calibstatus_r = IntegerField(null=True) - calibstatus_u = IntegerField(null=True) - calibstatus_z = IntegerField(null=True) - camcol = IntegerField(null=True) - colc = FloatField(null=True) - colcerr = FloatField(null=True) - colv = FloatField(null=True) - colverr = FloatField(null=True) - dec = DoubleField(index=True, null=True) - devflux_g = FloatField(null=True) - devflux_i = FloatField(null=True) - devflux_r = FloatField(null=True) - devflux_u = FloatField(null=True) - devflux_z = FloatField(null=True) - devfluxivar_g = FloatField(null=True) - devfluxivar_i = FloatField(null=True) - devfluxivar_r = FloatField(null=True) - devfluxivar_u = FloatField(null=True) - devfluxivar_z = FloatField(null=True) - expflux_g = FloatField(null=True) - expflux_i = FloatField(null=True) - expflux_r = FloatField(null=True) - expflux_u = FloatField(null=True) - expflux_z = FloatField(null=True) - expfluxivar_g = FloatField(null=True) - expfluxivar_i = FloatField(null=True) - expfluxivar_r = FloatField(null=True) - expfluxivar_u = FloatField(null=True) - expfluxivar_z = FloatField(null=True) - extinction_g = FloatField(null=True) - extinction_i = FloatField(null=True) - extinction_r = FloatField(null=True) - extinction_u = FloatField(null=True) - extinction_z = FloatField(null=True) - fiber2flux_g = FloatField(null=True) - fiber2flux_i = FloatField(null=True) - fiber2flux_r = FloatField(null=True) - fiber2flux_u = FloatField(null=True) - fiber2flux_z = FloatField(null=True) - fiber2fluxivar_g = FloatField(null=True) - fiber2fluxivar_i = FloatField(null=True) - fiber2fluxivar_r = FloatField(null=True) - fiber2fluxivar_u = FloatField(null=True) - fiber2fluxivar_z = FloatField(null=True) - fiber2mag_g = FloatField(null=True) - fiber2mag_i = FloatField(null=True) - fiber2mag_r = FloatField(null=True) - fiber2mag_u = FloatField(null=True) - fiber2mag_z = FloatField(null=True) - fiber2magerr_g = FloatField(null=True) - fiber2magerr_i = FloatField(null=True) - fiber2magerr_r = FloatField(null=True) - fiber2magerr_u = FloatField(null=True) - fiber2magerr_z = FloatField(null=True) - fiberflux_g = FloatField(null=True) - fiberflux_i = FloatField(null=True) - fiberflux_r = FloatField(null=True) - fiberflux_u = FloatField(null=True) - fiberflux_z = FloatField(null=True) - fiberfluxivar_g = FloatField(null=True) - fiberfluxivar_i = FloatField(null=True) - fiberfluxivar_r = FloatField(null=True) - fiberfluxivar_u = FloatField(null=True) - fiberfluxivar_z = FloatField(null=True) - fibermag_g = FloatField(null=True) - fibermag_i = FloatField(null=True) - fibermag_r = FloatField(null=True) - fibermag_u = FloatField(null=True) - fibermag_z = FloatField(null=True) - fibermagerr_g = FloatField(null=True) - fibermagerr_i = FloatField(null=True) - fibermagerr_r = FloatField(null=True) - fibermagerr_u = FloatField(null=True) - fibermagerr_z = FloatField(null=True) - field = IntegerField(null=True) - fieldid = BigIntegerField(null=True) - flags = BigIntegerField(null=True) - htmid = BigIntegerField(null=True) - l = DoubleField(index=True, null=True) - lnldev_g = FloatField(null=True) - lnldev_i = FloatField(null=True) - lnldev_r = FloatField(null=True) - lnldev_u = FloatField(null=True) - lnldev_z = FloatField(null=True) - lnlexp_g = FloatField(null=True) - lnlexp_i = FloatField(null=True) - lnlexp_r = FloatField(null=True) - lnlexp_u = FloatField(null=True) - lnlexp_z = FloatField(null=True) - lnlstar_g = FloatField(null=True) - lnlstar_i = FloatField(null=True) - lnlstar_r = FloatField(null=True) - lnlstar_u = FloatField(null=True) - lnlstar_z = FloatField(null=True) - mrrcc_g = FloatField(null=True) - mrrcc_i = FloatField(null=True) - mrrcc_r = FloatField(null=True) - mrrcc_u = FloatField(null=True) - mrrcc_z = FloatField(null=True) - ndetect = IntegerField(null=True) - nmgypercount_g = FloatField(null=True) - nmgypercount_i = FloatField(null=True) - nmgypercount_r = FloatField(null=True) - nmgypercount_u = FloatField(null=True) - nmgypercount_z = FloatField(null=True) - obj = IntegerField(null=True) - objid = BigAutoField() - parentid = BigIntegerField(null=True) - petroflux_g = FloatField(null=True) - petroflux_i = FloatField(null=True) - petroflux_r = FloatField(null=True) - petroflux_u = FloatField(null=True) - petroflux_z = FloatField(null=True) - petrofluxivar_g = FloatField(null=True) - petrofluxivar_i = FloatField(null=True) - petrofluxivar_r = FloatField(null=True) - petrofluxivar_u = FloatField(null=True) - petrofluxivar_z = FloatField(null=True) - petromag_g = FloatField(null=True) - petromag_i = FloatField(null=True) - petromag_r = FloatField(null=True) - petromag_u = FloatField(null=True) - petromag_z = FloatField(null=True) - petromagerr_g = FloatField(null=True) - petromagerr_i = FloatField(null=True) - petromagerr_r = FloatField(null=True) - petromagerr_u = FloatField(null=True) - petromagerr_z = FloatField(null=True) - petror50_g = FloatField(null=True) - petror50_i = FloatField(null=True) - petror50_r = FloatField(null=True) - petror50_u = FloatField(null=True) - petror50_z = FloatField(null=True) - petror50err_g = FloatField(null=True) - petror50err_i = FloatField(null=True) - petror50err_r = FloatField(null=True) - petror50err_u = FloatField(null=True) - petror50err_z = FloatField(null=True) - petror90_g = FloatField(null=True) - petror90_i = FloatField(null=True) - petror90_r = FloatField(null=True) - petror90_u = FloatField(null=True) - petror90_z = FloatField(null=True) - petror90err_g = FloatField(null=True) - petror90err_i = FloatField(null=True) - petror90err_r = FloatField(null=True) - petror90err_u = FloatField(null=True) - petror90err_z = FloatField(null=True) - petrorad_g = FloatField(null=True) - petrorad_i = FloatField(null=True) - petrorad_r = FloatField(null=True) - petrorad_u = FloatField(null=True) - petrorad_z = FloatField(null=True) - petroraderr_g = FloatField(null=True) - petroraderr_i = FloatField(null=True) - petroraderr_r = FloatField(null=True) - petroraderr_u = FloatField(null=True) - petroraderr_z = FloatField(null=True) - psfflux_g = FloatField(null=True) - psfflux_i = FloatField(null=True) - psfflux_r = FloatField(null=True) - psfflux_u = FloatField(null=True) - psfflux_z = FloatField(null=True) - psffluxivar_g = FloatField(null=True) - psffluxivar_i = FloatField(null=True) - psffluxivar_r = FloatField(null=True) - psffluxivar_u = FloatField(null=True) - psffluxivar_z = FloatField(null=True) - psffwhm_g = FloatField(null=True) - psffwhm_i = FloatField(null=True) - psffwhm_r = FloatField(null=True) - psffwhm_u = FloatField(null=True) - psffwhm_z = FloatField(null=True) - psfmag_g = FloatField(index=True, null=True) - psfmag_i = FloatField(index=True, null=True) - psfmag_r = FloatField(index=True, null=True) - psfmag_u = FloatField(index=True, null=True) - psfmag_z = FloatField(index=True, null=True) - psfmagerr_g = FloatField(null=True) - psfmagerr_i = FloatField(null=True) - psfmagerr_r = FloatField(null=True) - psfmagerr_u = FloatField(null=True) - psfmagerr_z = FloatField(null=True) - ra = DoubleField(index=True, null=True) - rerun = IntegerField(null=True) - resolvestatus = IntegerField(null=True) - rowc = FloatField(null=True) - rowcerr = FloatField(null=True) - rowv = FloatField(null=True) - rowverr = FloatField(null=True) - run = IntegerField(null=True) - sky_g = FloatField(null=True) - sky_i = FloatField(null=True) - sky_r = FloatField(null=True) - sky_u = FloatField(null=True) - sky_z = FloatField(null=True) - skyivar_g = FloatField(null=True) - skyivar_i = FloatField(null=True) - skyivar_r = FloatField(null=True) - skyivar_u = FloatField(null=True) - skyivar_z = FloatField(null=True) - specobjid = DecimalField(null=True) - thingid = IntegerField(null=True) - type = IntegerField(null=True) + visit = ForeignKeyField(SDSS_DR14_APOGEE_Visit, + backref='+', + lazy_load=False) class Meta: - table_name = 'sdss_dr13_photoobj' - schema = 'catalogdb' + table_name = 'sdss_dr14_apogeestarvisit' + +class SDSS_DR14_ASCAP_Star(CatalogdbModel): -class SDSSDR14APOGEEStar(SDSS5dbModel): - - andflag = BigIntegerField(null=True) - apogee2_target1 = BigIntegerField(null=True) - apogee2_target2 = BigIntegerField(null=True) - apogee2_target3 = BigIntegerField(null=True) - apogee_id = CharField(null=True) - apogee_target1 = BigIntegerField(null=True) - apogee_target2 = BigIntegerField(null=True) - apogee_target3 = BigIntegerField(null=True) - apstar_id = CharField(primary_key=True) - apstar_version = CharField(null=True) - chi2_threshold = FloatField(null=True) - commiss = BigIntegerField(null=True) - dec = DoubleField(index=True, null=True) - extratarg = BigIntegerField(null=True) - field = CharField(null=True) - file = CharField(null=True) - glat = DoubleField(index=True, null=True) - glon = DoubleField(index=True, null=True) - htmid = BigIntegerField(null=True) - location_id = BigIntegerField(null=True) - max_h = FloatField(null=True) - max_jk = FloatField(null=True) - min_h = FloatField(null=True) - min_jk = FloatField(null=True) - nvisits = BigIntegerField(null=True) - ra = DoubleField(index=True, null=True) - reduction_id = CharField(null=True) - rv_autofwhm = FloatField(null=True) - rv_ccfwhm = FloatField(null=True) - rv_feh = FloatField(null=True) - rv_logg = FloatField(null=True) - rv_teff = FloatField(null=True) - snr = FloatField(null=True) - stablerv_chi2 = FloatField(null=True) - stablerv_chi2_prob = FloatField(null=True) - stablerv_rchi2 = FloatField(null=True) - starflag = BigIntegerField(null=True) - survey = CharField(null=True) - synthscatter = FloatField(null=True) - synthverr = FloatField(null=True) - synthverr_med = FloatField(null=True) - synthvhelio_avg = FloatField(null=True) - synthvscatter = FloatField(null=True) - target_id = CharField(null=True) - telescope = CharField(null=True) - verr = FloatField(null=True) - verr_med = FloatField(null=True) - vhelio_avg = FloatField(null=True) - vscatter = FloatField(null=True) + apstar_id = TextField(primary_key=True) + apstar = ForeignKeyField(SDSS_DR14_APOGEE_Star, + backref='ascap_stars') class Meta: - table_name = 'sdss_dr14_apogeestar' - schema = 'catalogdb' + table_name = 'sdss_dr14_ascapstar' + +class SDSS_DR14_Cannon_Star(CatalogdbModel): -class SDSSDR14APOGEEStarVisit(SDSS5dbModel): + @property + def apstars(self): + """Returns the associated stars in ``SDSSDR14APOGEEStar``.""" - visit_id = CharField(primary_key=True) - apstar_id = CharField(index=True, null=True) + return (SDSS_DR14_APOGEE_Star + .select() + .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id)) class Meta: - table_name = 'sdss_dr14_apogeestarvisit' - schema = 'catalogdb' + table_name = 'sdss_dr14_cannonstar' + + +class SDSS_APOGEE_AllStarMerge_r13(CatalogdbModel): + @property + def apstars(self): + """Returns the stars on `.SDSS_DR14_APOGEE_Star` with matching ``apogee_id``.""" -class SDSSDR14APOGEEVisit(SDSS5dbModel): - - apogee2_target1 = BigIntegerField(null=True) - apogee2_target2 = BigIntegerField(null=True) - apogee2_target3 = BigIntegerField(null=True) - apogee_id = CharField(null=True) - apogee_target1 = BigIntegerField(null=True) - apogee_target2 = BigIntegerField(null=True) - apogee_target3 = BigIntegerField(null=True) - apred_version = CharField(null=True) - bc = FloatField(null=True) - chisq = FloatField(null=True) - dateobs = CharField(null=True) - dec = DoubleField(index=True, null=True) - estvhelio = FloatField(null=True) - estvrel = FloatField(null=True) - estvrelerr = FloatField(null=True) - estvtype = BigIntegerField(null=True) - extratarg = BigIntegerField(null=True) - fiberid = BigIntegerField(null=True) - file = CharField(null=True) - glat = DoubleField(index=True, null=True) - glon = DoubleField(index=True, null=True) - jd = DoubleField(null=True) - location_id = BigIntegerField(null=True) - max_h = FloatField(null=True) - max_jk = FloatField(null=True) - min_h = FloatField(null=True) - min_jk = FloatField(null=True) - mjd = BigIntegerField(null=True) - plate = CharField(null=True) - ra = DoubleField(index=True, null=True) - reduction_id = CharField(null=True) - rv_alpha = FloatField(null=True) - rv_carb = FloatField(null=True) - rv_feh = FloatField(null=True) - rv_logg = FloatField(null=True) - rv_teff = FloatField(null=True) - snr = FloatField(null=True) - starflag = BigIntegerField(null=True) - survey = CharField(null=True) - synthfile = CharField(null=True) - synthvhelio = FloatField(null=True) - synthvrel = FloatField(null=True) - synthvrelerr = FloatField(null=True) - target_id = CharField(null=True) - telescope = CharField(null=True) - vhelio = FloatField(null=True) - visit_id = CharField(primary_key=True) - vrel = FloatField(null=True) - vrelerr = FloatField(null=True) - vtype = FloatField(null=True) + return (SDSS_DR14_APOGEE_Star + .select() + .where(SDSS_DR14_APOGEE_Star.apogee_id == self.apogee_id)) class Meta: - table_name = 'sdss_dr14_apogeevisit' - schema = 'catalogdb' + table_name = 'sdss_apogeeallstarmerge_r13' -class SDSSDR14ASCAPStar(SDSS5dbModel): - - al_fe = FloatField(null=True) - al_fe_err = FloatField(null=True) - al_fe_flag = IntegerField(null=True) - alpha_m = FloatField(null=True) - alpha_m_err = FloatField(null=True) - alpha_m_flag = IntegerField(null=True) - apogee_id = CharField(index=True, null=True) - apstar_id = CharField(primary_key=True) - aspcap_chi2 = FloatField(null=True) - aspcap_class = CharField(null=True) - aspcap_id = CharField(null=True) - aspcap_version = CharField(null=True) - aspcapflag = BigIntegerField(null=True) - c_fe = FloatField(null=True) - c_fe_err = FloatField(null=True) - c_fe_flag = IntegerField(null=True) - ca_fe = FloatField(null=True) - ca_fe_err = FloatField(null=True) - ca_fe_flag = IntegerField(null=True) - ci_fe = FloatField(null=True) - ci_fe_err = FloatField(null=True) - ci_fe_flag = IntegerField(null=True) - co_fe = FloatField(null=True) - co_fe_err = FloatField(null=True) - co_fe_flag = IntegerField(null=True) - cr_fe = FloatField(null=True) - cr_fe_err = FloatField(null=True) - cr_fe_flag = IntegerField(null=True) - cu_fe = FloatField(null=True) - cu_fe_err = FloatField(null=True) - cu_fe_flag = IntegerField(null=True) - fe_h = FloatField(null=True) - fe_h_err = FloatField(null=True) - fe_h_flag = IntegerField(null=True) - felem_al_h = FloatField(null=True) - felem_al_h_err = FloatField(null=True) - felem_c_m = FloatField(null=True) - felem_c_m_err = FloatField(null=True) - felem_ca_m = FloatField(null=True) - felem_ca_m_err = FloatField(null=True) - felem_ci_m = FloatField(null=True) - felem_ci_m_err = FloatField(null=True) - felem_co_h = FloatField(null=True) - felem_co_h_err = FloatField(null=True) - felem_cr_h = FloatField(null=True) - felem_cr_h_err = FloatField(null=True) - felem_cu_h = FloatField(null=True) - felem_cu_h_err = FloatField(null=True) - felem_fe_h = FloatField(null=True) - felem_fe_h_err = FloatField(null=True) - felem_ge_h = FloatField(null=True) - felem_ge_h_err = FloatField(null=True) - felem_k_h = FloatField(null=True) - felem_k_h_err = FloatField(null=True) - felem_mg_m = FloatField(null=True) - felem_mg_m_err = FloatField(null=True) - felem_mn_h = FloatField(null=True) - felem_mn_h_err = FloatField(null=True) - felem_n_m = FloatField(null=True) - felem_n_m_err = FloatField(null=True) - felem_na_h = FloatField(null=True) - felem_na_h_err = FloatField(null=True) - felem_nd_h = FloatField(null=True) - felem_nd_h_err = FloatField(null=True) - felem_ni_h = FloatField(null=True) - felem_ni_h_err = FloatField(null=True) - felem_o_m = FloatField(null=True) - felem_o_m_err = FloatField(null=True) - felem_p_m = FloatField(null=True) - felem_p_m_err = FloatField(null=True) - felem_rb_h = FloatField(null=True) - felem_rb_h_err = FloatField(null=True) - felem_s_m = FloatField(null=True) - felem_s_m_err = FloatField(null=True) - felem_si_m = FloatField(null=True) - felem_si_m_err = FloatField(null=True) - felem_ti_m = FloatField(null=True) - felem_ti_m_err = FloatField(null=True) - felem_tiii_m = FloatField(null=True) - felem_tiii_m_err = FloatField(null=True) - felem_v_h = FloatField(null=True) - felem_v_h_err = FloatField(null=True) - felem_y_h = FloatField(null=True) - felem_y_h_err = FloatField(null=True) - fparam_alpha_m = FloatField(null=True) - fparam_c_m = FloatField(null=True) - fparam_logg = FloatField(null=True) - fparam_logvmicro = FloatField(null=True) - fparam_m_h = FloatField(null=True) - fparam_n_m = FloatField(null=True) - fparam_teff = FloatField(null=True) - ge_fe = FloatField(null=True) - ge_fe_err = FloatField(null=True) - ge_fe_flag = IntegerField(null=True) - k_fe = FloatField(null=True) - k_fe_err = FloatField(null=True) - k_fe_flag = IntegerField(null=True) - logg = FloatField(null=True) - logg_err = FloatField(null=True) - logg_flag = IntegerField(null=True) - m_h = FloatField(null=True) - m_h_err = FloatField(null=True) - m_h_flag = IntegerField(null=True) - mg_fe = FloatField(null=True) - mg_fe_err = FloatField(null=True) - mg_fe_flag = IntegerField(null=True) - mn_fe = FloatField(null=True) - mn_fe_err = FloatField(null=True) - mn_fe_flag = IntegerField(null=True) - n_fe = FloatField(null=True) - n_fe_err = FloatField(null=True) - n_fe_flag = IntegerField(null=True) - na_fe = FloatField(null=True) - na_fe_err = FloatField(null=True) - na_fe_flag = IntegerField(null=True) - nd_fe = FloatField(null=True) - nd_fe_err = FloatField(null=True) - nd_fe_flag = IntegerField(null=True) - ni_fe = FloatField(null=True) - ni_fe_err = FloatField(null=True) - ni_fe_flag = IntegerField(null=True) - o_fe = FloatField(null=True) - o_fe_err = FloatField(null=True) - o_fe_flag = IntegerField(null=True) - p_fe = FloatField(null=True) - p_fe_err = FloatField(null=True) - p_fe_flag = IntegerField(null=True) - param_c_m = FloatField(null=True) - param_c_m_flag = IntegerField(null=True) - param_n_m = FloatField(null=True) - param_n_m_flag = IntegerField(null=True) - rb_fe = FloatField(null=True) - rb_fe_err = FloatField(null=True) - rb_fe_flag = IntegerField(null=True) - results_version = CharField(null=True) - s_fe = FloatField(null=True) - s_fe_err = FloatField(null=True) - s_fe_flag = IntegerField(null=True) - si_fe = FloatField(null=True) - si_fe_err = FloatField(null=True) - si_fe_flag = IntegerField(null=True) - target_id = CharField(index=True, null=True) - teff = FloatField(null=True) - teff_err = FloatField(null=True) - teff_flag = IntegerField(null=True) - ti_fe = FloatField(null=True) - ti_fe_err = FloatField(null=True) - ti_fe_flag = IntegerField(null=True) - tiii_fe = FloatField(null=True) - tiii_fe_err = FloatField(null=True) - tiii_fe_flag = IntegerField(null=True) - v_fe = FloatField(null=True) - v_fe_err = FloatField(null=True) - v_fe_flag = IntegerField(null=True) - vmacro = FloatField(null=True) - vmicro = FloatField(null=True) - vsini = FloatField(null=True) - y_fe = FloatField(null=True) - y_fe_err = FloatField(null=True) - y_fe_flag = IntegerField(null=True) +class SDSS_DR14_SpecObj(CatalogdbModel): + + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='bestobjid', + object_id_name='bestobjid', + backref='specobj_dr14') class Meta: - table_name = 'sdss_dr14_ascapstar' - schema = 'catalogdb' + table_name = 'sdss_dr14_specobj' -class SDSSDR14CannonStar(SDSS5dbModel): - - al_h = FloatField(null=True) - al_h_err = FloatField(null=True) - al_h_rawerr = FloatField(null=True) - alpha_m = FloatField(null=True) - alpha_m_err = FloatField(null=True) - alpha_m_rawerr = FloatField(null=True) - apogee_id = CharField(index=True, null=True) - c_h = FloatField(null=True) - c_h_err = FloatField(null=True) - c_h_rawerr = FloatField(null=True) - ca_h = FloatField(null=True) - ca_h_err = FloatField(null=True) - ca_h_rawerr = FloatField(null=True) - cannon_id = CharField(primary_key=True) - chi_sq = FloatField(null=True) - ci_h = FloatField(null=True) - ci_h_err = FloatField(null=True) - ci_h_rawerr = FloatField(null=True) - co_h = FloatField(null=True) - co_h_err = FloatField(null=True) - co_h_rawerr = FloatField(null=True) - cr_h = FloatField(null=True) - cr_h_err = FloatField(null=True) - cr_h_rawerr = FloatField(null=True) - fe_h = FloatField(null=True) - fe_h_err = FloatField(null=True) - fe_h_rawerr = FloatField(null=True) - field = CharField(null=True) - filename = CharField(null=True) - k_h = FloatField(null=True) - k_h_err = FloatField(null=True) - k_h_rawerr = FloatField(null=True) - location_id = BigIntegerField(null=True) - logg = FloatField(null=True) - logg_err = FloatField(null=True) - logg_rawerr = FloatField(null=True) - m_h = FloatField(null=True) - m_h_err = FloatField(null=True) - m_h_rawerr = FloatField(null=True) - mg_h = FloatField(null=True) - mg_h_err = FloatField(null=True) - mg_h_rawerr = FloatField(null=True) - mn_h = FloatField(null=True) - mn_h_err = FloatField(null=True) - mn_h_rawerr = FloatField(null=True) - n_h = FloatField(null=True) - n_h_err = FloatField(null=True) - n_h_rawerr = FloatField(null=True) - na_h = FloatField(null=True) - na_h_err = FloatField(null=True) - na_h_rawerr = FloatField(null=True) - ni_h = FloatField(null=True) - ni_h_err = FloatField(null=True) - ni_h_rawerr = FloatField(null=True) - o_h = FloatField(null=True) - o_h_err = FloatField(null=True) - o_h_rawerr = FloatField(null=True) - p_h = FloatField(null=True) - p_h_err = FloatField(null=True) - p_h_rawerr = FloatField(null=True) - r_chi_sq = FloatField(null=True) - s_h = FloatField(null=True) - s_h_err = FloatField(null=True) - s_h_rawerr = FloatField(null=True) - si_h = FloatField(null=True) - si_h_err = FloatField(null=True) - si_h_rawerr = FloatField(null=True) - teff = FloatField(null=True) - teff_err = FloatField(null=True) - teff_rawerr = FloatField(null=True) - ti_h = FloatField(null=True) - ti_h_err = FloatField(null=True) - ti_h_rawerr = FloatField(null=True) - tiii_h = FloatField(null=True) - tiii_h_err = FloatField(null=True) - tiii_h_rawerr = FloatField(null=True) - v_h = FloatField(null=True) - v_h_err = FloatField(null=True) - v_h_rawerr = FloatField(null=True) +class SDSS_DR16_SpecObj(SDSS_DR14_SpecObj): + + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='bestobjid', + object_id_name='bestobjid', + backref='specobj_dr16') class Meta: - table_name = 'sdss_dr14_cannonstar' - schema = 'catalogdb' + table_name = 'sdss_dr16_specobj' + + +class Gaia_DR2_TwoMass_Best_Neighbour(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) + gaia = ForeignKeyField(Gaia_DR2, + field='source_id', + column_name='source_id', + backref='+', + lazy_load=False) -class SDSSDR14SpecObj(SDSS5dbModel): - - ancillary_target1 = BigIntegerField(null=True) - ancillary_target2 = BigIntegerField(null=True) - anyandmask = IntegerField(null=True) - anyormask = IntegerField(null=True) - bestobjid = BigIntegerField(index=True, null=True) - bluefiber = IntegerField(null=True) - boss_target1 = BigIntegerField(null=True) - bossprimary = IntegerField(null=True) - bossspecobjid = IntegerField(null=True) - chi68p = FloatField(null=True) - chunk = CharField(null=True) - class_ = CharField(column_name='class', null=True) - class_noqso = CharField(null=True) - class_person = CharField(null=True) - comments_person = CharField(null=True) - cx = DoubleField(null=True) - cy = DoubleField(null=True) - cz = DoubleField(null=True) - dec = DoubleField(index=True, null=True) - deredsn2 = FloatField(null=True) - designid = IntegerField(null=True) - dof = FloatField(null=True) - eboss_target0 = BigIntegerField(null=True) - eboss_target1 = BigIntegerField(null=True) - eboss_target2 = BigIntegerField(null=True) - eboss_target_id = BigIntegerField(null=True) - elodiebv = FloatField(null=True) - elodiedof = FloatField(null=True) - elodiefeh = FloatField(null=True) - elodiefilename = CharField(null=True) - elodielogg = FloatField(null=True) - elodieobject = CharField(null=True) - elodierchi2 = FloatField(null=True) - elodiesptype = CharField(null=True) - elodieteff = FloatField(null=True) - elodiez = FloatField(null=True) - elodiezerr = FloatField(null=True) - elodiezmodelerr = FloatField(null=True) - fiberid = IntegerField(null=True) - firstrelease = CharField(null=True) - fluxobjid = BigIntegerField(index=True, null=True) - fracnsighi_1 = FloatField(null=True) - fracnsighi_10 = FloatField(null=True) - fracnsighi_2 = FloatField(null=True) - fracnsighi_3 = FloatField(null=True) - fracnsighi_4 = FloatField(null=True) - fracnsighi_5 = FloatField(null=True) - fracnsighi_6 = FloatField(null=True) - fracnsighi_7 = FloatField(null=True) - fracnsighi_8 = FloatField(null=True) - fracnsighi_9 = FloatField(null=True) - fracnsiglo_1 = FloatField(null=True) - fracnsiglo_10 = FloatField(null=True) - fracnsiglo_2 = FloatField(null=True) - fracnsiglo_3 = FloatField(null=True) - fracnsiglo_4 = FloatField(null=True) - fracnsiglo_5 = FloatField(null=True) - fracnsiglo_6 = FloatField(null=True) - fracnsiglo_7 = FloatField(null=True) - fracnsiglo_8 = FloatField(null=True) - fracnsiglo_9 = FloatField(null=True) - fracnsigma_1 = FloatField(null=True) - fracnsigma_10 = FloatField(null=True) - fracnsigma_2 = FloatField(null=True) - fracnsigma_3 = FloatField(null=True) - fracnsigma_4 = FloatField(null=True) - fracnsigma_5 = FloatField(null=True) - fracnsigma_6 = FloatField(null=True) - fracnsigma_7 = FloatField(null=True) - fracnsigma_8 = FloatField(null=True) - fracnsigma_9 = FloatField(null=True) - htmid = BigIntegerField(null=True) - instrument = CharField(null=True) - lambdaeff = FloatField(null=True) - legacy_target1 = BigIntegerField(null=True) - legacy_target2 = BigIntegerField(null=True) - legacyprimary = IntegerField(null=True) - loadversion = IntegerField(null=True) - mjd = IntegerField(null=True) - npoly = FloatField(null=True) - plate = IntegerField(null=True) - plateid = DecimalField(null=True) - platerun = CharField(null=True) - platesn2 = FloatField(null=True) - primtarget = BigIntegerField(null=True) - programname = CharField(null=True) - ra = DoubleField(index=True, null=True) - rchi2 = FloatField(null=True) - rchi2diff = FloatField(null=True) - rchi2diff_noqso = FloatField(null=True) - run1d = CharField(null=True) - run2d = CharField(null=True) - scienceprimary = IntegerField(null=True) - sdssprimary = IntegerField(null=True) - sectarget = BigIntegerField(null=True) - segue1_target1 = BigIntegerField(null=True) - segue1_target2 = BigIntegerField(null=True) - segue1primary = IntegerField(null=True) - segue2_target1 = BigIntegerField(null=True) - segue2_target2 = BigIntegerField(null=True) - segue2primary = IntegerField(null=True) - segueprimary = IntegerField(null=True) - sn1_g = FloatField(null=True) - sn1_i = FloatField(null=True) - sn1_r = FloatField(null=True) - sn2_g = FloatField(null=True) - sn2_i = FloatField(null=True) - sn2_r = FloatField(null=True) - snmedian = FloatField(null=True) - snmedian_g = FloatField(null=True) - snmedian_i = FloatField(null=True) - snmedian_r = FloatField(null=True) - snmedian_u = FloatField(null=True) - snmedian_z = FloatField(null=True) - snturnoff = FloatField(null=True) - sourcetype = CharField(null=True) - special_target1 = BigIntegerField(null=True) - special_target2 = BigIntegerField(null=True) - specobjid = DecimalField(primary_key=True) - spectroflux_g = FloatField(null=True) - spectroflux_i = FloatField(null=True) - spectroflux_r = FloatField(null=True) - spectroflux_u = FloatField(null=True) - spectroflux_z = FloatField(null=True) - spectrofluxivar_g = FloatField(null=True) - spectrofluxivar_i = FloatField(null=True) - spectrofluxivar_r = FloatField(null=True) - spectrofluxivar_u = FloatField(null=True) - spectrofluxivar_z = FloatField(null=True) - spectrographid = IntegerField(null=True) - spectroskyflux_g = FloatField(null=True) - spectroskyflux_i = FloatField(null=True) - spectroskyflux_r = FloatField(null=True) - spectroskyflux_u = FloatField(null=True) - spectroskyflux_z = FloatField(null=True) - spectrosynflux_g = FloatField(null=True) - spectrosynflux_i = FloatField(null=True) - spectrosynflux_r = FloatField(null=True) - spectrosynflux_u = FloatField(null=True) - spectrosynflux_z = FloatField(null=True) - spectrosynfluxivar_g = FloatField(null=True) - spectrosynfluxivar_i = FloatField(null=True) - spectrosynfluxivar_r = FloatField(null=True) - spectrosynfluxivar_u = FloatField(null=True) - spectrosynfluxivar_z = FloatField(null=True) - subclass = CharField(null=True) - subclass_noqso = CharField(null=True) - survey = CharField(null=True) - targetobjid = BigIntegerField(index=True, null=True) - targettype = CharField(null=True) - tcolumn_0 = IntegerField(null=True) - tcolumn_1 = IntegerField(null=True) - tcolumn_2 = IntegerField(null=True) - tcolumn_3 = IntegerField(null=True) - tcolumn_4 = IntegerField(null=True) - tcolumn_5 = IntegerField(null=True) - tcolumn_6 = IntegerField(null=True) - tcolumn_7 = IntegerField(null=True) - tcolumn_8 = IntegerField(null=True) - tcolumn_9 = IntegerField(null=True) - tfile = CharField(null=True) - theta_0 = FloatField(null=True) - theta_1 = FloatField(null=True) - theta_2 = FloatField(null=True) - theta_3 = FloatField(null=True) - theta_4 = FloatField(null=True) - theta_5 = FloatField(null=True) - theta_6 = FloatField(null=True) - theta_7 = FloatField(null=True) - theta_8 = FloatField(null=True) - theta_9 = FloatField(null=True) - thing_id = IntegerField(null=True) - thing_id_targeting = BigIntegerField(null=True) - tile = IntegerField(null=True) - veldisp = FloatField(null=True) - veldispchi2 = FloatField(null=True) - veldispdof = IntegerField(null=True) - veldisperr = FloatField(null=True) - veldispnpix = IntegerField(null=True) - veldispz = FloatField(null=True) - veldispzerr = FloatField(null=True) - wavemax = FloatField(null=True) - wavemin = FloatField(null=True) - wcoverage = FloatField(null=True) - xfocal = FloatField(null=True) - yfocal = FloatField(null=True) - z = FloatField(null=True) - z_noqso = FloatField(null=True) - z_person = FloatField(null=True) - zerr = FloatField(null=True) - zerr_noqso = FloatField(null=True) - zoffset = FloatField(null=True) - zwarning = IntegerField(null=True) - zwarning_noqso = IntegerField(null=True) + twomass = ForeignKeyField(TwoMassPSC, + field='pts_key', + column_name='tmass_pts_key', + backref='+', + lazy_load=False) class Meta: - table_name = 'sdss_dr14_specobj' - schema = 'catalogdb' + table_name = 'gaiadr2_tmass_best_neighbour' + + +class SDSS_DR14_QSO(CatalogdbModel): + + @property + def specobj(self): + """Returns the matching record in `.SDSS_DR16_SpecObj`.""" + + return SDSS_DR16_SpecObj.get(SDSS_DR16_SpecObj.plate == self.plate, + SDSS_DR16_SpecObj.mjd == self.mjd, + SDSS_DR16_SpecObj.fiberid == self.fiberid) + + class Meta: + table_name = 'sdss_dr14_qso' -class TIC_v8(SDSS5dbModel): - - allwise = CharField(index=True, null=True) - apass = CharField(index=True, null=True) - bmag = FloatField(null=True) - bmagflag = CharField(null=True) - contratio = FloatField(null=True) - d = FloatField(null=True) - dec = DoubleField(null=True) - dec_orig = DoubleField(null=True) - disposition = CharField(null=True) - distflag = CharField(null=True) - duplicate_id = BigIntegerField(null=True) - e_bmag = FloatField(null=True) - e_d = FloatField(null=True) - e_dec = DoubleField(null=True) - e_dec_orig = DoubleField(null=True) - e_ebv = FloatField(null=True) - e_gaiabp = FloatField(null=True) - e_gaiamag = FloatField(null=True) - e_gaiarp = FloatField(null=True) - e_gmag = FloatField(null=True) - e_hmag = FloatField(null=True) - e_imag = FloatField(null=True) - e_jmag = FloatField(null=True) - e_kmag = FloatField(null=True) - e_logg = FloatField(null=True) - e_lum = FloatField(null=True) - e_mass = FloatField(null=True) - e_mh = FloatField(null=True) - e_plx = FloatField(null=True) - e_pmdec = FloatField(null=True) - e_pmra = FloatField(null=True) - e_ra = DoubleField(null=True) - e_ra_orig = DoubleField(null=True) - e_rad = FloatField(null=True) - e_rho = FloatField(null=True) - e_rmag = FloatField(null=True) - e_teff = FloatField(null=True) - e_tmag = FloatField(null=True) - e_umag = FloatField(null=True) - e_vmag = FloatField(null=True) - e_w1mag = FloatField(null=True) - e_w2mag = FloatField(null=True) - e_w3mag = FloatField(null=True) - e_w4mag = FloatField(null=True) - e_zmag = FloatField(null=True) - ebv = FloatField(null=True) - ebvflag = CharField(null=True) - eclat = DoubleField(null=True) - eclong = DoubleField(null=True) - eneg_dist = FloatField(null=True) - eneg_ebv = FloatField(null=True) - eneg_logg = FloatField(null=True) - eneg_lum = FloatField(null=True) - eneg_mass = FloatField(null=True) - eneg_rad = FloatField(null=True) - eneg_rho = FloatField(null=True) - eneg_teff = FloatField(null=True) - epos_dist = FloatField(null=True) - epos_ebv = FloatField(null=True) - epos_logg = FloatField(null=True) - epos_lum = FloatField(null=True) - epos_mass = FloatField(null=True) - epos_rad = FloatField(null=True) - epos_rho = FloatField(null=True) - epos_teff = FloatField(null=True) - gaia = CharField(index=True, null=True) - gaiabp = FloatField(null=True) - gaiamag = FloatField(null=True) - gaiaqflag = IntegerField(null=True) - gaiarp = FloatField(null=True) - gallat = DoubleField(null=True) - gallong = DoubleField(null=True) - gmag = FloatField(null=True) - hip = IntegerField(index=True, null=True) - hmag = FloatField(null=True) - id = BigIntegerField(null=True) - imag = FloatField(null=True) - jmag = FloatField(null=True) - kic = IntegerField(index=True, null=True) - kmag = FloatField(null=True) - logg = FloatField(null=True) - lum = FloatField(null=True) - lumclass = CharField(null=True) - mass = FloatField(null=True) - mh = FloatField(null=True) - numcont = IntegerField(null=True) - objid = BigIntegerField(null=True) - objtype = CharField(null=True) - parflag = CharField(null=True) - plx = FloatField(null=True) - pmdec = FloatField(null=True) - pmflag = CharField(null=True) - pmra = FloatField(null=True) - posflag = CharField(null=True) - priority = FloatField(null=True) - prox = FloatField(null=True) - ra = DoubleField(null=True) - ra_orig = DoubleField(null=True) - rad = FloatField(null=True) - raddflag = IntegerField(null=True) - rho = FloatField(null=True) - rmag = FloatField(null=True) - sdss = BigIntegerField(index=True, null=True) - spflag = CharField(null=True) - splits = CharField(null=True) - starchareflag = CharField(null=True) - teff = FloatField(null=True) - tefflag = CharField(null=True) - tessflag = CharField(null=True) - tmag = FloatField(null=True) - twomass = CharField(index=True, null=True) - twomflag = CharField(null=True) - tyc = CharField(index=True, null=True) - typesrc = CharField(null=True) - ucac = CharField(index=True, null=True) - umag = FloatField(null=True) - version = CharField(null=True) - vmag = FloatField(null=True) - vmagflag = CharField(null=True) - w1mag = FloatField(null=True) - w2mag = FloatField(null=True) - w3mag = FloatField(null=True) - w4mag = FloatField(null=True) - wdflag = IntegerField(null=True) - zmag = FloatField(null=True) +class unWISE(CatalogdbModel): + + unwise_objid = TextField(primary_key=True) + + class Meta: + table_name = 'unwise' + + +class Legacy_Survey_DR8(CatalogdbModel): + + ref_cat = TextField() + ref_id = BigIntegerField() + + gaia = ForeignKeyField(Gaia_DR2, + column_name='gaia_sourceid', + object_id_name='gaia_sourceid', + backref='legacy_survey') + + class Meta: + table_name = 'legacy_survey_dr8' + + +class Gaia_unWISE_AGN(CatalogdbModel): + + gaia_sourceid = BigIntegerField(primary_key=True) + + unwise = ForeignKeyField(unWISE, field='unwise_objid', + column_name='unwise_objid', + object_id_name='unwise_objid', + backref='+') + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_sourceid', + object_id_name='gaia_sourceid_id', + backref='+') + + class Meta: + table_name = 'gaia_unwise_agn' + + +class eBOSS_Taarget_v5(CatalogdbModel): + + class Meta: + table_name = 'ebosstarget_v5' + + +# The following model (BhmSpidersGenericSuperset) does not need to be represented +# as a table in the database it is used only as the parent class of BhmSpidersAGNSuperset, +# BhmSpidersClustersSuperset which are real database tables. I am assuming that PeeWee +# can handle such a scheme without problems. If not, then we will have to duplicate the + +class BHM_Spiders_Generic_Superset(CatalogdbModel): + + # Not using reflection here to preserve Tom's notes. + + pk = BigAutoField() + + # Parameters derived from eROSITA eSASS catalogue + # Chosen to match X-ray columns defined in eROSITA/SDSS-V MoU (v2.0, April 2019) + ero_version = TextField(index=True, null=True) # string identifying this eROSITA data + # reduction version + ero_souuid = TextField(index=True, null=True) # string identifying this X-ray source + ero_flux = FloatField(null=True) # X-ray flux, 0.5-8keV band, erg/cm2/s + ero_flux_err = FloatField(null=True) # X-ray flux uncertainty, 0.5-8keV band, + # erg/cm2/s + ero_ext = FloatField(null=True) # X-ray extent parameter - arcsec + ero_ext_err = FloatField(null=True) # X-ray extent parameter uncertainty - arcsec + ero_ext_like = FloatField(null=True) # X-ray extent likelihood + ero_det_like = FloatField(null=True) # X-ray detection likelihood + ero_ra = DoubleField(index=True, null=True) # X-ray position, RA, ICRS, degrees + ero_dec = DoubleField(index=True, null=True) # X-ray position, Dec, ICRS, degrees + ero_radec_err = FloatField(null=True) # X-ray position uncertainty, arcsec + + # Parameters describing the cross-matching of X-ray to optical/IR catalogue(s) + + # 'ML+NWAY', 'LR' , 'SDSS_REDMAPPER', 'LS_REDMAPPER', 'HSC_REDMAPPER', 'MCMF' etc + xmatch_method = TextField(null=True) + # version identifier for cross-matching algorithm + xmatch_version = TextField(null=True) + # separation between X-ray position and opt positions - arcsec + xmatch_dist = FloatField(null=True) + # measure of quality of xmatch (e.g. p_any for Nway, LR) + xmatch_metric = FloatField(null=True) + # flavour of match, quality flags, e.g. NWAY match_flag - treat as bitmask + xmatch_flags = BigIntegerField(null=True) + + # Parameters that describe the major class of the object + target_class = TextField(null=True) # TBD, but e.g. 'unknown', 'AGN', 'Star', 'Galaxy' + + target_priority = IntegerField(null=True) # allows priority ranking based on + # info not available in catalogdb + target_has_spec = IntegerField(null=True) # (bitmask) allows flagging of targets + # that have a redshift from a catalogue + # that might not be listed in catalogdb + # follow bit pattern in spec compilation + # values < 0 means 'unknown' + + # Parameters derived from the cross-matched opt/IR catalogue + + # which optical catalogue(and version) provided this counterpart, + # e.g. 'ls_dr8', 'ps1_dr2' ... will also be the origin of the photometry columns below + best_opt = TextField(null=True) + + # arithmetically derived from ls_release, ls_brickid and ls_objid + # ls_id = ls_objid + ls_brickid * 2**16 + ls_release * 2**40 + # - make sure that we have a common definition within CatalogDB + # must be used when ls was the main source of counterparts to + # erosita source, otherwise is optional + ls_id = BigIntegerField(index=True, null=True) + + # Pan-STARRS1-DR2 object id (= ObjectThin.ObjID = StackObjectThin.ObjID) + # Must be used when ps1-dr2(+unWISE) was the main source of counterparts + # to an erosita source, otherwise is optional + ps1_dr2_objid = BigIntegerField(index=True, null=True) + + # derived from legacysurvey sweeps OPT_REF_ID when OPT_REF_CAT='G2' + # must be used when ls was the main source of counterparts to erosita source, + # otherwise is optional + # - SPIDERS team should also pre-match to gaia dr2 when using counterparts + # from non-LS catalogues + gaia_dr2_source_id = BigIntegerField(index=True, null=True) + + # Corresponds to the unWISE catalog band-merged 'unwise_objid' + # - should be used when ps1-dr2+unWISE was the main source of + # counterparts to erosita sources, otherwise is optional + unwise_dr1_objid = CharField(index=True, null=True, max_length=16) + + # provisional: + # Corresponds to the DES dr1 coadd_object_id + # - must be used when DES-dr1 was the primary source of counterpart to an + # erosita source, otherwise is optional + des_dr1_coadd_object_id = BigIntegerField(index=True, null=True) + + # Corresponds to the SDSS DR16 photoObj https://www.sdss.org/dr13/help/glossary/#ObjID + # - must be used when SDSS photoObj was the primary source of counterpart + # to an erosita source, otherwise is optional + sdss_dr16_objid = BigIntegerField(index=True, null=True) + + # included for convenience, but are copied from columns in other tables + opt_ra = DoubleField(index=True, null=True) + opt_dec = DoubleField(index=True, null=True) + opt_pmra = FloatField(null=True) + opt_pmdec = FloatField(null=True) + opt_epoch = FloatField(null=True) + + # For convenience we send a subset of magnitude columns over to the database + # - the full set of columns is available via a database JOIN to e.g. main ls_dr8 catalogue + # Note to self: Be careful with use of modelflux, fiberflux, fiber2flux etc! + opt_modelflux_g = FloatField(null=True) + opt_modelflux_ivar_g = FloatField(null=True) + opt_modelflux_r = FloatField(null=True) + opt_modelflux_ivar_r = FloatField(null=True) + opt_modelflux_r = FloatField(null=True) + opt_modelflux_ivar_r = FloatField(null=True) + opt_modelflux_i = FloatField(null=True) + opt_modelflux_ivar_i = FloatField(null=True) + opt_modelflux_z = FloatField(null=True) + opt_modelflux_ivar_z = FloatField(null=True) + + ls = ForeignKeyField(Legacy_Survey_DR8, backref='+') + gaia = ForeignKeyField(Gaia_DR2, object_id_name='gaia_dr2_source_id', backref='+') + + class Meta: + use_reflection = False + + +# Note that following models are currently identical in form, but may well diverge in the future + +class BHM_Spiders_AGN_Superset(BHM_Spiders_Generic_Superset): + + class Meta: + table_name = 'bhm_spiders_agn_superset' + + +class BHM_Spiders_Clusters_Superset(BHM_Spiders_Generic_Superset): + + class Meta: + table_name = 'bhm_spiders_clusters_superset' + + +class BHM_CSC(CatalogdbModel): + + class Meta: + table_name = 'bhm_csc' + + +class Gaia_DR2_WD_SDSS(CatalogdbModel): + + @property + def specobj(self): + """Returns the matching record in `.SDSS_DR16_SpecObj`.""" + + return SDSS_DR16_SpecObj.get(SDSS_DR16_SpecObj.plate == self.plate, + SDSS_DR16_SpecObj.mjd == self.mjd, + SDSS_DR16_SpecObj.fiberid == self.fiber) + + class Meta: + table_name = 'gaia_dr2_wd_sdss' + + +class Gaia_DR2_WD(CatalogdbModel): + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='wd', + unique=True) + + @property + def sdss(self): + """Returns records from `.Gaia_DR2_WD_SDSS` with matching ``wd``.""" + + return Gaia_DR2_WD_SDSS.select().where(Gaia_DR2_WD_SDSS.wd == self.wd) + + class Meta: + table_name = 'gaia_dr2_wd' + + +class Tycho2(CatalogdbModel): + + name = TextField() + + class Meta: + table_name = 'tycho2' + + +class GaiaQSO(CatalogdbModel): + + class Meta: + table_name = 'gaia_qso' + + +class TIC_v8(CatalogdbModel): + + id = BigIntegerField(primary_key=True) + + tycho2 = ForeignKeyField(Tycho2, field='name', + column_name='tyc', object_id_name='tyc', + backref='tic') + + twomass_psc = ForeignKeyField(TwoMassPSC, field='designation', + column_name='twomass_psc', + backref='tic') + + photoobj = ForeignKeyField(SDSS_DR13_PhotoObj, field='objid', + column_name='sdss', object_id_name='sdss', + backref='tic') + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_int', object_id_name='gaia_int', + backref='tic') + + allwise_target = ForeignKeyField(AllWise, field='designation', + column_name='allwise', object_id_name='allwise', + backref='tic') + + kepler_input = ForeignKeyField(KeplerInput_DR10, field='kic_kepler_id', + column_name='kic', object_id_name='kic', + backref='tic') class Meta: table_name = 'tic_v8' - schema = 'catalogdb' -class TwoMassClean(SDSS5dbModel): +class CatWISE(CatalogdbModel): + + class Meta: + table_name = 'catwise' - decl = DoubleField(null=True) - designation = CharField(primary_key=True) - h_m = FloatField(index=True, null=True) - ra = DoubleField(null=True) - twomassbrightneighbor = BooleanField(index=True, null=True) - psc = ForeignKeyField(TwoMassPsc, field='pts_key', - column_name='pts_key', backref='tmass_clean') +class CatWISEReject(CatalogdbModel): class Meta: - table_name = 'twomass_clean' - schema = 'catalogdb' + table_name = 'catwise_reject' + +class Watchdog(CatalogdbModel): -class TwoMassCleanNoNeighbor(SDSS5dbModel): + gaia_source_id = BigIntegerField(primary_key=True) - decl = DoubleField(null=True) - designation = CharField(primary_key=True) - h_m = FloatField(index=True, null=True) - pts_key = IntegerField(null=True) - ra = DoubleField(null=True) + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') class Meta: - table_name = 'twomass_clean_noneighbor' - schema = 'catalogdb' + table_name = 'watchdog' -class GaiaDR2TmassBestNeighbour(SDSS5dbModel): +class BlackCAT(CatalogdbModel): - angular_distance = DoubleField(null=True) - best_neighbour_multiplicity = IntegerField(null=True) - gaia_astrometric_params = IntegerField(null=True) - number_of_mates = IntegerField(null=True) - number_of_neighbours = IntegerField(null=True) - tmass_oid = BigIntegerField(index=True, null=True) + gaia_source_id = BigIntegerField(primary_key=True) - gaia_source = ForeignKeyField(GaiaDR2Source, - column_name='source_id', - backref='tmass_best_neighbour') - tmass_source = ForeignKeyField(TwoMassPsc, - column_name='tmass_pts_key', - field='pts_key', - backref='gaia_best_neighbour') + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') class Meta: - table_name = 'gaiadr2_tmass_best_neighbour' - schema = 'catalogdb' + table_name = 'blackcat' + + +class XRay_Pulsars(CatalogdbModel): + gaia_source_id = BigIntegerField(primary_key=True) -class DR14QV44(SDSS5dbModel): - - pk = AutoField(null=False, primary_key=True) - ancillary_target1 = BigIntegerField(null=True) - ancillary_target2 = BigIntegerField(null=True) - bi_civ = FloatField(null=True) - boss_target1 = BigIntegerField(null=True) - cc_flags = CharField(null=True) - col_number = IntegerField(null=True) - dec = FloatField(null=True) - eboss_target0 = BigIntegerField(null=True) - eboss_target1 = BigIntegerField(null=True) - eboss_target2 = BigIntegerField(null=True) - err_bi_civ = FloatField(null=True) - err_hmag = FloatField(null=True) - err_jmag = FloatField(null=True) - err_kmag = FloatField(null=True) - err_psfmag = ArrayField(null=True) # ARRAY - err_w1mag = FloatField(null=True) - err_w2mag = FloatField(null=True) - err_w3mag = FloatField(null=True) - err_w4mag = FloatField(null=True) - fiberid = IntegerField(null=True) - fiberid_duplicate = ArrayField(field_class=IntegerField, null=True) - field_number = IntegerField(null=True) - first_flux = FloatField(null=True) - first_matched = IntegerField(null=True) - first_snr = FloatField(null=True) - flux_0_2_12_0kev = FloatField(column_name='flux_0.2_12.0kev', null=True) - flux_0_2_12_0kev_err = FloatField(column_name='flux_0.2_12.0kev_err', null=True) - flux_0_2_2_0kev = FloatField(column_name='flux_0.2_2.0kev', null=True) - flux_0_2_2_0kev_err = FloatField(column_name='flux_0.2_2.0kev_err', null=True) - flux_2_0_12_0kev = FloatField(column_name='flux_2.0_12.0kev', null=True) - flux_2_0_12_0kev_err = FloatField(column_name='flux_2.0_12.0kev_err', null=True) - fuv = FloatField(null=True) - fuv_ivar = FloatField(null=True) - gal_ext = ArrayField(null=True) # ARRAY - galex_matched = FloatField(null=True) - hflux = FloatField(null=True) - hflux_err = FloatField(null=True) - hmag = FloatField(null=True) - hrdflag = IntegerField(null=True) - hsnr = FloatField(null=True) - ivar_psfflux = ArrayField(null=True) # ARRAY - jflux = FloatField(null=True) - jflux_err = FloatField(null=True) - jmag = FloatField(null=True) - jrdflag = IntegerField(null=True) - jsnr = FloatField(null=True) - kflux = FloatField(null=True) - kflux_err = FloatField(null=True) - kmag = FloatField(null=True) - krdflag = IntegerField(null=True) - ksnr = FloatField(null=True) - lum_0_2_12_0kev = FloatField(column_name='lum_0.2_12.0kev', null=True) - mi = FloatField(null=True) - mjd = IntegerField(null=True) - mjd_duplicate = ArrayField(field_class=IntegerField, null=True) - n_spec = IntegerField(null=True) - n_spec_boss = IntegerField(null=True) - n_spec_sdss = IntegerField(null=True) - nuv = FloatField(null=True) - nuv_ivar = FloatField(null=True) - obj_id = CharField(null=True) - ph_flags = CharField(null=True) - plate = IntegerField(null=True) - plate_duplicate = ArrayField(field_class=IntegerField, null=True) - psfflux = ArrayField(null=True) # ARRAY - psfmag = ArrayField(null=True) # ARRAY - ra = FloatField(null=True) - rass_counts = FloatField(null=True) - rass_counts_snr = FloatField(null=True) - rerun_number = CharField(null=True) - run_number = IntegerField(null=True) - sdss2first_sep = FloatField(null=True) - sdss2mass_sep = FloatField(null=True) - sdss2rosat_sep = FloatField(null=True) - sdss2wise_sep = FloatField(null=True) - sdss2xmm_sep = FloatField(null=True) - sdss_name = CharField(null=True) - source_z = CharField(null=True) - spectro = CharField(null=True) - spectro_duplicate = ArrayField(field_class=CharField, null=True) - thing_id = IntegerField(null=True) - ukidss_matched = FloatField(null=True) - w1chi2 = FloatField(null=True) - w1mag = FloatField(null=True) - w1snr = FloatField(null=True) - w2chi2 = FloatField(null=True) - w2mag = FloatField(null=True) - w2snr = FloatField(null=True) - w3chi2 = FloatField(null=True) - w3mag = FloatField(null=True) - w3snr = FloatField(null=True) - w4chi2 = FloatField(null=True) - w4mag = FloatField(null=True) - w4snr = FloatField(null=True) - yflux = FloatField(null=True) - yflux_err = FloatField(null=True) - z = FloatField(null=True) - z_err = FloatField(null=True) - z_mgii = FloatField(null=True) - z_pca = FloatField(null=True) - z_pca_er = FloatField(null=True) - z_pipe = FloatField(null=True) - z_pipe_err = FloatField(null=True) - z_vi = FloatField(null=True) - zwarning = IntegerField(null=True) + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') class Meta: - table_name = 'dr14q_v4_4' - schema = 'catalogdb' + table_name = 'xray_pulsars' + + +class LowMassXRayBinaries(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'lmxb' + + +class GalacticMillisecondPulsars(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'galactic_millisecond_pulsars' + + +class TransitionalMillisecondPulsars(CatalogdbModel): + + gaia_source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'transitional_msps' + + +class ATNF(CatalogdbModel): + + name = TextField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + class Meta: + table_name = 'atnf' + + +class SkyMapper_DR1_1(CatalogdbModel): + + allwise = ForeignKeyField(AllWise, field='cntr', + column_name='allwise_cntr', + object_id_name='allwise_cntr', + backref='skymapper') + + gaia1 = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_dr2_id1', + object_id_name='gaia_dr2_id1', + backref='skymapper1') + + gaia2 = ForeignKeyField(Gaia_DR2, field='source_id', + column_name='gaia_dr2_id2', + object_id_name='gaia_dr2_id2', + backref='skymapper2') + + @property + def twomass1(self): + """Returns the closest 2MASS PSC source, if defined.""" + + if self.twomass_cat1 != 'PSC': + return None + + return TwoMassPSC.get(pts_key=self.twomass_key1) + + @property + def twomass2(self): + """Returns the second closest 2MASS PSC source, if defined.""" + + if self.twomass_cat2 != 'PSC': + return None + + return TwoMassPSC.get(pts_key=self.twomass_key2) + + class Meta: + table_name = 'skymapper_dr1_1' + + +class PS1_g18(CatalogdbModel): + + class Meta: + table_name = 'ps1_g18' + + +class GLIMPSE(CatalogdbModel): + + twomass = ForeignKeyField(TwoMassPSC, field='pts_key', + column_name='tmass_cntr', + backref='glimpse_targets') + + class Meta: + table_name = 'glimpse' + print_fields = ['designation'] + + +class BHM_eFEDS_Veto(CatalogdbModel): + + class Meta: + table_name = 'bhm_efeds_veto' + + +class BestBrightest(CatalogdbModel): + + class Meta: + table_name = 'best_brightest' + + +class SkyMapperGaia(CatalogdbModel): + + skymapper_object_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, + column_name='gaia_source_id', + object_id_name='gaia_source_id', + backref='+') + + skymapper = ForeignKeyField(SkyMapper_DR1_1, + column_name='skymapper_object_id', + object_id_name='skymapper_object_id', + backref='+') + + class Meta: + table_name = 'skymapper_gaia' + print_fields = ['gaia_source_id'] + + +class UVOT_SSC_1(CatalogdbModel): + + class Meta: + table_name = 'uvotssc1' + + +class CataclysmicVariables(CatalogdbModel): + + class Meta: + table_name = 'cataclysmic_variables' + + +class YSO_Clustering(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='+') + + class Meta: + table_name = 'yso_clustering' + + +class MIPSGAL(CatalogdbModel): + + twomass = ForeignKeyField(TwoMassPSC, + object_id_name='twomass_name', + backref='+') + + @property + def glimpse(self): + """One-to-many for GLIMPSE targets.""" + + return GLIMPSE.select().where(GLIMPSE.designation == self.glimpse) + + class Meta: + table_name = 'mipsgal' + + +class TESS_TOI(CatalogdbModel): + + tic = ForeignKeyField(TIC_v8, + column_name='ticid', + object_id_name='ticid', + backref='+') + + class Meta: + table_name = 'tess_toi' + + +class XMM_OM_SUSS_4_1(CatalogdbModel): + + class Meta: + table_name = 'xmm_om_suss_4_1' + + +class GeometricDistances_Gaia_DR2(CatalogdbModel): + + source_id = BigIntegerField(primary_key=True) + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id', + object_id_name='source_id', + backref='+') + + class Meta: + table_name = 'geometric_distances_gaia_dr2' + + +class BHM_RM_v0(CatalogdbModel): + + gaia = ForeignKeyField(Gaia_DR2, + column_name='source_id_gaia', + object_id_name='source_id_gaia', + backref='+') + + unwise = ForeignKeyField(unWISE, + column_name='objid_unwise', + object_id_name='objid_unwise', + backref='+') + + sdss = ForeignKeyField(SDSS_DR13_PhotoObj, + column_name='objid_sdss', + object_id_name='objid_sdss', + backref='+') + + class Meta: + table_name = 'bhm_rm_v0' + + +_Gaia_DR2_TwoMass_Deferred.set_model(Gaia_DR2_TwoMass_Best_Neighbour) +_APOGEE_Star_Visit_Deferred.set_model(SDSS_DR14_APOGEE_Star_Visit) + + +# Add relational tables to namespace. +all_tables = database.get_tables('catalogdb') + +for rtname in all_tables: + if rtname.startswith('catalog_to_'): + + tname = rtname[len('catalog_to_'):] + if tname not in database.models: + warnings.warn(f'{rtname}: cannot find related table {tname!r}', + SdssdbUserWarning) + continue + + rel_model = database.models[tname] + model_name = 'CatalogTo' + rel_model.__name__ + + class Meta: + table_name = rtname + + RelationalModel = type(model_name, (CatalogdbModel,), {'Meta': Meta}) + RelationalModel._meta.add_field('catalog', + ForeignKeyField(Catalog, + column_name='catalogid', + backref='+')) + RelationalModel._meta.add_field('target', + ForeignKeyField(rel_model, + column_name='target_id', + backref='+')) -_GaiaDR2TmassBestNeighbourDeferred.set_model(GaiaDR2TmassBestNeighbour) + globals()[model_name] = RelationalModel diff --git a/python/sdssdb/peewee/sdss5db/targetdb.py b/python/sdssdb/peewee/sdss5db/targetdb.py index 52c9497c..a839eb79 100644 --- a/python/sdssdb/peewee/sdss5db/targetdb.py +++ b/python/sdssdb/peewee/sdss5db/targetdb.py @@ -11,14 +11,21 @@ ManyToManyField, SmallIntegerField, TextField) from playhouse.postgres_ext import ArrayField -from . import SDSS5dbModel, catalogdb, database # noqa +from . import BaseModel, database # noqa + + +class TargetdbBase(BaseModel): + + class Meta: + schema = 'targetdb' + database = database AssignmentDeferred = DeferredThroughModel() ProgramToTargetDeferred = DeferredThroughModel() -class Cadence(SDSS5dbModel): +class Cadence(TargetdbBase): delta = ArrayField(field_class=FloatField, null=True) delta_max = ArrayField(field_class=FloatField, null=True) delta_min = ArrayField(field_class=FloatField, null=True) @@ -30,19 +37,17 @@ class Cadence(SDSS5dbModel): class Meta: table_name = 'cadence' - schema = 'targetdb' -class Observatory(SDSS5dbModel): +class Observatory(TargetdbBase): label = TextField() pk = AutoField() class Meta: table_name = 'observatory' - schema = 'targetdb' -class Field(SDSS5dbModel): +class Field(TargetdbBase): cadence = ForeignKeyField(column_name='cadence_pk', field='pk', model=Cadence, @@ -58,10 +63,9 @@ class Field(SDSS5dbModel): class Meta: table_name = 'field' - schema = 'targetdb' -class Design(SDSS5dbModel): +class Design(TargetdbBase): field = ForeignKeyField(column_name='field_pk', field='pk', model=Field, @@ -71,28 +75,25 @@ class Design(SDSS5dbModel): class Meta: table_name = 'design' - schema = 'targetdb' -class Instrument(SDSS5dbModel): +class Instrument(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'instrument' - schema = 'targetdb' -class PositionerStatus(SDSS5dbModel): +class PositionerStatus(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'positioner_status' - schema = 'targetdb' -class PositionerInfo(SDSS5dbModel): +class PositionerInfo(TargetdbBase): apogee = BooleanField(null=False) boss = BooleanField(null=False) fiducial = BooleanField(null=False) @@ -100,10 +101,9 @@ class PositionerInfo(SDSS5dbModel): class Meta: table_name = 'positioner_info' - schema = 'targetdb' -class Positioner(SDSS5dbModel): +class Positioner(TargetdbBase): id = IntegerField(null=True) observatory = ForeignKeyField(column_name='observatory_pk', field='pk', @@ -120,10 +120,9 @@ class Positioner(SDSS5dbModel): class Meta: table_name = 'positioner' - schema = 'targetdb' -class Magnitude(SDSS5dbModel): +class Magnitude(TargetdbBase): bp = FloatField(null=True) g = FloatField(null=True) h = FloatField(null=True) @@ -134,37 +133,33 @@ class Magnitude(SDSS5dbModel): class Meta: table_name = 'magnitude' - schema = 'targetdb' -class Version(SDSS5dbModel): +class Version(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'version' - schema = 'targetdb' -class Category(SDSS5dbModel): +class Category(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'category' - schema = 'targetdb' -class Survey(SDSS5dbModel): +class Survey(TargetdbBase): label = TextField(null=True) pk = AutoField() class Meta: table_name = 'survey' - schema = 'targetdb' -class Program(SDSS5dbModel): +class Program(TargetdbBase): category = ForeignKeyField(column_name='category_pk', field='pk', model=Category, @@ -178,10 +173,9 @@ class Program(SDSS5dbModel): class Meta: table_name = 'program' - schema = 'targetdb' -class Target(SDSS5dbModel): +class Target(TargetdbBase): catalogid = BigIntegerField(null=True) dec = DoubleField(null=True) epoch = FloatField(null=True) @@ -215,10 +209,9 @@ class Target(SDSS5dbModel): class Meta: table_name = 'target' - schema = 'targetdb' -class Assignment(SDSS5dbModel): +class Assignment(TargetdbBase): design = ForeignKeyField(Design, column_name='design_pk', field='pk') @@ -235,10 +228,9 @@ class Assignment(SDSS5dbModel): class Meta: table_name = 'assignment' - schema = 'targetdb' -class ProgramToTarget(SDSS5dbModel): +class ProgramToTarget(TargetdbBase): cadence = ForeignKeyField(Cadence, column_name='cadence_pk', field='pk') @@ -259,7 +251,6 @@ class ProgramToTarget(SDSS5dbModel): class Meta: table_name = 'program_to_target' - schema = 'targetdb' AssignmentDeferred.set_model(Assignment) diff --git a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py index 5614c0e0..b3835e7c 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py @@ -10,13 +10,20 @@ from __future__ import absolute_import, division, print_function +import warnings + from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr from sqlalchemy.orm import relationship +from sdssdb.core.exceptions import SdssdbWarning from sdssdb.sqlalchemy.sdss5db import SDSS5Base, database +warnings.warn('support for catalogdb in SQLalchemy is incomplete. Use Peewee instead.', + SdssdbWarning) + + class Base(AbstractConcreteBase, SDSS5Base): __abstract__ = True _schema = 'catalogdb' diff --git a/python/sdssdb/utils/__init__.py b/python/sdssdb/utils/__init__.py index a8ecc0a9..3dd95c52 100644 --- a/python/sdssdb/utils/__init__.py +++ b/python/sdssdb/utils/__init__.py @@ -1,3 +1,5 @@ # flake8: noqa from .schemadisplay import * +from .ingest import * +from .internals import * diff --git a/python/sdssdb/utils/database.py b/python/sdssdb/utils/ingest.py similarity index 87% rename from python/sdssdb/utils/database.py rename to python/sdssdb/utils/ingest.py index 6bf7fb8e..3db7e433 100644 --- a/python/sdssdb/utils/database.py +++ b/python/sdssdb/utils/ingest.py @@ -3,19 +3,20 @@ # # @Author: José Sánchez-Gallego (gallegoj@uw.edu) # @Date: 2019-09-21 -# @Filename: load.py +# @Filename: ingest.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) # # @Last modified by: José Sánchez-Gallego (gallegoj@uw.edu) # @Last modified time: 2019-09-23 11:47:00 +import functools import io +import multiprocessing import os import re import warnings import inflect -import astropy.table import numpy import peewee from playhouse.postgres_ext import ArrayField @@ -33,6 +34,10 @@ progressbar = False +__all__ = ('to_csv', 'copy_data', 'drop_table', 'create_model_from_table', + 'bulk_insert', 'file_to_db') + + DTYPE_TO_FIELD = { 'i2': peewee.SmallIntegerField, 'i4': peewee.IntegerField, @@ -43,6 +48,44 @@ } +def to_csv(table, path, header=True, delimiter='\t', use_multiprocessing=False, workers=4): + """Creates a PostgreSQL-valid CSV file from a table, handling arrays. + + Parameters + ---------- + table : astropy.table.Table + The table to convert. + path : str + The path to which to write the CSV file. + header : bool + Whether to add a header with the column names. + delimiter : str + The delimiter between columns in the CSV files. + use_multiprocessing : bool + Whether to use multiple cores. The rows of the resulting file will not + have the same ordering as the original table. + workers : int + How many workers to use with multiprocessing. + + """ + + if use_multiprocessing: + pool = multiprocessing.Pool(workers) + tmp_list = pool.map(functools.partial(convert_row_to_psql, + delimiter=delimiter), + table, chunksize=1000) + else: + tmp_list = [convert_row_to_psql(row, delimiter=delimiter) for row in table] + + csv_str = '\n'.join(tmp_list) + + if header: + csv_str = delimiter.join(table.colnames) + '\n' + csv_str + + unit = open(path, 'w') + unit.write(csv_str) + + def table_exists(table_name, connection, schema=None): """Returns `True` if a table exists in a database. @@ -171,6 +214,28 @@ class Meta: return type(str(table_name), (BaseModel,), attrs) +def convert_row_to_psql(row, delimiter='\t', null='\\N'): + """Concerts an astropy table row to a Postgresql-valid CSV string.""" + + row_data = [] + + for col_value in row: + if numpy.isscalar(col_value): + row_data.append(str(col_value)) + elif numpy.ma.is_masked(col_value): + row_data.append(null) + else: + if col_value.dtype.base.str[1] == 'S': + col_value = col_value.astype('U') + row_data.append( + str(col_value.tolist()) + .replace('\n', '') + .replace('\'', '\"') + .replace('[', '{').replace(']', '}')) + + return delimiter.join(row_data) + + def copy_data(data, connection, table_name, schema=None, chunk_size=10000, show_progress=False): """Loads data into a DB table using ``COPY``. @@ -219,23 +284,7 @@ def copy_data(data, connection, table_name, schema=None, chunk_size=10000, for ii in iterable: row = data[ii] - - # Adds the pk - row_data = [] - - for col_value in row: - if numpy.isscalar(col_value): - row_data.append(str(col_value)) - else: - if col_value.dtype.base.str[1] == 'S': - col_value = col_value.astype('U') - row_data.append( - str(col_value.tolist()) - .replace('\n', '') - .replace('\'', '\"') - .replace('[', '{').replace(']', '}')) - - tmp_list.append('\t'.join(row_data)) + tmp_list.append(convert_row_to_psql(row)) chunk += 1 # If we have reached a chunk commit point, or this is the last item, @@ -354,6 +403,8 @@ def file_to_db(input_, connection, table_name, schema=None, lowercase=False, """ + import astropy.table + # If we drop we need to re-create but there is no need to truncate. if drop: create = True diff --git a/python/sdssdb/utils/internals.py b/python/sdssdb/utils/internals.py new file mode 100644 index 00000000..2d28f87c --- /dev/null +++ b/python/sdssdb/utils/internals.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-28 +# @Filename: internals.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import sys +import time + + +__all__ = ('vacuum_all', 'get_cluster_index', 'get_unclustered_tables', + 'get_row_count', 'is_table_locked') + + +def vacuum_all(database, analyze=True, verbose=False, schema=None): + """Vacuums all the tables in a database or schema. + + Parameters + ---------- + database : .PeeweeDatabaseConnection + A PeeWee connection to the database to vacuum. + analyze : bool + Whether to run ``ANALYZE`` when vacuumming. + verbose : bool + Whether to run in verbose mode. + schema : str + The schema to vacuum. If `None`, vacuums the entire database. + + """ + + def execute_sql(statement): + + if 'VEBOSE' in statement: + print(statement + ' ... ') + else: + sys.stdout.write(statement + ' ... ') + sys.stdout.flush() + + tstart = time.time() + + # Change isolation level to allow executing commands such as VACUUM. + connection = database.connection() + original_isolation_level = connection.isolation_level + connection.set_isolation_level(0) + + database.execute_sql(statement) + + tend = time.time() + telapsed = tend - tstart + + if 'VEBOSE' in statement: + print('Elapsed {telap sed:.1f} s') + else: + print(f'{telapsed:.1f} s') + + connection.set_isolation_level(original_isolation_level) + + assert database.is_connection_usable(), 'connection is not usable.' + + if schema is None: + + statement = 'VACUUM' + (' VEBOSE' if verbose else '') + (' ANALYZE' if analyze else '') + with database.atomic(): + execute_sql(statement) + + return + + tables = database.get_tables(schema=schema) + + for table in sorted(tables): + + table_name = table if schema is None else schema + '.' + table + statement = ('VACUUM' + + (' VEBOSE' if verbose else '') + + (' ANALYZE' if analyze else '') + + ' ' + table_name) + + with database.atomic(): + execute_sql(statement) + + +def get_cluster_index(connection, table_name=None, schema=None): + """Returns a tuple with the index on which a table has been clustered.""" + + table_name = table_name or '%%' + schema = schema or '%%' + + with connection.atomic(): + cursor = connection.execute_sql(f""" + SELECT + c.relname AS tablename, + n.nspname AS schemaname, + i.relname AS indexname + FROM pg_index x + JOIN pg_class c ON c.oid = x.indrelid + JOIN pg_class i ON i.oid = x.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE + x.indisclustered AND + n.nspname LIKE '{schema}' AND + c.relname LIKE '{table_name}'; + """) + + return cursor.fetchall() + + +def get_unclustered_tables(connection, schema=None): + """Lists tables not clustered.""" + + schema_like = schema or '%%' + + with connection.atomic(): + table_names = connection.execute_sql(f""" + SELECT + tablename + FROM pg_tables + WHERE + schemaname LIKE '{schema_like}'; + """).fetchall() + + table_names = [table_name[0] for table_name in table_names] + + clustered = get_cluster_index(connection, schema=schema) + if schema: + clustered = [idx for idx in clustered if idx[1] == schema] + + unclustered = [table for table in table_names + if table not in list(zip(*clustered))[0]] + + return unclustered + + +def get_row_count(connection, table_name, schema=None, approximate=True): + """Returns the model row count. + + Parameters + ---------- + connection : .PeeweeDatabaseConnection + The database connection. + table : str + The table name. + schema : str + The schema in which the table lives. + approximate : bool + If True, returns the approximate row count from the ``pg_class`` + table (much faster). Otherwise calculates the exact count. + + """ + + if approximate: + if schema is None: + sql = ('SELECT reltuples AS approximate_row_count ' + 'FROM pg_class WHERE relname = \'{table_name}\';') + else: + sql = ('SELECT reltuples AS approximate_row_count FROM pg_class ' + 'JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace ' + 'WHERE relname = \'{table_name}\' AND pg_namespace.nspname = \'{schema}\';') + else: + if schema is None: + sql = 'SELECT count(*) FROM {table_name};' + else: + sql = 'SELECT count(*) FROM {schema}.{table_name};' + + with connection.atomic(): + count = connection.execute_sql(sql.format(table_name=table_name, + schema=schema)).fetchall() + + if len(count) == 0: + raise ValueError('failed retrieving the row count. Check the table name and schema.') + + return count[0][0] + + +def is_table_locked(connection, table_name, schema=None): + """Returns the locks for a table or `None` if no lock is present.""" + + schema = schema or '%%' + + sql = ('SELECT mode FROM pg_locks JOIN pg_class ' + 'ON pg_class.oid = pg_locks.relation ' + 'JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace ' + f'WHERE pg_class.relname = {table_name!r} ' + f'AND pg_namespace.nspname LIKE {schema!r};') + + with connection.atomic(): + locks = connection.execute_sql(sql).fetchall() + + if not locks: + return None + + return list(zip(*locks))[0] diff --git a/python/sdssdb/utils/schemadisplay.py b/python/sdssdb/utils/schemadisplay.py index e4803c20..f0a3e3ba 100644 --- a/python/sdssdb/utils/schemadisplay.py +++ b/python/sdssdb/utils/schemadisplay.py @@ -12,12 +12,20 @@ # The following functions are adapted from the sqlalchemy_schemadisplay by # Florian Schulze (https://github.com/fschulze/sqlalchemy_schemadisplay). -import pydot -from peewee import ForeignKeyField +import re + +from peewee import ForeignKeyField, IndexMetadata + + +try: + import pydot +except ImportError: + pydot = None __all__ = ['create_schema_graph', 'show_schema_graph'] + field_type_psql = {'AUTO': 'SERIAL', 'BIGAUTO': 'BIGSERIAL', 'BIGINT': 'BIGINT', @@ -39,13 +47,18 @@ 'VARCHAR': 'VARCHAR'} -def _render_table_html(model, show_indices=True, show_datatypes=True): +def _render_table_html(model, show_columns=True, show_pks=True, + show_indices=True, show_datatypes=True): """Creates the HTML tags for a table, including PKs, FKs, and indices. Parameters ---------- model : `peewee.Model` The Peewee model for which to create the table. + show_columns : bool + Whether to show the column names. + show_pks : bool + Whether to show the primary key. Supersedes ``show_columns``. show_indices : `bool` Whether to show the indices from the table as separate rows. show_datatypes : `bool` @@ -56,27 +69,30 @@ def _render_table_html(model, show_indices=True, show_datatypes=True): table_name = model._meta.table_name fields = model._meta.fields - pk_col_names = set([fields[field_name].column_name for field_name in fields - if fields[field_name].primary_key]) + # pk_col_names = set([fields[field_name].column_name for field_name in fields + # if fields[field_name].primary_key]) - fk_col_names = set([fields[field_name].column_name for field_name in fields - if isinstance(fields[field_name], ForeignKeyField)]) + # fk_col_names = set([fields[field_name].column_name for field_name in fields + # if isinstance(fields[field_name], ForeignKeyField)]) def format_field_str(field): """Add in (PK) OR (FK) suffixes to column names.""" - column_name = field.column_name - suffixes = [] - if column_name in pk_col_names: + column_name = field.column_name + if column_name == '__composite_key__': + column_name = '(' + ', '.join(pk.field_names) + ')' + suffixes.append('PK') # Composite keys get .primary_key == False + + if field.primary_key: suffixes.append('PK') - if column_name in fk_col_names: + if isinstance(field, ForeignKeyField): suffixes.append('FK') suffix = ' (' + ', '.join(suffixes) + ')' if len(suffixes) > 0 else '' - if show_datatypes: + if show_datatypes and field.column_name != '__composite_key__': field_type = field.field_type if field_type in field_type_psql: field_type = field_type_psql[field_type] @@ -85,47 +101,87 @@ def format_field_str(field): return f'- {column_name}{suffix}' html = (f'<' - f'' - f'') + f'') added_col_name = [] fields_html = [] + pk = model._meta.primary_key + if show_pks and pk: + if model._meta.composite_key: + column_name = '(' + ', '.join(pk.field_names) + ')' + else: + column_name = pk.column_name + fields_html.append( + ''.format( + column_name, format_field_str(pk))) + # Add a row for each column in the table. - for field in fields.values(): + if show_columns: + for field in fields.values(): - column_name = field.column_name + if field.primary_key: + continue - # Avoids repeating columns. This can happen if there are multiple - # FKs pointing to the same column. - if column_name in added_col_name: - continue + column_name = field.column_name - fields_html.append( - ''.format( - column_name, format_field_str(field))) + # Avoids repeating columns. This can happen if there are multiple + # FKs pointing to the same column. + if column_name in added_col_name: + continue - added_col_name.append(column_name) + fields_html.append( + ''.format( + column_name, format_field_str(field))) - html += ''.join(fields_html) + added_col_name.append(column_name) + + if len(fields_html) > 0: + html += '' + html += ''.join(fields_html) # Add indexes and unique constraints if show_indices: - indexes = [index for index in model._meta.fields_to_index() - if not isinstance(index._expressions[0], ForeignKeyField)] + if model._meta.database.connected: + indexes = model._meta.database.get_indexes(model._meta.table_name, + schema=model._meta.schema) + else: + indexes = [index._expressions[0] for index in model._meta.fields_to_index() + if not isinstance(index._expressions[0], ForeignKeyField)] if len(indexes) > 0: - html += '' + first = True for index in indexes: + if not isinstance(index, IndexMetadata): + continue + + column_names = index.columns + ilabel = 'INDEX' + + if len(column_names) == 1: + column_name = column_names[0] + if column_name == '': + match = re.match(r'.+q3c_ang2ipix\("*(\w+)"*, "*(\w+)"*\).+', + index.sql) + if match: + column_name = '(' + ', '.join(match.groups()) + ')' + ilabel = 'Q3C' + else: + continue + else: + column_name = '(' + ', '.join(column_names) + ')' - column_name = index._expressions[0].column_name - - if index._unique: + if index.unique: + if pk and column_name == pk.column_name: + continue ilabel = 'UNIQUE' - else: - ilabel = 'INDEX' + + if first: + html += '' + first = False html += f'' @@ -134,9 +190,10 @@ def format_field_str(field): return html -def create_schema_graph(models=None, base=None, schema=None, show_indices=True, - show_datatypes=True, font='Bitstream-Vera Sans', - concentrate=True, relation_options={}, rankdir='TB'): +def create_schema_graph(models=None, base=None, schema=None, show_columns=True, + show_pks=True, show_indices=True, show_datatypes=True, + skip_tables=[], font='Bitstream-Vera Sans', + graph_options={}, relation_options={}): """Creates a graph visualisation from a series of Peewee models. Produces a `pydot `__ graph including the @@ -145,27 +202,35 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, Parameters ---------- - models : `list` + models : list A list of Peewee `models ` to be graphed. - base : `peewee:Model` + base : peewee:Model A base model class. If passed, all the model classes that were created by subclassing from the base model will be used. - schema : `str` + schema : str A schema name. If passed, will be used to limit the list of models or ``base`` subclasses to only the models that match the schema name. - show_indices : `bool` + show_columns : bool + Whether to show the column names. + show_pks : bool + Whether to show the primary key. Supersedes ``show_columns``. + show_indices : bool Whether to show the indices from the table as separate rows. - show_datatypes : `bool` + show_datatypes : bool Whether to show the data type of each column. - font : `str` + skip_tables : list + List of table names to skip. + font : str The name of the font to use. - relation_options : `dict` + graph_options : dict + Options for creating the graph. Any valid Graphviz option. + relation_options : dict Additional parameters to be passed to ``pydot.Edge`` when creating the relationships. Returns ------- - graph : pydot.Dot + graph : `pydot.Dot` A ``pydot.Dot`` object with the graph representation of the schema. Example @@ -178,29 +243,47 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, """ assert models or base, 'either model or base must be passed.' + assert pydot, ('pydot is required for create_schema_graph. ' + 'Try running "pip install sdssdb[all]"') relation_kwargs = {'fontsize': '7.0'} relation_kwargs.update(relation_options) if base and not models: - models = base.__subclasses__() + models = set(base.__subclasses__()) + while True: + old_models = models.copy() + for model in old_models: + models |= set(model.__subclasses__()) + if models == old_models: + break if schema: models = [model for model in models if model._meta.schema == schema] - graph = pydot.Dot(prog='dot', - mode='ipsep', - overlap='ipsep', - sep='0.01', - concentrate=str(concentrate), - rankdir=rankdir) + default_graph_options = dict(program='dot', + rankdir='TB', + sep='0.01', + mode='ipsep', + overlap='ipsep') + default_graph_options.update(graph_options) + + graph = pydot.Dot(prog='dot', **graph_options) for model in models: + if model._meta.table_name in skip_tables: + continue + + if model._meta.database.connected and not model.table_exists(): + continue + graph.add_node( pydot.Node(str(model._meta.table_name), shape='plaintext', label=_render_table_html(model, + show_columns=show_columns, + show_pks=show_pks, show_indices=show_indices, show_datatypes=show_datatypes), fontname=font, @@ -213,8 +296,13 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, field.rel_model not in models): continue - from_col_name = field.column_name + from_col_name = '+ ' + field.column_name + to_col_name = field.rel_field.column_name + if field.rel_field.primary_key: + to_col_name = '' + else: + to_col_name = '+ ' + to_col_name edge = [model._meta.table_name, field.rel_model._meta.table_name] @@ -226,8 +314,8 @@ def create_schema_graph(models=None, base=None, schema=None, show_indices=True, graph_edge = pydot.Edge( dir='both', - headlabel=f'+ {to_col_name}', - taillabel=f'+ {from_col_name}', + headlabel=to_col_name, + taillabel=from_col_name, arrowhead='none', arrowtail='none', # arrowhead=is_inheritance and 'none' or 'odot', diff --git a/readthedocs.yml b/readthedocs.yml index e331a5b3..fa3c76b4 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -9,5 +9,6 @@ python: - method: pip path: . extra_requirements: + - all - dev - docs diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql new file mode 100644 index 00000000..53ad0e36 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE.sql @@ -0,0 +1,603 @@ +/* + +CatWISE Catalog Output Table File Description (Rev. 1.4, May 11, 2019) +Based on WPHot Output Table File Description [pht12] Rev 1.5 Dec. 10, 2018 +and on output from stf 1.7 B90210 run on IRSA table format files with -L option. + +Interface Name: CatWISE catalog table file for Preliminary CatWISE Processing +Type of Interface: disk i/o + +Written By: J.Fowler, P.Eisenhardt + +Read By: CatWISE Software Team + +Description: + +Extension of Version 7 of WPHotpm for multiframe processing of two bands +(unWISE W1 and W2 coadds) with additional table-file columns derived from: +- merging the separate mdex files for ascending and descending scans of a tile +- adding cc_flags from the AllWISE catalog +- adding flags based on unWISE artifact bitmasks +- adding primary source in tile designation flag + +_______________________________________________________________________________________ + +Specification of output variables: + +Table Header information: + +-- # sources tabulated +-- unWISE epoch coadds engaged (ascending and descending for option 1) +-- WISE bands engaged (always bands 1 and 2 only for CatWISE) +-- zero point mag used +-- standard aperture radius in arcsec and pixel units (one line per band) +-- circular aperture radii in arcsec and pixel units (one line per band) +-- Modified Julian Date of the CatWISE coordinate epoch (MJD 56700.00 for Preliminary Catalog) +-- CatWISE processing history (6 lines) + +Tabular data: + +Note that the first column in the width is occupied by a "pipe" or "bar" delimiter ("|") + + Field cols Width Units Type Format Name Description + ---- ---- ----- ------- ---- ------- ------------ ------------------------------------ + 1 1-21 21 -- char a21 source_name source hexagesimal designation + 2 22-46 25 -- char a25 source_id tile name + processing code + wphot index + 3 47-58 12 deg R*8 f12.7 ra right ascension (J2000) + 4 59-70 12 deg R*8 f12.7 dec declination (J2000) + 5 71-79 9 asec R*4 f9.4 sigra uncertainty in ra (arcsec) + 6 80-88 9 asec R*4 f9.4 sigdec uncertainty in dec (arcsec) + 7 89-97 9 asec R*4 f9.4 sigradec uncertainty cross-term (arcsec) + +# the following positions reflect the source xy position in the unWISE full depth coadd + + 8 98-106 9 pix R*4 f9.3 wx x-pixel coordinate + 9 107-115 9 pix R*4 f9.3 wy y-pixel coordinate + +# The next set of columns are aperture/annulus measurements from the unWISE epoch coadds: + + 10 116-125 10 dn R*4 f10.3 w1sky frame sky background value, band-1 + 11 126-133 8 dn R*4 f8.3 w1sigsk frame sky background value uncertainty, band-1 + 12 134-141 8 dn R*4 f8.3 w1conf frame sky confusion based on the UNC images + 13 142-151 10 dn R*4 f10.3 w2sky frame sky background value, band-2 + 14 152-159 8 dn R*4 f8.3 w2sigsk frame sky background value uncertainty, band-2 + 15 160-167 8 dn R*4 f8.3 w2conf frame sky confusion based on the UNC images + +# WPRO epoch coadd measurements: + + 16 168-174 7 asec R*4 f7.2 w1fitr fitting radius, band-1 (clipped at 999.99) + 17 175-181 7 asec R*4 f7.2 w2fitr fitting radius, band-2 (clipped at 999.99) + 18 182-188 7 -- R*4 f7.1 w1snr S/N ratio, band-1 + 19 189-195 7 -- R*4 f7.1 w2snr S/N ratio, band-2 + 20 196-207 12 dn R*4 1pe12.4 w1flux WPRO raw flux, band-1 + 21 208-221 14 dn R*4 1pe14.4 w1sigflux WPRO raw flux uncertainty, band-1 + 22 222-233 12 dn R*4 1pe12.4 w2flux WPRO raw flux, band-2 + 23 234-247 14 dn R*4 1pe14.4 w2sigflux WPRO raw flux uncertainty, band-2 + 24 248-254 7 mag R*4 f7.3 w1mpro WPRO flux in mag units, band-1 + 25 255-264 10 mag R*4 f10.3 w1sigmpro WPRO flux uncertainty in mag units, band-1 + 26 265-275 11 -- R*4 1pe11.3 w1rchi2 WPRO reduced chi^2, band-1 + 27 276-282 7 mag R*4 f7.3 w2mpro WPRO flux in mag units, band-2 + 28 283-292 10 mag R*4 f10.3 w2sigmpro WPRO flux uncertainty in mag units, band-2 + 29 293-303 11 -- R*4 1pe11.3 w2rchi2 WPRO reduced chi^2, band-2 + 30 304-314 11 -- R*4 1pe11.3 rchi2 reduced chi squared, total + + 31 315-318 4 -- int i4 nb number of blend components used in each fit + 32 319-322 4 -- int i4 na number of actively deblended components + 33 323-330 8 -- R*4 f8.5 w1Sat fraction of pixels saturated, band-1 + 34 331-338 8 -- R*4 f8.5 w2Sat fraction of pixels saturated, band-2 + +# full depth coadd measurements: WAPPco, standard aperture w/ aperture correction; +# the standard (aperture corrected) aperture radius is 8.25 arcsec. + + 35 339-345 7 mag R*4 f7.3 w1mag standard aperture mag w/ correction applied + 36 346-352 7 mag R*4 f7.3 w1sigm standard aperture mag uncertainty, band-1 + 37 353-358 6 -- int i6 w1flg standard aperture flag, band-1 + 38 359-366 8 -- R*4 f8.2 w1Cov mean coverage depth, band-1 + 39 367-373 7 mag R*4 f7.3 w2mag standard aperture mag w/ correction applied + 40 374-380 7 mag R*4 f7.3 w2sigm standard aperture mag uncertainty, band-2 + 41 381-386 6 -- int i6 w2flg standard aperture flag, band-2 + 42 387-394 8 -- R*4 f8.2 w2Cov mean coverage depth, band-2 + +# full depth coadd measurements: WAPPco, circular apertures, no aperture correction is applied; +# radii: 5.50 8.25 11.00 13.75 16.50 19.25 22.00 24.75 arcsec + + 43 395-404 10 mag R*4 f10.3 w1mag_1 Aper 1, aperture mag, band-1 + 44 405-414 10 mag R*4 f10.3 w1sigm_1 Aper 1, aperture mag uncertainty, band-1 + 45 415-422 8 -- int i8 w1flg_1 Aper 1, aperture flag, band-1 + 46 423-432 10 mag R*4 f10.3 w2mag_1 Aper 1, aperture mag, band-2 + 47 433-442 10 mag R*4 f10.3 w2sigm_1 Aper 1, aperture mag uncertainty, band-2 + 48 443-450 8 -- int i8 w2flg_1 Aper 1, aperture flag, band-2 + + 49 451-460 10 mag R*4 f10.3 w1mag_2 Aper 2, aperture mag, band-1 + 50 461-470 10 mag R*4 f10.3 w1sigm_2 Aper 2, aperture mag uncertainty, band-1 + 51 471-478 8 -- int i8 w1flg_2 Aper 2, aperture flag, band-1 + 52 479-488 10 mag R*4 f10.3 w2mag_2 Aper 2, aperture mag, band-2 + 53 489-498 10 mag R*4 f10.3 w2sigm_2 Aper 2, aperture mag uncertainty, band-2 + 54 499-506 8 -- int i8 w2flg_2 Aper 2, aperture flag, band-2 + + 55 507-516 10 mag R*4 f10.3 w1mag_3 Aper 3, aperture mag, band-1 + 56 517-526 10 mag R*4 f10.3 w1sigm_3 Aper 3, aperture mag uncertainty, band-1 + 57 527-534 8 -- int i8 w1flg_3 Aper 3, aperture flag, band-1 + 58 535-544 10 mag R*4 f10.3 w2mag_3 Aper 3, aperture mag, band-2 + 59 545-554 10 mag R*4 f10.3 w2sigm_3 Aper 3, aperture mag uncertainty, band-2 + 60 555-562 8 -- int i8 w2flg_3 Aper 3, aperture flag, band-2 + + 61 563-572 10 mag R*4 f10.3 w1mag_4 Aper 4, aperture mag, band-1 + 62 573-582 10 mag R*4 f10.3 w1sigm_4 Aper 4, aperture mag uncertainty, band-1 + 63 583-590 8 -- int i8 w1flg_4 Aper 4, aperture flag, band-1 + 64 591-600 10 mag R*4 f10.3 w2mag_4 Aper 4, aperture mag, band-2 + 65 601-610 10 mag R*4 f10.3 w2sigm_4 Aper 4, aperture mag uncertainty, band-2 + 66 611-618 8 -- int i8 w2flg_4 Aper 4, aperture flag, band-2 + + 67 619-628 10 mag R*4 f10.3 w1mag_5 Aper 5, aperture mag, band-1 + 68 629-638 10 mag R*4 f10.3 w1sigm_5 Aper 5, aperture mag uncertainty, band-1 + 69 639-646 8 -- int i8 w1flg_5 Aper 5, aperture flag, band-1 + 70 647-656 10 mag R*4 f10.3 w2mag_5 Aper 5, aperture mag, band-2 + 71 657-666 10 mag R*4 f10.3 w2sigm_5 Aper 5, aperture mag uncertainty, band-2 + 72 667-674 8 -- int i8 w2flg_5 Aper 5, aperture flag, band-2 + + 73 675-684 10 mag R*4 f10.3 w1mag_6 Aper 6, aperture mag, band-1 + 74 685-694 10 mag R*4 f10.3 w1sigm_6 Aper 6, aperture mag uncertainty, band-1 + 75 695-702 8 -- int i8 w1flg_6 Aper 6, aperture flag, band-1 + 76 703-712 10 mag R*4 f10.3 w2mag_6 Aper 6, aperture mag, band-2 + 77 713-722 10 mag R*4 f10.3 w2sigm_6 Aper 6, aperture mag uncertainty, band-2 + 78 723-730 8 -- int i8 w2flg_6 Aper 6, aperture flag, band-2 + + 79 731-740 10 mag R*4 f10.3 w1mag_7 Aper 7, aperture mag, band-1 + 80 741-750 10 mag R*4 f10.3 w1sigm_7 Aper 7, aperture mag uncertainty, band-1 + 81 751-758 8 -- int i8 w1flg_7 Aper 7, aperture flag, band-1 + 82 759-768 10 mag R*4 f10.3 w2mag_7 Aper 7, aperture mag, band-2 + 83 769-778 10 mag R*4 f10.3 w2sigm_7 Aper 7, aperture mag uncertainty, band-2 + 84 779-786 8 -- int i8 w2flg_7 Aper 7, aperture flag, band-2 + + 85 787-796 10 mag R*4 f10.3 w1mag_8 Aper 8, aperture mag, band-1 + 86 797-806 10 mag R*4 f10.3 w1sigm_8 Aper 8, aperture mag uncertainty, band-1 + 87 807-814 8 -- int i8 w1flg_8 Aper 8, aperture flag, band-1 + 88 815-824 10 mag R*4 f10.3 w2mag_8 Aper 8, aperture mag, band-2 + 89 825-834 10 mag R*4 f10.3 w2sigm_8 Aper 8, aperture mag uncertainty, band-2 + 90 835-842 8 -- int i8 w2flg_8 Aper 8, aperture flag, band-2 + +# the following are "N of M" counters for WPRO measurements +# w?M - The number of individual epochs for band ? that are +# available to make a profile-fit measurement. +# w?NM - The number of individual epochs for band ? on which +# WPRO extracted a flux measurement that has snr>3. +# w?mLQ - variability indicator mLogQ for the flux array + + 91 843-849 7 -- int i7 w1NM WPRO, N (of M), band-1 + 92 850-855 6 -- int i6 w1M WPRO, M , band-1 + 93 856-863 8 mag R*4 f8.3 w1magP WPRO repeatability mag; band-1 + 94 864-871 8 mag R*4 f8.3 w1sigP1 WPRO mag population sigma; band-1 + 95 872-879 8 mag R*4 f8.3 w1sigP2 WPRO mag uncertainty of the mean; band-1 + 96 880-891 12 -- R*4 f12.5 w1k Stetson k index for variability; band-1 + 97 892-897 6 -- int i6 w1Ndf No. degrees of freedom in var chi-square; band-1 + 98 898-903 6 -- R*4 f6.2 w1mLQ -log(Q), Q = 1-P(chi-square); band-1 + 99 904-921 18 -- R*8 f18.8 w1mJDmin minimum mJD; band-1 + 100 922-939 18 -- R*8 f18.8 w1mJDmax maximum mJD; band-1 + 101 940-957 18 -- R*8 f18.8 w1mJDmean mean mJD; band-1 + + 102 958-964 7 -- int i7 w2NM WPRO, N (of M), band-2 + 103 965-970 6 -- int i6 w2M WPRO, M , band-2 + 104 971-978 8 mag R*4 f8.3 w2magP WPRO repeatability mag; band-2 + 105 979-986 8 mag R*4 f8.3 w2sigP1 WPRO mag population sigma; band-2 + 106 987-994 8 mag R*4 f8.3 w2sigP2 WPRO mag uncertainty of the mean; band-2 + 107 995-1006 12 -- R*4 f12.5 w2k Stetson k index for variability; band-2 + 108 1007-1012 6 -- int i6 w2Ndf No. degrees of freedom in var chi-square; band-2 + 109 1013-1018 6 -- R*4 f6.2 w2mLQ -log(Q), Q = 1-P(chi-square); band-2 + 110 1019-1036 18 -- R*8 f18.8 w2mJDmin minimum mJD; band-2 + 111 1037-1054 18 -- R*8 f18.8 w2mJDmax maximum mJD; band-2 + 112 1055-1072 18 -- R*8 f18.8 w2mJDmean mean mJD; band-2 + +# band-to-band sample correlation coefficients and their probabilities for adjacent bands + + 113 1073-1078 6 % int i4 rho12 W1W2 correlation coefficient + 114 1079-1084 6 -- int i4 q12 -log10(1-P(rho12)) given no real correlation + + 115 1085-1091 7 -- int i7 nIters number of chi-square-minimization iterations + 116 1092-1098 7 -- int i7 nSteps number of steps in all iterations + +# MDET-Related Parameters +# p1 and p2 are clipped to the range (-99.99999, +99.99999) if necessary + + 117 1099-1105 7 -- int i7 mdetID source ID in mdet list + 118 1106-1115 10 asec R*4 f10.5 p1 P vector component 1 + 119 1116-1125 10 asec R*4 f10.5 p2 P vector component 2 + +# Motion Fit Solution Parameters +# ra_pm and dec_pm are in the equinox J2000 frame at the CatWISE epoch +# (MJD 56700.00 for the Preliminary Catalog) based on PMRA and PMDec + + 120 1126-1138 13 MJD R*8 f13.6 MeanObsMJD mean observation epoch + 121 1139-1150 12 deg R*8 f12.7 ra_pm right ascension (J2000) + 122 1151-1162 12 deg R*8 f12.7 dec_pm declination (J2000) + 123 1163-1171 9 asec R*4 f9.4 sigra_pm uncertainty in ra_pm + 124 1172-1181 10 asec R*4 f10.4 sigdec_pm uncertainty in dec_pm + 125 1182-1193 12 asec R*4 f12.4 sigradec_pm uncertainty cross-term + 126 1194-1203 10 asec/yr R*4 f10.4 PMRA motion in ra + 127 1204-1213 10 asec/yr R*4 f10.4 PMDec proper motion in dec + 128 1214-1222 9 asec/yr R*4 f9.4 sigPMRA uncertainty in PMRA + 129 1223-1231 9 asec/yr R*4 f9.4 sigPMDec uncertainty in PMDec + + 130 1232-1240 9 -- R*4 f9.1 w1snr_pm flux S/N ratio, band-1 + 131 1241-1249 9 -- R*4 f9.1 w2snr_pm flux S/N ratio, band-2 + 132 1250-1261 12 dn R*4 1pe12.4 w1flux_pm WPRO raw flux, band-1 + 133 1262-1275 14 dn R*4 1pe14.4 w1sigflux_pm WPRO raw flux uncertainty, band-1 + 134 1276-1287 12 dn R*4 1pe12.4 w2flux_pm WPRO raw flux, band-2 + 135 1288-1301 14 dn R*4 1pe14.4 w2sigflux_pm fit WPRO raw flux uncertainty, band-2 + 136 1302-1311 10 mag R*4 f10.3 w1mpro_pm WPRO flux in mag units, band-1 + 137 1312-1324 13 mag R*4 f13.3 w1sigmpro_pm WPRO flux uncertainty in mag units, band-1 + 138 1325-1335 11 -- R*4 1pe11.3 w1rchi2_pm WPRO reduced chi^2, band-1 + 139 1336-1345 10 mag R*4 f10.3 w2mpro_pm WPRO flux in mag units, band-2 + 140 1346-1358 13 mag R*4 f13.3 w2sigmpro_pm WPRO flux uncertainty in mag units, band-2 + 141 1359-1369 11 -- R*4 1pe11.3 w2rchi2_pm WPRO reduced chi^2, band-2 + 142 1370-1380 11 -- R*4 1pe11.3 rchi2_pm reduced chi squared, total + +# pmcode provides information that may correlate with the quality of the PM solution. +# The format is ABCCC, where A is the number of components in the passive blend group +# (including the primary) before any are removed or added, +# B is "Y" or "N" to indicate "Yes" or "No" that a secondary blend component replaced the primary, +# CCC is the distance in hundredths of an arcsec between the +# PM position solution for the mean observation epoch and the stationary solution + + 143 1381-1387 7 -- char A5 pmcode quality of the PM solution + + 144 1388-1397 10 -- int i10 nIters_pm number of chi-square-minimization iterations + 145 1398-1407 10 -- int i10 nSteps_pm number of steps in all iterations + +# Parameters Derived From Ascending-Descending Scan Differences + + 146 1408-1416 9 asec R*4 f9.3 dist radial distance between apparitions + 147 1417-1423 7 mag R*4 f7.3 dw1mag w1mpro difference + 148 1424-1430 7 -- R*4 f7.3 rch2w1 chi-square for dw1mag (1 DF) + 149 1431-1437 7 mag R*4 f7.3 dw2mag w2mpro difference + 150 1438-1444 7 -- R*4 f7.3 rch2w2 chi-square for dw2mag (1 DF) + 151 1445-1455 11 deg R*8 f11.6 elon_avg averaged ecliptic longitude + 152 1456-1466 11 asec R*4 f11.3 elonSig one-sigma uncertainty in elon + 153 1467-1477 11 deg R*8 f11.6 elat_avg averaged ecliptic latitude + 154 1478-1487 10 asec R*4 f10.3 elatSig one-sigma uncertainty in elat + 155 1488-1498 11 asec R*4 f11.3 Delon desc-asce ecliptic longitude + 156 1499-1509 11 asec R*4 f11.3 DelonSig one-sigma uncertainty in Delon + 157 1510-1520 11 asec R*4 f11.3 Delat desc-asce ecliptic latitude + 158 1521-1530 10 asec R*4 f10.3 DelatSig one-sigma uncertainty in Delat + 159 1531-1541 11 -- R*4 f11.3 DelonSNR |Delon|/DelonSig + 160 1542-1552 11 -- R*4 f11.3 DelatSNR |Delat|/DelatSig + 161 1553-1562 10 -- R*4 1pE10.3 chi2pmra chi-square for PMRA difference (1 DF) + 162 1563-1572 10 -- R*4 1pE10.3 chi2pmdec chi-square for PMRA difference (1 DF) + 163 1573-1575 3 -- int i3 ka astrometry usage code + 164 1576-1578 3 -- int i3 k1 W1 photometry usage code + 165 1579-1581 3 -- int i3 k2 W2 photometry usage code + 166 1582-1584 3 -- int i3 km proper motion usage code + 167 1585-1595 11 asec R*4 f11.3 par_pm parallax from PM desc-asce elon + 168 1596-1606 11 asec R*4 f11.3 par_pmSig one-sigma uncertainty in par_pm + 169 1607-1617 11 asec R*4 f11.3 par_stat parallax estimate from stationary solution + 170 1618-1628 11 asec R*4 f11.3 par_sigma one-sigma uncertainty in par_stat + +# Parameters Derived from AllWISE Artifact Flags + + 171 1629-1641 13 asec R*4 f13.6 dist_x distance between CatWISE and AllWISE source + 172 1642-1657 16 -- char a16 cc_flags worst case 4 character cc_flag from AllWISE + 173 1658-1670 13 -- int i13 w1cc_map worst case w1cc_map from AllWISE + 174 1671-1690 20 -- char a20 w1cc_map_str worst case w1cc_map_str from AllWISE + 175 1691-1703 13 -- int i13 w2cc_map worst case w2cc_map from AllWISE + 176 1704-1723 20 -- char a20 w2cc_map_str worst case w2cc_map_str from AllWISE + 177 1724-1728 5 -- int i5 n_aw number of AllWISE matches within 2.75 asec + +# Parameters Derived from unWISE Artifact Bitmasks + + 178 1729-1737 9 -- char a9 ab_flags Two character (W1 W2) artifact flag + 179 1738-1746 9 -- int i9 w1ab_map w1 artifact code value + 180 1747-1759 13 -- char a13 w1ab_map_str w1 artifact string + 181 1760-1768 9 -- int i9 w2ab_map w2 artifact code value + 182 1769-1781 13 -- char a13 w2ab_map_str w2 artifact string + +# Indexed Coordinates Derived from Stationary ra and dec + + 183 1782-1793 12 deg R*8 f12.6 glon galactic longitude + 184 1794-1805 12 deg R*8 f12.6 glat galactic latitude + 185 1806-1817 12 deg R*8 f12.7 elon ecliptic longitude + 186 1818-1829 12 deg R*8 f12.7 elat ecliptic latitude + +# The reject table also includes + + 187 1830-1835 6 -- int i6 P Flag to indicate if source is primary in tile + + +NOTES +1.) Delon may have a bias correction for a systematic error induced by residual + PSF errors that manifest themselves in ascending-descending differences; see + the "-pb" command line option of the mrgad program. +2.) Delon is defined as descending position minus ascending position in order to + have the proper sign for parallax (~Delon/2). +3.) the usage codes ka, k1, k2, and km have values of 0 - 3 meaning + 0: neither the ascending nor the descending scan provided a solution + 1: only the ascending scan provided a solution + 2: only the descending scan provided a solution + 3: both scans provided solutions which were combined in the relevant way +4.) Delon is the straightforward difference in ecliptic longitude, ignoring the + different effective observation epochs of ascending and descending scans; + par_pm is computed from the motion-solution positions, which are translated + by WPHotpmc to the standard epoch (MJD0), so except for estimation errors, + par_pm is the parallax; the par_stat column is computed by using the + motion estimate to move the ascending stationary-solution position from the + ascending effective observation epoch to that of the descending solution, then + dividing the ecliptic longitude difference by 2. +5.) Delon will be null unless ka = 3; par_pm will be null unless km = 3; + par_stat will be null unless ka = 3 AND km > 0 AND all W?mJDmin/max/mean values + are non-null in both ascending and descending mdex files. + +____________________________________________________________________________________ + +N of M Statistics + +M == number of flux measurements for a given source +N == number of M sources that have SNR >= 3 + +____________________________________________________________________________________ + +Photometry Flags: + +Standard aperture measurement quality flag. This flag +indicates if one or more image pixels in the measurement aperture for this +band is confused with nearby objects, is contaminated by saturated +or otherwise ususable pixels, or is an upper limit. The flag value +is the integer sum of any of following values which correspond to +different conditions. + +value Condition +----- ------------------------------------------------------ + 0 nominal -- no contamination + 1 source confusion -- another source falls within the measurement aperture + 2 bad or fatal pixels: presence of bad pixels in the measurement aperture + (bit 2 or 18 set) + 4 non-zero bit flag tripped (other than 2 or 18) + 8 corruption -- all pixels are flagged as unusable, or the aperture flux is + negative; in the former case, the aperture magnitude is NULL; in the + latter case, the aperture magnitude is a 95% confidence upper limit + 16 saturation -- here are one or more saturated pixels in the measurement + aperture + 32 upper limit -- the magnitude is a 95% confidence upper limit + +combinations: + + 3 source confusion + bad pixels + 5 source confusion + non-zero bit flag + 6 bad pixels + non-zero bit flag + 7 source confusion + bad pixels + non-zero bit flag + 9 source confusion + corruption + 10 bad pixels + corruption + 11 source confusion + bad pixels + corruption + 12 non-zero bit flag + corruption + 13 source confusion + non-zero bit flag + corruption + 14 bad pixels + non-zero bit flag + corruption + 15 source confusion + bad pixels + non-zero bit flag + corruption + 17 source confusion + saturation + 18 bad pixels + saturation + 19 source confusion + bad pixels + saturation + 20 non-zero bit flag + saturation + 21 source confusion + non-zero bit flag + saturation + 22 bad pixels + non-zero bit flag + saturation + 23 source confusion + bad pixels + non-zero bit flag + saturation + 24 corruption + saturation + 25 source confusion + corruption + saturation + 26 bad pixels + corruption + saturation + 27 source confusion + bad pixels + corruption + saturation + 28 non-zero bit flag + corruption + saturation + 29 source confusion + non-zero bit flag + corruption + saturation + 30 bad pixels + non-zero bit flag + corruption + saturation + 31 source confusion + bad pixels + non-zero bit flag + corruption + saturation + +______________________________________________________________________________________________ + +Note about upper limits: + +threshold = 2 * RMS +if ( f < threshold) then + if (f < 0) then + upper_limit = threshold + else + upper_limit = f + threshold + +where f is the flux of the source, RMS is the source flux uncertainty + +The upper limit is reported in the WPRO and standard aperture MAG columns; +the reported uncertainty is set to 9.99 mag. + +______________________________________________________________________________________________ + +Note: Null values are now indicated by the character string "null" + +*/ + + +CREATE TABLE catalogdb.catwise ( + source_name VARCHAR(21), + source_id VARCHAR(25), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + sigra REAL, + sigdec REAL, + sigradec REAL, + wx REAL, + wy REAL, + w1sky REAL, + w1sigsk REAL, + w1conf REAL, + w2sky REAL, + w2sigsk REAL, + w2conf REAL, + w1fitr REAL, + w2fitr REAL, + w1snr REAL, + w2snr REAL, + w1flux DOUBLE PRECISION, + w1sigflux DOUBLE PRECISION, + w2flux DOUBLE PRECISION, + w2sigflux DOUBLE PRECISION, + w1mpro REAL, + w1sigmpro DOUBLE PRECISION, + w1rchi2 DOUBLE PRECISION, + w2mpro REAL, + w2sigmpro DOUBLE PRECISION, + w2rchi2 DOUBLE PRECISION, + rchi2 DOUBLE PRECISION, + nb SMALLINT, + na SMALLINT, + w1Sat DOUBLE PRECISION, + w2Sat DOUBLE PRECISION, + w1mag REAL, + w1sigm REAL, + w1flg INTEGER, + w1Cov REAL, + w2mag REAL, + w2sigm REAL, + w2flg INTEGER, + w2Cov REAL, + w1mag_1 REAL, + w1sigm_1 REAL, + w1flg_1 BIGINT, + w2mag_1 REAL, + w2sigm_1 REAL, + w2flg_1 BIGINT, + w1mag_2 REAL, + w1sigm_2 REAL, + w1flg_2 BIGINT, + w2mag_2 REAL, + w2sigm_2 REAL, + w2flg_2 BIGINT, + w1mag_3 REAL, + w1sigm_3 REAL, + w1flg_3 BIGINT, + w2mag_3 REAL, + w2sigm_3 REAL, + w2flg_3 BIGINT, + w1mag_4 REAL, + w1sigm_4 REAL, + w1flg_4 BIGINT, + w2mag_4 REAL, + w2sigm_4 REAL, + w2flg_4 BIGINT, + w1mag_5 REAL, + w1sigm_5 REAL, + w1flg_5 BIGINT, + w2mag_5 REAL, + w2sigm_5 REAL, + w2flg_5 BIGINT, + w1mag_6 REAL, + w1sigm_6 REAL, + w1flg_6 BIGINT, + w2mag_6 REAL, + w2sigm_6 REAL, + w2flg_6 BIGINT, + w1mag_7 REAL, + w1sigm_7 REAL, + w1flg_7 BIGINT, + w2mag_7 REAL, + w2sigm_7 REAL, + w2flg_7 BIGINT, + w1mag_8 REAL, + w1sigm_8 REAL, + w1flg_8 BIGINT, + w2mag_8 REAL, + w2sigm_8 REAL, + w2flg_8 BIGINT, + w1NM BIGINT, + w1M INTEGER, + w1magP REAL, + w1sigP1 REAL, + w1sigP2 REAL, + w1k REAL, + w1Ndf INTEGER, + w1mLQ REAL, + w1mJDmin DOUBLE PRECISION, + w1mJDmax DOUBLE PRECISION, + w1mJDmean DOUBLE PRECISION, + w2NM BIGINT, + w2M INTEGER, + w2magP REAL, + w2sigP1 REAL, + w2sigP2 REAL, + w2k REAL, + w2Ndf INTEGER, + w2mLQ REAL, + w2mJDmin DOUBLE PRECISION, + w2mJDmax DOUBLE PRECISION, + w2mJDmean DOUBLE PRECISION, + rho12 SMALLINT, + q12 SMALLINT, + nIters BIGINT, + nSteps BIGINT, + mdetID BIGINT, + p1 DOUBLE PRECISION, + p2 DOUBLE PRECISION, + MeanObsMJD DOUBLE PRECISION, + ra_pm DOUBLE PRECISION, + dec_pm DOUBLE PRECISION, + sigra_pm REAL, + sigdec_pm REAL, + sigradec_pm REAL, + PMRA REAL, + PMDec REAL, + sigPMRA REAL, + sigPMDec REAL, + w1snr_pm REAL, + w2snr_pm REAL, + w1flux_pm DOUBLE PRECISION, + w1sigflux_pm DOUBLE PRECISION, + w2flux_pm DOUBLE PRECISION, + w2sigflux_pm DOUBLE PRECISION, + w1mpro_pm REAL, + w1sigmpro_pm REAL, + w1rchi2_pm DOUBLE PRECISION, + w2mpro_pm REAL, + w2sigmpro_pm REAL, + w2rchi2_pm DOUBLE PRECISION, + rchi2_pm DOUBLE PRECISION, + pmcode VARCHAR(5), + nIters_pm BIGINT, + nSteps_pm BIGINT, + dist REAL, + dw1mag REAL, + rch2w1 REAL, + dw2mag REAL, + rch2w2 REAL, + elon_avg DOUBLE PRECISION, + elonSig REAL, + elat_avg DOUBLE PRECISION, + elatSig REAL, + Delon REAL, + DelonSig REAL, + Delat REAL, + DelatSig REAL, + DelonSNR REAL, + DelatSNR REAL, + chi2pmra DOUBLE PRECISION, + chi2pmdec DOUBLE PRECISION, + ka INTEGER, + k1 INTEGER, + k2 INTEGER, + km INTEGER, + par_pm REAL, + par_pmSig REAL, + par_stat REAL, + par_sigma REAL, + dist_x DOUBLE PRECISION, + cc_flags VARCHAR(16), + w1cc_map BIGINT, + w1cc_map_str VARCHAR(20), + w2cc_map BIGINT, + w2cc_map_str VARCHAR(20), + n_aw INTEGER, + ab_flags VARCHAR(9), + w1ab_map BIGINT, + w1ab_map_str VARCHAR(13), + w2ab_map BIGINT, + w2ab_map_str VARCHAR(13), + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + elon DOUBLE PRECISION, + elat DOUBLE PRECISION +); + + +CREATE TABLE catalogdb.catwise_reject AS (SELECT * FROM catalogdb.catwise) WITH NO DATA; +ALTER TABLE catalogdb.catwise_reject ADD COLUMN p BOOLEAN; diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_download b/schema/sdss5db/catalogdb/CatWISE/CatWISE_download new file mode 100644 index 00000000..7c66d07b --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_download @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +wget -r -nH --no-parent --reject="index.html*" -P ./ https://portal.nersc.gov/project/cosmo/data/CatWISE/ diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql new file mode 100644 index 00000000..78919be4 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_indexes.sql @@ -0,0 +1,16 @@ + +-- PKs + +ALTER TABLE catalogdb.catwise ADD PRIMARY KEY (source_id); +ALTER TABLE catalogdb.catwise_reject ADD PRIMARY KEY (source_id); + + +-- Indexes + +CREATE INDEX CONCURRENTLY ON catalogdb.catwise (q3c_ang2ipix(ra, dec)); +CLUSTER catwise_q3c_ang2ipix_idx ON catalogdb.catwise; +ANALYZE catalogdb.catwise; + +CREATE INDEX CONCURRENTLY ON catalogdb.catwise_reject (q3c_ang2ipix(ra, dec)); +CLUSTER catwise_reject_q3c_ang2ipix_idx ON catalogdb.catwise_reject; +ANALYZE catalogdb.catwise_reject; diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load new file mode 100755 index 00000000..d4373a30 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +# encoding: utf-8 + + +path=/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/CatWISE + +for dir in $path/prelim/*/ ; do + echo $dir + ls $dir/*.tbl.gz | parallel -j15 "python $SDSSDB_DIR/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py {}" + ls $dir/*_cat_*.tbl.gz.csv | parallel -j16 "psql -U sdss sdss5db -c \"\copy catalogdb.catwise FROM '{}' WITH CSV HEADER NULL '\N';\"" + ls $dir/*_rej_*.tbl.gz.csv | parallel -j16 "psql -U sdss sdss5db -c \"\copy catalogdb.catwise_reject FROM '{}' WITH CSV HEADER NULL '\N';\"" +done diff --git a/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py new file mode 100644 index 00000000..d173e496 --- /dev/null +++ b/schema/sdss5db/catalogdb/CatWISE/CatWISE_load_helper.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-23 +# @Filename: CatWISE_load_helper.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import sys + +import astropy.table +from astropy.io import ascii + + +def main(): + + file_ = sys.argv[1] + dest = file_ + '.csv' + + table = astropy.table.Table.read(file_, format='ascii.ipac') + table.meta = {} + table.write(dest, format='csv', fill_values=[(ascii.masked, '\\N')], + overwrite=True) + + return + + +if __name__ == '__main__': + main() diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql new file mode 100644 index 00000000..c2eb2915 --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE.sql @@ -0,0 +1,183 @@ +/* + +GLIMPSE I+II+3D (Archive) + +https://irsa.ipac.caltech.edu/data/SPITZER/GLIMPSE/overview.html + +\fixlen = T +\ Default Data Dictionary +\ +\ +\alias = none +\longitude = ra +\latitude = dec +\primary = cntr +\spt_ind = spt_ind +\x = x +\y = y +\z = z +\ +\ +| cntr| name| original_name| description| units| intype| format| dbtype|nulls|tableflg| groupid| irsadef| sel|indx| notes| +| integer| char| char| char| char| char| char| char| char| integer| integer| char |char|char| char| + 1 designation designation GLIMPSEI source designation name char character(26) n 2 1 y y n + 2 tmass_designation tmass_designation Source name (designation) from 2MASS All-Sky point source catalog (PSC) char character(16) y 2 2 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#designation + 3 tmass_cntr tmass_cntr Unique id number (counter) for the 2MASS source from the 2MASS All-Sky PSC int integer y 2 3 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#pts_key + 4 l l Galactic longitude deg double float n 2 4 y y n + 5 b b Galactic latitude deg double float n 2 5 y y n + 6 dl dl error in Galactic longitude arcsec double float n 2 6 y y n + 7 db db error in Galactic latitude arcsec double float n 2 7 y y n + 8 ra ra Right ascension J2000 deg double float n 2 8 y y n + 9 dec dec Declination J2000 deg double float n 2 9 y y n + 10 dra dra error in Right ascension arcsec double float n 2 10 y y n + 11 ddec ddec error in Declination arcsec double float n 2 11 y y n + 12 csf csf close source flag int integer n 2 12 y y n + 13 mag_J mag_J 2MASS All-Sky PSC J Band magnitude mag real smallfloat y 2 13 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_m + 14 dJ_m dJ_m 2MASS All-Sky PSC J Band 1 sigma error mag real smallfloat y 2 13 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_msigcom + 15 mag_H mag_H 2MASS All-Sky PSC H Band magnitude mag real smallfloat y 2 15 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_m + 16 dH_m dH_m 2MASS All-Sky PSC H Band 1 sigma error mag real smallfloat y 2 15 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_msigcom + 17 mag_Ks mag_Ks 2MASS All-Sky PSC Ks Band magnitude mag real smallfloat y 2 17 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_m + 18 dKs_m dKs_m 2MASS All-Sky PSC Ks Band 1 sigma error mag real smallfloat y 2 17 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_msigcom + 19 mag3_6 mag3_6 3.6um IRAC (Band 1) magnitude mag real smallfloat y 2 19 y y n + 20 d3_6m d3_6m 3.6um IRAC (Band 1) 1 sigma error mag real smallfloat y 2 19 y y n + 21 mag4_5 mag4_5 4.5um IRAC (Band 2) magnitude mag real smallfloat y 2 21 y y n + 22 d4_5m d4_5m 4.5um IRAC (Band 2) 1 sigma error mag real smallfloat y 2 21 y y n + 23 mag5_8 mag5_8 5.8um IRAC (Band 3) magnitude mag real smallfloat y 2 23 y y n + 24 d5_8m d5_8m 5.8um IRAC (Band 3) 1 sigma error mag real smallfloat y 2 23 y y n + 25 mag8_0 mag8_0 8.0um IRAC (Band 4) magnitude mag real smallfloat y 2 25 y y n + 26 d8_0m d8_0m 8.0um IRAC (Band 4) 1 sigma error mag real smallfloat y 2 25 y y n + 27 f_J f_J 2MASS All-Sky PSC J Band flux mJy real smallfloat y 2 27 y y n + 28 df_J df_J 2MASS All-Sky PSC J Band 1 sigma error mJy real smallfloat y 2 27 y y n + 29 f_H f_H 2MASS All-Sky PSC H Band flux mJy real smallfloat y 2 29 y y n + 30 df_H df_H 2MASS All-Sky PSC H Band 1 sigma error mJy real smallfloat y 2 29 y y n + 31 f_Ks f_Ks 2MASS All-Sky PSC Ks Band flux mJy real smallfloat y 2 31 y y n + 32 df_Ks df_Ks 2MASS All-Sky PSC Ks Band 1 sigma error mJy real smallfloat y 2 31 y y n + 33 f3_6 f3_6 3.6um IRAC (Band 1) flux mJy real smallfloat y 2 33 y y n + 34 df3_6 df3_6 3.6um IRAC (Band 1) 1 sigma error mJy real smallfloat y 2 33 y y n + 35 f4_5 f4_5 4.5um IRAC (Band 2) flux mJy real smallfloat y 2 35 y y n + 36 df4_5 df4_5 4.5um IRAC (Band 2) 1 sigma error mJy real smallfloat y 2 35 y y n + 37 f5_8 f5_8 5.8um IRAC (Band 3) flux mJy real smallfloat y 2 37 y y n + 38 df5_8 df5_8 5.8um IRAC (Band 3) 1 sigma error mJy real smallfloat y 2 37 y y n + 39 f8_0 f8_0 8.0um IRAC (Band 4) flux mJy real smallfloat y 2 39 y y n + 40 df8_0 df8_0 8.0um IRAC (Band 4) 1 sigma error mJy real smallfloat y 2 39 y y n + 41 rms_f3_6 rms_f3_6 rms deviation of the individual detections from the final flux for 3.6um IRAC mJy real smallfloat y 2 41 y y n + 42 rms_f4_5 rms_f4_5 rms deviation of the individual detections from the final flux for 4.5um IRAC mJy real smallfloat y 2 42 y y n + 43 rms_f5_8 rms_f5_8 rms deviation of the individual detections from the final flux for 5.8um IRAC mJy real smallfloat y 2 43 y y n + 44 rms_f8_0 rms_f8_0 rms deviation of the individual detections from the final flux for 8.0um IRAC mJy real smallfloat y 2 44 y y n + 45 sky_3_6 sky_3_6 Local sky background for 3.6um IRAC (Band 1) MJy/sr real smallfloat y 2 45 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 46 sky_4_5 sky_4_5 Local sky background for 4.5um IRAC (Band 2) MJy/sr real smallfloat y 2 46 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 47 sky_5_8 sky_5_8 Local sky background for 5.8um IRAC (Band 3) MJy/sr real smallfloat y 2 47 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 48 sky_8_0 sky_8_0 Local sky background for 8.0um IRAC (Band 4) MJy/sr real smallfloat y 2 48 y y n see Appendix B of http://www.astro.wisc.edu/glimpse/glimpse_photometry_v1.0.pdf + 49 sn_J sn_J 2MASS All-Sky PSC J Band Signal/Noise real smallfloat y 2 49 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#j_snr + 50 sn_H sn_H 2MASS All-Sky PSC H Band Signal/Noise real smallfloat y 2 50 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#h_snr + 51 sn_Ks sn_Ks 2MASS All-Sky PSC Ks Band Signal/Noise real smallfloat y 2 51 y y n http://www.ipac.caltech.edu/2mass/releases/allsky/doc/sec2_2a.html#k_snr + 52 sn_3_6 sn_3_6 3.6um IRAC (Band 1) Signal/Noise real smallfloat y 2 52 y y n + 53 sn_4_5 sn_4_5 4.5um IRAC (Band 2) Signal/Noise real smallfloat y 2 53 y y n + 54 sn_5_8 sn_5_8 5.8um IRAC (Band 3) Signal/Noise real smallfloat y 2 54 y y n + 55 sn_8_0 sn_8_0 8.0um IRAC (Band 4) Signal/Noise real smallfloat y 2 55 y y n + 56 dens_3_6 dens_3_6 Local source density for 3.6um IRAC (Band 1) #/sqamin real smallfloat y 2 56 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 57 dens_4_5 dens_4_5 Local source density for 4.5um IRAC (Band 2) #/sqamin real smallfloat y 2 57 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 58 dens_5_8 dens_5_8 Local source density for 5.8um IRAC (Band 3) #/sqamin real smallfloat y 2 58 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 59 dens_8_0 dens_8_0 Local source density for 8.0um IRAC (Band 4) #/sqamin real smallfloat y 2 59 y y n see section 5.1 of http://www.astro.wisc.edu/glimpse/glimpse1_dataprod_v2.0.pdf + 60 m3_6 m3_6 Number of detections for 3.6um IRAC (Band 1) int integer n 2 60 y y n + 61 m4_5 m4_5 Number of detections for 4.5um IRAC (Band 2) int integer n 2 61 y y n + 62 m5_8 m5_8 Number of detections for 5.8um IRAC (Band 3) int integer n 2 62 y y n + 63 m8_0 m8_0 Number of detections for 8.0um IRAC (Band 4) int integer n 2 63 y y n + 64 n3_6 n3_6 Number of possible detections for 3.6um IRAC (Band 1) int integer n 2 64 y y n + 65 n4_5 n4_5 Number of possible detections for 4.5um IRAC (Band 2) int integer n 2 65 y y n + 66 n5_8 n5_8 Number of possible detections for 5.8um IRAC (Band 3) int integer n 2 66 y y n + 67 n8_0 n8_0 Number of possible detections for 8.0um IRAC (Band 4) int integer n 2 67 y y n + 68 sqf_J sqf_J 2MASS All-Sky PSC J Band Source Quality Flag int integer y 2 68 y y n + 69 sqf_H sqf_H 2MASS All-Sky PSC H Band Source Quality Flag int integer y 2 69 y y n + 70 sqf_Ks sqf_Ks 2MASS All-Sky PSC Ks Band Source Quality Flag int integer y 2 70 y y n + 71 sqf_3_6 sqf_3_6 Source Quality Flag for 3.6um IRAC (Band 1) int integer y 2 71 y y n + 72 sqf_4_5 sqf_4_5 Source Quality Flag for 4.5um IRAC (Band 2) int integer y 2 72 y y n + 73 sqf_5_8 sqf_5_8 Source Quality Flag for 5.8um IRAC (Band 3) int integer y 2 73 y y n + 74 sqf_8_0 sqf_8_0 Source Quality Flag for 8.0um IRAC (Band 4) int integer y 2 74 y y n + 75 mf3_6 mf3_6 Flux calculation method flag 3.6um IRAC (Band 1) int integer y 2 75 y y n + 76 mf4_5 mf4_5 Flux calculation method flag 4.5um IRAC (Band 2) int integer y 2 76 y y n + 77 mf5_8 mf5_8 Flux calculation method flag 5.8um IRAC (Band 3) int integer y 2 77 y y n + 78 mf8_0 mf8_0 Flux calculation method flag 8.0um IRAC (Band 4) int integer y 2 78 y y n + +*/ + +CREATE TABLE catalogdb.glimpse ( + designation TEXT, + tmass_designation VARCHAR(18), + tmass_cntr INTEGER, + l DOUBLE PRECISION, + b DOUBLE PRECISION, + dl DOUBLE PRECISION, + db DOUBLE PRECISION, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + dra DOUBLE PRECISION, + ddec DOUBLE PRECISION, + csf INTEGER, + mag_J REAL, + dJ_m REAL, + mag_H REAL, + dH_m REAL, + mag_Ks REAL, + dKs_m REAL, + mag3_6 REAL, + d3_6m REAL, + mag4_5 REAL, + d4_5m REAL, + mag5_8 REAL, + d5_8m REAL, + mag8_0 REAL, + d8_0m REAL, + f_J REAL, + df_J REAL, + f_H REAL, + df_H REAL, + f_Ks REAL, + df_Ks REAL, + f3_6 REAL, + df3_6 REAL, + f4_5 REAL, + df4_5 REAL, + f5_8 REAL, + df5_8 REAL, + f8_0 REAL, + df8_0 REAL, + rms_f3_6 REAL, + rms_f4_5 REAL, + rms_f5_8 REAL, + rms_f8_0 REAL, + sky_3_6 REAL, + sky_4_5 REAL, + sky_5_8 REAL, + sky_8_0 REAL, + sn_J REAL, + sn_H REAL, + sn_Ks REAL, + sn_3_6 REAL, + sn_4_5 REAL, + sn_5_8 REAL, + sn_8_0 REAL, + dens_3_6 REAL, + dens_4_5 REAL, + dens_5_8 REAL, + dens_8_0 REAL, + m3_6 INTEGER, + m4_5 INTEGER, + m5_8 INTEGER, + m8_0 INTEGER, + n3_6 INTEGER, + n4_5 INTEGER, + n5_8 INTEGER, + n8_0 INTEGER, + sqf_J INTEGER, + sqf_H INTEGER, + sqf_Ks INTEGER, + sqf_3_6 INTEGER, + sqf_4_5 INTEGER, + sqf_5_8 INTEGER, + sqf_8_0 INTEGER, + mf3_6 INTEGER, + mf4_5 INTEGER, + mf5_8 INTEGER, + mf8_0 INTEGER +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql new file mode 100644 index 00000000..7c94d775 --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_index.sql @@ -0,0 +1,11 @@ + +-- PK + +ALTER TABLE catalogdb.glimpse ADD COLUMN pk BIGSERIAL PRIMARY KEY; + + +-- Indexes + +CREATE INDEX CONCURRENTLY ON catalogdb.glimpse (q3c_ang2ipix(ra, dec)); +CLUSTER glimpse_q3c_ang2ipix_idx ON catalogdb.glimpse; +ANALYZE catalogdb.glimpse; diff --git a/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py new file mode 100755 index 00000000..6c6c1552 --- /dev/null +++ b/schema/sdss5db/catalogdb/GLIMPSE/GLIMPSE_load.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-27 +# @Filename: GLIMPSE_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import io +import os + +import pandas + +from sdssdb.peewee.sdss5db import database + + +def main(): + + assert database.connected + + skiprows = {'GLMIA.tbl.gz': 9, + 'GLMIIA.tbl.gz': 13, + 'GLM3D_jan2009_Archive.tbl.gz': 13, + 'GLM3DA_l330+02.tbl.gz': 14, + 'GLM3DA_l330-02.tbl.gz': 14, + 'GLM3DA_l335-02.tbl.gz': 14} + + files = glob.glob(os.environ['CATALOGDB_DIR'] + '/GLIMPSE/*.tbl.gz') + + for file_ in files: + + print(file_) + + basename = os.path.basename(file_) + nrows = skiprows[basename] + + tables = pandas.read_csv(file_, skipinitialspace=True, delimiter=' ', + chunksize=1000000, header=None, skiprows=nrows) + + cursor = database.cursor() + + for ii, table in enumerate(tables): + + print(f'Chunk {ii+1} ...') + + table[0] = table[0] + table[1] + table = table.drop(columns=1) + table[3] = table[3].replace({0: pandas.NA}) + + stream = io.StringIO() + table.to_csv(stream, header=False, index=False, na_rep='\\0') + + stream.seek(0) + cursor.copy_from(stream, 'catalogdb.glimpse', sep=',', null='\\0') + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql new file mode 100644 index 00000000..f5b53cee --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN.sql @@ -0,0 +1,48 @@ +/* + +Schema for Gaia_unWISE_AGN + +Docs: https://people.ast.cam.ac.uk/~ypshu/AGN_Catalogues.html +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/gaia_unwise_agn/v1 + +*/ + +CREATE TABLE catalogdb.gaia_unwise_agn ( + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + gaia_sourceid BIGINT PRIMARY KEY, + unwise_objid TEXT, + plx DOUBLE PRECISION, + plx_err DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_err DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_err DOUBLE PRECISION, + plxsig DOUBLE PRECISION, + pmsig DOUBLE PRECISION, + ebv DOUBLE PRECISION, + n_obs INTEGER, + g DOUBLE PRECISION, + bp DOUBLE PRECISION, + rp DOUBLE PRECISION, + w1 DOUBLE PRECISION, + w2 DOUBLE PRECISION, + bp_g DOUBLE PRECISION, + bp_rp DOUBLE PRECISION, + g_rp DOUBLE PRECISION, + g_w1 DOUBLE PRECISION, + gw_sep DOUBLE PRECISION, + w1_w2 DOUBLE PRECISION, + g_var DOUBLE PRECISION, + bprp_ef DOUBLE PRECISION, + aen DOUBLE PRECISION, + gof DOUBLE PRECISION, + cnt1 INTEGER, + cnt2 INTEGER, + cnt4 INTEGER, + cnt8 INTEGER, + cnt16 INTEGER, + cnt32 INTEGER, + phot_z DOUBLE PRECISION, + prob_rf DOUBLE PRECISION +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql new file mode 100644 index 00000000..334dc6e9 --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_index.sql @@ -0,0 +1,7 @@ + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_unwise_agn_q3c_ang2ipix_idx ON catalogdb.gaia_unwise_agn; +ANALYZE catalogdb.gaia_unwise_agn; + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (g); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (prob_rf); diff --git a/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py new file mode 100755 index 00000000..e10d1e6a --- /dev/null +++ b/schema/sdss5db/catalogdb/Gaia_unWISE_AGN/Gaia_unWISE_AGN_load.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-13 +# @Filename: Gaia_unWISE_AGN_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + file_ = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/gaia_unwise_agn/v1/Gaia_unWISE_AGNs.fits' # noqa + + data = astropy.table.Table.read(file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, overwrite=True) + del data + + cursor = database.cursor() + fileobj = open(file_ + '.csv') + fileobj.readline() # Read header + cursor.copy_from(fileobj, 'catalogdb.gaia_unwise_agn', sep=',') + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql new file mode 100644 index 00000000..71b20640 --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18.sql @@ -0,0 +1,23 @@ +/* + +PS1 gMeanPSFMag < 18. + +*/ + + +CREATE TABLE catalogdb.ps1_g18 ( + objid BIGINT, + ndetections INTEGER, + ramean DOUBLE PRECISION, + decmean DOUBLE PRECISION, + qualityflag INTEGER, + gmeanpsfmag DOUBLE PRECISION, + gmeanpsfmagerr DOUBLE PRECISION, + gflags BIGINT, + rmeanpsfmag DOUBLE PRECISION, + rmeanpsfmagerr DOUBLE PRECISION, + rflags BIGINT, + imeanpsfmag DOUBLE PRECISION, + imeanpsfmagerr DOUBLE PRECISION, + iflags BIGINT +); diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql new file mode 100644 index 00000000..0f06ec7f --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_index.sql @@ -0,0 +1,14 @@ + +-- PK + +ALTER TABLE catalogdb.ps1_g18 ADD PRIMARY KEY (objid); + +-- Indexes + +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (q3c_ang2ipix(raMean, decMean)); +CLUSTER ps1_g18_q3c_ang2ipix_idx ON catalogdb.ps1_g18; +ANALYZE catalogdb.ps1_g18; + +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (gmeanpsfmag); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (rmeanpsfmag); +CREATE INDEX CONCURRENTLY ON catalogdb.ps1_g18 (imeanpsfmag); diff --git a/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py new file mode 100755 index 00000000..3508c0c5 --- /dev/null +++ b/schema/sdss5db/catalogdb/PS1/g18/PS1_g18_load.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-27 +# @Filename: PS1_g18_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import io +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database + + +def main(): + + assert database.connected + + files = glob.glob(os.environ['CATALOGDB_DIR'] + '/PS1/g18/*.fit') + + for file_ in files: + + print(file_) + + data = astropy.table.Table.read(file_) + data.meta = {} + + stream = io.StringIO() + data.write(stream, format='csv', fast_writer=True) + + cursor = database.cursor() + stream.seek(0) + stream.readline() # Read header + cursor.copy_from(stream, 'catalogdb.ps1_g18', sep=',') + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql b/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql new file mode 100644 index 00000000..1e8c5f0b --- /dev/null +++ b/schema/sdss5db/catalogdb/TESS-TOI/tess_toi.sql @@ -0,0 +1,27 @@ +/* + +TESS Targets of Opportunity + +Original file /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/TESSobserved/TESS_Observed.fits +converted to CSV with + +t = table.Table.read('./TESS_Observed.fits') +t.convert_bytestring_to_unicode() +t.to_pandas().to_csv('./TESS_Observed.csv', index=False) + +*/ + +CREATE TABLE catalogdb.tess_toi ( + ticid BIGINT, + target_type VARCHAR(8), + toi VARCHAR(32), + tess_disposition VARCHAR(2), + tfopwg_disposition VARCHAR(2), + ctoi VARCHAR(32), + user_disposition VARCHAR(2), + num_sectors REAL +) WITHOUT OIDS; + +\COPY catalogdb.tess_toi FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/TESSobserved/TESS_Observed.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.tess_toi ADD COLUMN pk BIGSERIAL PRIMARY KEY; diff --git a/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql new file mode 100644 index 00000000..cf51f5ba --- /dev/null +++ b/schema/sdss5db/catalogdb/XMMOMSUOB/XMMOMSUOB.sql @@ -0,0 +1,132 @@ +/* + +XMM-Newton Optical Monitor SUSS Catalog, v4.1 +https://heasarc.gsfc.nasa.gov/W3Browse/all/xmmomsuob.html + +*/ + +CREATE TABLE catalogdb.xmm_om_suss_4_1 ( + iauname VARCHAR(22), + n_summary INTEGER, + obsid VARCHAR(10), + srcnum INTEGER, + uvw2_srcdist REAL, + uvm2_srcdist REAL, + uvw1_srcdist REAL, + u_srcdist REAL, + b_srcdist REAL, + v_srcdist REAL, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + ra_hms VARCHAR(13), + dec_dms VARCHAR(14), + poserr REAL, + lii DOUBLE PRECISION, + bii DOUBLE PRECISION, + n_obsid INTEGER, + uvw2_signif REAL, + uvm2_signif REAL, + uvw1_signif REAL, + u_signif REAL, + b_signif REAL, + v_signif REAL, + uvw2_rate REAL, + uvw2_rate_err REAL, + uvm2_rate REAL, + uvm2_rate_err REAL, + uvw1_rate REAL, + uvw1_rate_err REAL, + u_rate REAL, + u_rate_err REAL, + b_rate REAL, + b_rate_err REAL, + v_rate REAL, + v_rate_err REAL, + uvw2_ab_flux REAL, + uvw2_ab_flux_err REAL, + uvm2_ab_flux REAL, + uvm2_ab_flux_err REAL, + uvw1_ab_flux REAL, + uvw1_ab_flux_err REAL, + u_ab_flux REAL, + u_ab_flux_err REAL, + b_ab_flux REAL, + b_ab_flux_err REAL, + v_ab_flux REAL, + v_ab_flux_err REAL, + uvw2_ab_mag REAL, + uvw2_ab_mag_err REAL, + uvm2_ab_mag REAL, + uvm2_ab_mag_err REAL, + uvw1_ab_mag REAL, + uvw1_ab_mag_err REAL, + u_ab_mag REAL, + u_ab_mag_err REAL, + b_ab_mag REAL, + b_ab_mag_err REAL, + v_ab_mag REAL, + v_ab_mag_err REAL, + uvw2_vega_mag REAL, + uvw2_vega_mag_err REAL, + uvm2_vega_mag REAL, + uvm2_vega_mag_err REAL, + uvw1_vega_mag REAL, + uvw1_vega_mag_err REAL, + u_vega_mag REAL, + u_vega_mag_err REAL, + b_vega_mag REAL, + b_vega_mag_err REAL, + v_vega_mag REAL, + v_vega_mag_err REAL, + uvw2_major_axis REAL, + uvm2_major_axis REAL, + uvw1_major_axis REAL, + u_major_axis REAL, + b_major_axis REAL, + v_major_axis REAL, + uvw2_minor_axis REAL, + uvm2_minor_axis REAL, + uvw1_minor_axis REAL, + u_minor_axis REAL, + b_minor_axis REAL, + v_minor_axis REAL, + uvw2_posang REAL, + uvm2_posang REAL, + uvw1_posang REAL, + u_posang REAL, + b_posang REAL, + v_posang REAL, + uvw2_quality_flag SMALLINT, + uvm2_quality_flag SMALLINT, + uvw1_quality_flag SMALLINT, + u_quality_flag SMALLINT, + b_quality_flag SMALLINT, + v_quality_flag SMALLINT, + uvw2_quality_flag_st VARCHAR(12), + uvm2_quality_flag_st VARCHAR(12), + uvw1_quality_flag_st VARCHAR(12), + u_quality_flag_st VARCHAR(12), + b_quality_flag_st VARCHAR(12), + v_quality_flag_st VARCHAR(12), + uvw2_extended_flag BIGINT, + uvm2_extended_flag BIGINT, + uvw1_extended_flag BIGINT, + u_extended_flag BIGINT, + b_extended_flag BIGINT, + v_extended_flag BIGINT, + uvw2_sky_image VARCHAR(4), + uvm2_sky_image VARCHAR(4), + uvw1_sky_image VARCHAR(4), + u_sky_image VARCHAR(4), + b_sky_image VARCHAR(4), + v_sky_image VARCHAR(4) +) WITHOUT OIDS; + + +\COPY catalogdb.xmm_om_suss_4_1 FROM PROGRAM 'zcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/XMMOMSUOB/XMM-OM-SUSS4.1.csv.gz' WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.xmm_om_suss_4_1 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX CONCURRENTLY ON catalogdb.xmm_om_suss_4_1 (q3c_ang2ipix(ra, dec)); +CLUSTER xmm_om_suss_4_1_q3c_ang2ipix_idx on catalogdb.xmm_om_suss_4_1; +VACUUM ANALYZE catalogdb.xmm_om_suss_4_1; diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise.sql b/schema/sdss5db/catalogdb/allwise/source/allwise.sql index 0b6c7f6f..81b1a0d7 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise.sql @@ -20,7 +20,7 @@ CREATE TABLE catalogdb.allwise( elat decimal(9,7) , wx decimal(7,3) , wy decimal(7,3) , - cntr bigint , + cntr bigint PRIMARY KEY, source_id char(28) , coadd_id char(20) , src integer , @@ -307,8 +307,3 @@ CREATE TABLE catalogdb.allwise( spt_ind integer , htm20 bigint ); - - - - - diff --git a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql index ad46dafe..f51cc8ca 100644 --- a/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql +++ b/schema/sdss5db/catalogdb/allwise/source/allwise_index.sql @@ -8,17 +8,25 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; */ +-- Unique identification number for this object in the AllWISE Catalog/Reject Table. +-- This number is formed from the source_id, which is in turn formed from the +-- coadd_id and source number, src. + +ALTER TABLE catalogdb.allwise ADD PRIMARY KEY (cntr); + -- Indices -alter table catalogdb.allwise add primary key(designation); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glat); -CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (designation); -create index on catalogdb.allwise (q3c_ang2ipix(ra, dec)); -CLUSTER allwise_q3c_ang2ipix_idx on catalogdb.allwise; -analyze catalogdb.allwise; +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise USING BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise using BTREE (glon); +ALTER TABLE catalogdb.allwise + ADD CONSTRAINT allwise_designation_unique UNIQUE (designation); +CREATE INDEX CONCURRENTLY ON catalogdb.allwise (q3c_ang2ipix(ra, dec)); +CLUSTER allwise_q3c_ang2ipix_idx ON catalogdb.allwise; +ANALYZE catalogdb.allwise; diff --git a/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql new file mode 100644 index 00000000..a244d2f2 --- /dev/null +++ b/schema/sdss5db/catalogdb/best_brightest/best_brightest.sql @@ -0,0 +1,53 @@ +/* + +Best & Brightest Catalog + +*/ + + +CREATE TABLE catalogdb.best_brightest ( + designation VARCHAR(19), + ra_1 DOUBLE PRECISION, + dec_1 DOUBLE PRECISION, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + w1mpro REAL, + w2mpro REAL, + w3mpro REAL, + w4mpro VARCHAR(6), + pmra INTEGER, + pmdec INTEGER, + j_m_2mass REAL, + h_m_2mass REAL, + k_m_2mass REAL, + ra_2 DOUBLE PRECISION, + raerr DOUBLE PRECISION, + dec_2 DOUBLE PRECISION, + decerr DOUBLE PRECISION, + nobs INTEGER, + mobs INTEGER, + vjmag REAL, + bjmag REAL, + gmag REAL, + rmag REAL, + imag REAL, + evjmag REAL, + ebjmag REAL, + egmag REAL, + ermag REAL, + eimag REAL, + name INTEGER, + separation DOUBLE PRECISION, + ebv REAL, + version INTEGER, + original_ext_source_id VARCHAR(16) +); + + +\COPY catalogdb.best_brightest FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/best_brightest/sdss_v_best_and_brightest_merged.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.best_brightest ADD PRIMARY KEY (designation); + +CREATE INDEX CONCURRENTLY ON catalogdb.best_brightest (q3c_ang2ipix(ra_1, dec_1)); +CLUSTER best_brightest_q3c_ang2ipix_idx ON catalogdb.best_brightest; +ANALYZE catalogdb.best_brightest; diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql new file mode 100644 index 00000000..abeb2fab --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc.sql @@ -0,0 +1,26 @@ +/* + +Catalogue of optical/NIR counterparts to Chandra Source Catalogue v2 sources + +Note that 'fake' targets have been removed + +# Filename rows +# CSC2_stub1_realonly_v0.1.0.fits 86082 + +Files: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/csc/csc_for_catalogdb_v0 + +*/ + +CREATE TABLE catalogdb.bhm_csc ( + pk BIGSERIAL PRIMARY KEY, + csc_version TEXT, + cxo_name TEXT, + oir_ra DOUBLE PRECISION, + oir_dec DOUBLE PRECISION, + mag_g REAL, + mag_r REAL, + mag_i REAL, + mag_z REAL, + mag_h REAL, + spectrograph TEXT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql new file mode 100644 index 00000000..2c64cbb9 --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_index.sql @@ -0,0 +1,10 @@ + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc (q3c_ang2ipix(oir_ra, oir_dec)); +CLUSTER bhm_csc_q3c_ang2ipix_idx on catalogdb.bhm_csc; +ANALYZE catalogdb.bhm_csc; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_g); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_r); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_i); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_z); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_csc USING BTREE (mag_h); diff --git a/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py new file mode 100755 index 00000000..bdce528e --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_csc/v0.1.0/bhm_csc_load.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-17 +# @Filename: bhm_csc_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table +import numpy + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' + 'target/catalogs/csc/csc_for_catalogdb_v0/CSC2_stub1_realonly_v0.1.0.fits') + + data = astropy.table.Table.read(file_) + data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. + + copy_data(data, database, 'bhm_csc', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql new file mode 100644 index 00000000..243d00ce --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_rm/v0/bhm_rm_v0.sql @@ -0,0 +1,168 @@ +/* + +BHM Reverberation Mapping Photometry v0. + +See /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/RM/v0/rm_photo_v0_readme +CSV file generated using sdssdb.utils.ingest.to_csv with default options. + +*/ + +CREATE TABLE catalogdb.bhm_rm_v0 ( + field_name VARCHAR(8), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + distance DOUBLE PRECISION, + pos_ref VARCHAR(4), + ebv DOUBLE PRECISION, + des INTEGER, + coadd_object_id BIGINT, + ra_des DOUBLE PRECISION, + dec_des DOUBLE PRECISION, + extended_coadd INTEGER, + psfmag_des DOUBLE PRECISION[5], + psfmagerr_des DOUBLE PRECISION[5], + mag_auto_des DOUBLE PRECISION[5], + magerr_auto_des DOUBLE PRECISION[5], + imaflags_iso INTEGER[5], + separation_des DOUBLE PRECISION, + ps1 INTEGER, + objid_ps1 BIGINT, + ra_ps1 DOUBLE PRECISION, + dec_ps1 DOUBLE PRECISION, + class_ps1 DOUBLE PRECISION, + psfmag_ps1 DOUBLE PRECISION[5], + psfmagerr_ps1 DOUBLE PRECISION[5], + kronmag_ps1 DOUBLE PRECISION[5], + kronmagerr_ps1 DOUBLE PRECISION[5], + infoflag2 INTEGER[5], + separation_ps1 DOUBLE PRECISION, + nsc INTEGER, + id_nsc BIGINT, + ra_nsc DOUBLE PRECISION, + dec_nsc DOUBLE PRECISION, + class_star DOUBLE PRECISION, + mag_nsc DOUBLE PRECISION[5], + magerr_nsc DOUBLE PRECISION[5], + flags_nsc INTEGER, + separation_nsc DOUBLE PRECISION, + sdss INTEGER, + objid_sdss BIGINT, + ra_sdss DOUBLE PRECISION, + dec_sdss DOUBLE PRECISION, + type_sdss INTEGER, + psfmag_sdss DOUBLE PRECISION[5], + psfmagerr_sdss DOUBLE PRECISION[5], + modelmag_sdss DOUBLE PRECISION[5], + modelmagerr_sdss DOUBLE PRECISION[5], + clean_sdss INTEGER, + separation_sdss DOUBLE PRECISION, + gaia INTEGER, + source_id_gaia BIGINT, + mg DOUBLE PRECISION, + mag_gaia DOUBLE PRECISION[3], + magerr_gaia DOUBLE PRECISION[3], + parallax DOUBLE PRECISION, + parallax_error DOUBLE PRECISION, + plxsig DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_error DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_error DOUBLE PRECISION, + pmsig DOUBLE PRECISION, + unwise INTEGER, + objid_unwise VARCHAR(16), + ra_unwise DOUBLE PRECISION, + dec_unwise DOUBLE PRECISION, + mag_unwise DOUBLE PRECISION[2], + magerr_unwise DOUBLE PRECISION[2], + flags_unwise INTEGER[2], + separation_unwise DOUBLE PRECISION, + near_ir INTEGER, + survey_ir VARCHAR(6), + sourceid_ir BIGINT, + ra_ir DOUBLE PRECISION, + dec_ir DOUBLE PRECISION, + mag_ir DOUBLE PRECISION[4], + magerr_ir DOUBLE PRECISION[4], + separation_ir DOUBLE PRECISION, + optical_survey VARCHAR(4), + mi DOUBLE PRECISION, + cal_skewt_qso INTEGER, + nband_optical_use INTEGER, + use_unwise INTEGER, + use_nir INTEGER, + photo_combination VARCHAR(17), + log_qso DOUBLE PRECISION, + log_star DOUBLE PRECISION, + log_galaxy DOUBLE PRECISION, + p_qso DOUBLE PRECISION, + p_star REAL, + p_galaxy DOUBLE PRECISION, + class_skewt_qso VARCHAR(6), + skewt_qso INTEGER, + p_qso_prior DOUBLE PRECISION, + p_star_prior REAL, + p_galaxy_prior DOUBLE PRECISION, + class_skewt_qso_prior VARCHAR(6), + skewt_qso_prior INTEGER, + photoz_qso DOUBLE PRECISION, + photoz_qso_lower DOUBLE PRECISION, + photoz_qso_upper DOUBLE PRECISION, + prob_photoz_qso DOUBLE PRECISION, + photoz_galaxy DOUBLE PRECISION, + photoz_galaxy_lower DOUBLE PRECISION, + photoz_galaxy_upper DOUBLE PRECISION, + pqso_xdqso DOUBLE PRECISION, + photoz_xdqso DOUBLE PRECISION, + prob_rf_gaia_unwise DOUBLE PRECISION, + photoz_gaia_unwise DOUBLE PRECISION, + des_var_nepoch INTEGER[5], + des_var_status INTEGER[5], + des_var_rms DOUBLE PRECISION[5], + des_var_sigrms DOUBLE PRECISION[5], + des_var_sn DOUBLE PRECISION[5], + des_var_sn_max DOUBLE PRECISION, + ps1_var_nepoch INTEGER[5], + ps1_var_status INTEGER[5], + ps1_var_rms DOUBLE PRECISION[5], + ps1_var_sigrms DOUBLE PRECISION[5], + ps1_var_sn DOUBLE PRECISION[5], + ps1_var_sn_max DOUBLE PRECISION, + spec_q INTEGER, + spec_strmask VARCHAR(6), + spec_bitmask BIGINT, + specz DOUBLE PRECISION, + specz_ref VARCHAR(16), + photo_q INTEGER, + photo_strmask VARCHAR(3), + photo_bitmask BIGINT, + photoz DOUBLE PRECISION, + pqso_photo DOUBLE PRECISION, + photoz_ref VARCHAR(16) +) WITHOUT OIDS; + + +\COPY catalogdb.bhm_rm_v0 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/RM/v0/rm_photo_v0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; + +ALTER TABLE catalogdb.bhm_rm_v0 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (q3c_ang2ipix(ra, dec)); +CLUSTER bhm_rm_v0_q3c_ang2ipix_idx ON catalogdb.bhm_rm_v0; +VACUUM ANALYZE catalogdb.bhm_rm_v0; + +-- Update to NULL in columns that will be fks. +UPDATE catalogdb.bhm_rm_v0 SET source_id_gaia = NULL WHERE gaia <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_ps1 = NULL WHERE ps1 <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_sdss = NULL WHERE sdss <= 0; +UPDATE catalogdb.bhm_rm_v0 SET objid_unwise = NULL WHERE unwise <= 0; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (mi); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (skewt_qso); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (pmsig); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (plxsig); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (spec_q); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (des_var_rms); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (des_var_sn); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (ps1_var_rms); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (ps1_var_sn); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (photo_bitmask); diff --git a/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql new file mode 100644 index 00000000..5704ac3f --- /dev/null +++ b/schema/sdss5db/catalogdb/bhm_veto/SPIDERS_eFEDS/v0.1.0/BHM_SPIDERS_eFEDS_veto.sql @@ -0,0 +1,102 @@ +/* + +BHM-SPIDERS eFEDS veto catalogue + +* A minimalist catalogue of 6300 science targets in the SPIDERS eFEDS field +that received SDSS-IV spectroscopy during the March2020 SPIDERS observing run +(and hence are not in SDSS-SpecObj-DR16) +* Many of these objects have v low SNR and so we will want to observe them again, +but we will take account of this in the Carton code +* Available now on Utah systems under +* /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/ +* Datamodel: + * This is a subset (in both rows and columns) of the spAll-v5_13_1.fits file + * orig file from: https://data.sdss.org/sas/ebosswork/eboss/spectro/redux/v5_13_1/ + * orig data model at: https://data.sdss.org/datamodel/files/BOSS_SPECTRO_REDUX/RUN2D/spAll.html + + No. Type EXTNAME BITPIX Dimensions(columns) PCOUNT GCOUNT + + 0 PRIMARY 8 0 0 1 + 1 BINTABLE 8 174(26) 6300 0 1 + + Column Name Format Dims Units TLMIN TLMAX + 1 PROGRAMNAME 5A + 2 CHUNK 7A + 3 PLATESN2 E + 4 PLATE J + 5 TILE J + 6 MJD J + 7 FIBERID J + 8 RUN2D 7A + 9 RUN1D 7A + 10 PLUG_RA D + 11 PLUG_DEC D + 12 Z_ERR E + 13 RCHI2 E + 14 DOF J + 15 RCHI2DIFF E + 16 WAVEMIN E + 17 WAVEMAX E + 18 WCOVERAGE E + 19 ZWARNING J + 20 SN_MEDIAN 5E + 21 SN_MEDIAN_ALL E + 22 SPECTROFLUX 5E + 23 SPECTROFLUX_IVAR 5E + 24 ANYANDMASK J + 25 ANYORMASK J + 26 SPECOBJID K + +Note: the SPECOBJID column overflows the data type so it was removed when +converting the FITS table to CSV. + +*/ + +CREATE TABLE catalogdb.bhm_efeds_veto ( + programname VARCHAR(5), + chunk VARCHAR(7), + platesn2 REAL, + plate INTEGER, + tile INTEGER, + mjd INTEGER, + fiberid INTEGER, + run2d VARCHAR(7), + run1d VARCHAR(7), + plug_ra DOUBLE PRECISION, + plug_dec DOUBLE PRECISION, + z_err REAL, + rchi2 REAL, + dof INTEGER, + rchi2diff REAL, + wavemin REAL, + wavemax REAL, + wcoverage REAL, + zwarning INTEGER, + sn_median REAL[5], + sn_median_all REAL, + spectroflux REAL[5], + spectroflux_ivar REAL[5], + anyandmask INTEGER, + anyormask INTEGER +); + +\COPY catalogdb.bhm_efeds_veto FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/bhm_veto_lists/bhm_efeds_veto/v0.1.0/BHM_EFEDS_VETO_v0.1.0.csv WITH CSV HEADER DELIMITER E'\t' NULL '\N'; + +-- Query to create a new column with the correct specobjid. 1301 is the result of +-- (N-5)*10000+M*100+P for RUN2D v5_13_1 (see https://www.sdss.org/dr16/help/glossary/#specobj). +-- These specobjid are uint64 and larger than what can be stored as a BIGINT, so we're not +-- creating them for now. + +-- ALTER TABLE catalogdb.bhm_efeds_veto ADD COLUMN specobjid BIGINT; +-- UPDATE catalogdb.bhm_efeds_veto +-- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + +-- ((mjd-50000)::bigint<<24) + (1301::bigint<<10); +-- CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (specobjid); + +ALTER TABLE catalogdb.bhm_efeds_veto ADD PRIMARY KEY (plate, mjd, fiberid, run2d); + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (mjd, plate, fiberid, run2d); + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_efeds_veto (q3c_ang2ipix(plug_ra, plug_dec)); +CLUSTER bhm_efeds_veto_q3c_ang2ipix_idx ON catalogdb.bhm_efeds_veto; +ANALYZE catalogdb.bhm_efeds_veto; diff --git a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py new file mode 100755 index 00000000..48413210 --- /dev/null +++ b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: cataclysmic_variables.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/cataclysmicvariables.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'cataclysmic_variables', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql new file mode 100644 index 00000000..56ef15c8 --- /dev/null +++ b/schema/sdss5db/catalogdb/cataclysmic_variables/cataclysmic_variables.sql @@ -0,0 +1,109 @@ +/* + +AAVSO Cataclysmic Variables + +*/ + +CREATE TABLE catalogdb.cataclysmic_variables ( + ref_id BIGINT, + solution_id BIGINT, + designation TEXT, + source_id BIGINT, + random_index INTEGER, + ref_epoch REAL, + ra DOUBLE PRECISION, + ra_error DOUBLE PRECISION, + dec DOUBLE PRECISION, + dec_error DOUBLE PRECISION, + parallax DOUBLE PRECISION, + parallax_error DOUBLE PRECISION, + parallax_over_error DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmra_error DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pmdec_error DOUBLE PRECISION, + ra_dec_corr DOUBLE PRECISION, + ra_parallax_corr DOUBLE PRECISION, + ra_pmra_corr DOUBLE PRECISION, + ra_pmdec_corr DOUBLE PRECISION, + dec_parallax_corr DOUBLE PRECISION, + dec_pmra_corr DOUBLE PRECISION, + dec_pmdec_corr DOUBLE PRECISION, + parallax_pmra_corr DOUBLE PRECISION, + parallax_pmdec_corr DOUBLE PRECISION, + pmra_pmdec_corr DOUBLE PRECISION, + astrometric_n_obs_al SMALLINT, + astrometric_n_obs_ac SMALLINT, + astrometric_n_good_obs_al SMALLINT, + astrometric_n_bad_obs_al SMALLINT, + astrometric_gof_al DOUBLE PRECISION, + astrometric_chi2_al DOUBLE PRECISION, + astrometric_excess_noise DOUBLE PRECISION, + astrometric_excess_noise_sig DOUBLE PRECISION, + astrometric_params_solved SMALLINT, + astrometric_primary_flag BOOLEAN, + astrometric_weight_al DOUBLE PRECISION, + astrometric_pseudo_colour DOUBLE PRECISION, + astrometric_pseudo_colour_error DOUBLE PRECISION, + mean_varpi_factor_al DOUBLE PRECISION, + astrometric_matched_observations SMALLINT, + visibility_periods_used SMALLINT, + astrometric_sigma5d_max DOUBLE PRECISION, + frame_rotator_object_type SMALLINT, + matched_observations SMALLINT, + duplicated_source BOOLEAN, + phot_g_n_obs SMALLINT, + phot_g_mean_flux DOUBLE PRECISION, + phot_g_mean_flux_error DOUBLE PRECISION, + phot_g_mean_flux_over_error DOUBLE PRECISION, + phot_g_mean_mag DOUBLE PRECISION, + phot_bp_n_obs SMALLINT, + phot_bp_mean_flux DOUBLE PRECISION, + phot_bp_mean_flux_error DOUBLE PRECISION, + phot_bp_mean_flux_over_error DOUBLE PRECISION, + phot_bp_mean_mag DOUBLE PRECISION, + phot_rp_n_obs SMALLINT, + phot_rp_mean_flux DOUBLE PRECISION, + phot_rp_mean_flux_error DOUBLE PRECISION, + phot_rp_mean_flux_over_error DOUBLE PRECISION, + phot_rp_mean_mag DOUBLE PRECISION, + phot_bp_rp_excess_factor DOUBLE PRECISION, + phot_proc_mode SMALLINT, + bp_rp DOUBLE PRECISION, + bp_g DOUBLE PRECISION, + g_rp DOUBLE PRECISION, + radial_velocity DOUBLE PRECISION, + radial_velocity_error DOUBLE PRECISION, + rv_nb_transits SMALLINT, + rv_template_teff REAL, + rv_template_logg REAL, + rv_template_fe_h REAL, + phot_variable_flag TEXT, + l DOUBLE PRECISION, + b DOUBLE PRECISION, + ecl_lon DOUBLE PRECISION, + ecl_lat DOUBLE PRECISION, + priam_flags INTEGER, + teff_val DOUBLE PRECISION, + teff_percentile_lower DOUBLE PRECISION, + teff_percentile_upper DOUBLE PRECISION, + a_g_val REAL, + a_g_percentile_lower REAL, + a_g_percentile_upper REAL, + e_bp_min_rp_val REAL, + e_bp_min_rp_percentile_lower REAL, + e_bp_min_rp_percentile_upper REAL, + flame_flags INTEGER, + radius_val DOUBLE PRECISION, + radius_percentile_lower DOUBLE PRECISION, + radius_percentile_upper DOUBLE PRECISION, + lum_val DOUBLE PRECISION, + lum_percentile_lower DOUBLE PRECISION, + lum_percentile_upper DOUBLE PRECISION +); + +ALTER TABLE catalogdb.cataclysmic_variables ADD PRIMARY KEY (ref_id); + +CREATE INDEX CONCURRENTLY ON catalogdb.cataclysmic_variables (q3c_ang2ipix(ra, dec)); +CLUSTER cataclysmic_variables_q3c_ang2ipix_idx ON catalogdb.cataclysmic_variables; +ANALYZE catalogdb.cataclysmic_variables; diff --git a/schema/sdss5db/catalogdb/catalogdb.sql b/schema/sdss5db/catalogdb/catalog.sql similarity index 85% rename from schema/sdss5db/catalogdb/catalogdb.sql rename to schema/sdss5db/catalogdb/catalog.sql index 72153596..a0482349 100644 --- a/schema/sdss5db/catalogdb/catalogdb.sql +++ b/schema/sdss5db/catalogdb/catalog.sql @@ -17,4 +17,6 @@ CREATE TABLE catalogdb.catalog ( dec DOUBLE PRECISION, pmra REAL, pmdec REAL, - epoch REAL); + epoch REAL, + parallax REAL, + version TEXT); diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita.sql new file mode 100644 index 00000000..130036d9 --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita.sql @@ -0,0 +1,104 @@ +/* + +Catalogues of optical counterparts to eROSITA sources in the 'eFEDS' performace verification field (aka GAMA09) + +There are two catalogues here, one for 'AGN candidates' and one for 'Galaxy cluster candidates' + +PeeWee proto-data models are available for these two catalogues here: +https://github.com/sdss/sdssdb/blob/add_bhm_peewee_classes1/python/sdssdb/peewee/sdss5db/catalogdb.py +see classes 'BhmSpidersAgnSuperset' and 'BhmSpidersClustersSuperset' + +##Filename rows +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits 23994 +# catalogdb_v0/BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.1.fits 19636 + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/eRosita/eFEDS_for_catalogdb_v0 + +*/ + +CREATE TABLE catalogdb.bhm_spiders_clusters_superset ( + pk BIGSERIAL PRIMARY KEY, + ero_version TEXT, + ero_detuid TEXT, + ero_flux REAL, + ero_flux_err REAL, + ero_ext REAL, + ero_ext_err REAL, + ero_ext_like REAL, + ero_det_like REAL, + ero_ra DOUBLE PRECISION, + ero_dec DOUBLE PRECISION, + ero_radec_err REAL, + xmatch_method TEXT, + xmatch_version TEXT, + xmatch_dist REAL, + xmatch_metric REAL, + xmatch_flags BIGINT, + target_class TEXT, + target_priority INTEGER, + target_has_spec INTEGER, + best_opt TEXT, + ls_id BIGINT, + ps1_dr2_objid BIGINT, + gaia_dr2_source_id BIGINT, + unwise_dr1_objid TEXT, + des_dr1_coadd_object_id BIGINT, + sdss_dr16_objid BIGINT, + opt_ra DOUBLE PRECISION, + opt_dec DOUBLE PRECISION, + opt_pmra REAL, + opt_pmdec REAL, + opt_epoch REAL, + opt_modelflux_g REAL, + opt_modelflux_ivar_g REAL, + opt_modelflux_r REAL, + opt_modelflux_ivar_r REAL, + opt_modelflux_i REAL, + opt_modelflux_ivar_i REAL, + opt_modelflux_z REAL, + opt_modelflux_ivar_z REAL +) WITHOUT OIDS; + + +CREATE TABLE catalogdb.bhm_spiders_agn_superset ( + pk BIGSERIAL PRIMARY KEY, + ero_version TEXT, + ero_detuid TEXT, + ero_flux REAL, + ero_flux_err REAL, + ero_ext REAL, + ero_ext_err REAL, + ero_ext_like REAL, + ero_det_like REAL, + ero_ra DOUBLE PRECISION, + ero_dec DOUBLE PRECISION, + ero_radec_err REAL, + xmatch_method TEXT, + xmatch_version TEXT, + xmatch_dist REAL, + xmatch_metric REAL, + xmatch_flags BIGINT, + target_class TEXT, + target_priority INTEGER, + target_has_spec INTEGER, + best_opt TEXT, + ls_id BIGINT, + ps1_dr2_objid BIGINT, + gaia_dr2_source_id BIGINT, + unwise_dr1_objid TEXT, + des_dr1_coadd_object_id BIGINT, + sdss_dr16_objid BIGINT, + opt_ra DOUBLE PRECISION, + opt_dec DOUBLE PRECISION, + opt_pmra REAL, + opt_pmdec REAL, + opt_epoch REAL, + opt_modelflux_g REAL, + opt_modelflux_ivar_g REAL, + opt_modelflux_r REAL, + opt_modelflux_ivar_r REAL, + opt_modelflux_i REAL, + opt_modelflux_ivar_i REAL, + opt_modelflux_z REAL, + opt_modelflux_ivar_z REAL +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql new file mode 100644 index 00000000..a820f399 --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_index.sql @@ -0,0 +1,13 @@ + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CLUSTER bhm_spiders_agn_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_agn_superset; +ANALYZE catalogdb.bhm_spiders_agn_superset; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (ero_flux); + + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset (q3c_ang2ipix(ero_ra, ero_dec)); +CLUSTER bhm_spiders_clusters_superset_q3c_ang2ipix_idx ON catalogdb.bhm_spiders_clusters_superset; +ANALYZE catalogdb.bhm_spiders_clusters_superset; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ero_flux); diff --git a/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py new file mode 100755 index 00000000..fd94bf7a --- /dev/null +++ b/schema/sdss5db/catalogdb/eRosita/v0.1.2/eRosita_load.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-17 +# @Filename: eRosita_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table +import numpy + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/' + 'target/catalogs/eRosita/eFEDS_for_catalogdb_v0/') + + for file_, table in (('BHM_SPIDERS_EFEDS_SUPERSET_CLUS_v0.1.2.fits', + 'bhm_spiders_clusters_superset'), + ('BHM_SPIDERS_EFEDS_SUPERSET_AGN_v0.1.1.fits', + 'bhm_spiders_agn_superset')): + + data = astropy.table.Table.read(path_ + file_) + data.add_column(astropy.table.Column(numpy.arange(len(data)) + 1, 'pk'), 0) # Add id pk. + + copy_data(data, database, table, schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql new file mode 100644 index 00000000..1d1f1281 --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5.sql @@ -0,0 +1,53 @@ +/* + +Schema for eBOSS Target. + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005 + +*/ + +CREATE TABLE catalogdb.ebosstarget_v5 ( + run INTEGER, + camcol INTEGER, + field INTEGER, + id INTEGER, + rerun TEXT, + fibermag REAL[], + fiber2mag REAL[], + calib_status INTEGER[], + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + epoch REAL, + pmra REAL, + pmdec REAL, + eboss_target1 BIGINT, + eboss_target2 BIGINT, + eboss_target_id BIGINT, + thing_id_targeting INTEGER, + objc_type INTEGER, + objc_flags INTEGER, + objc_flags2 INTEGER, + flags INTEGER, + flags2 INTEGER, + psf_fwhm REAL[], + psfflux REAL[], + psfflux_ivar REAL[], + extinction REAL[], + fiberflux REAL[], + fiberflux_ivar REAL[], + fiber2flux REAL[], + fiber2flux_ivar REAL[], + modelflux REAL[], + modelflux_ivar REAL[], + modelmag REAL[], + modelmag_ivar REAL[], + resolve_status INTEGER, + w1_mag REAL, + w1_mag_err REAL, + w1_nanomaggies REAL, + w1_nanomaggies_ivar REAL, + w2_nanomaggies REAL, + w2_nanomaggies_ivar REAL, + has_wise_phot BOOLEAN NULL, + objid_targeting BIGINT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py new file mode 100755 index 00000000..1dd49bd9 --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_load.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-15 +# @Filename: ebosstarget_v5_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + path = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/ebosstarget/v0005/' + files = ['ebosstarget-v0005-qso.fits', 'ebosstarget-v0005-std.fits'] + + out = '/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/' + + for file_ in files: + + print(f'Converting {file_}') + data = astropy.table.Table.read(path + file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + if 'std' in file_: + data['has_wise_phot'] = '0' + to_csv(data, out + file_ + '.csv', header=True, overwrite=True, convert_arrays=True) + del data + + print(f'Copying {file_}') + cursor = database.cursor() + cursor.copy_expert('COPY catalogdb.ebosstarget_v5 FROM STDIN ' + 'WITH DELIMITER \',\' NULL \'\\N\' CSV HEADER;', + open(out + file_ + '.csv')) + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql new file mode 100644 index 00000000..0bd0b171 --- /dev/null +++ b/schema/sdss5db/catalogdb/ebosstarget/v5/ebosstarget_v5_post_load.sql @@ -0,0 +1,13 @@ +/* + +To be run after the data has been loaded. + +*/ + +ALTER TABLE catalogdb.ebosstarget_v5 ADD COLUMN pk BIGSERIAL PRIMARY KEY; + +CREATE INDEX CONCURRENTLY ON catalogdb.ebosstarget_v5 (q3c_ang2ipix(ra, dec)); +CLUSTER ebosstarget_v5_q3c_ang2ipix_idx on catalogdb.ebosstarget_v5; +ANALYZE catalogdb.ebosstarget_v5; + +CREATE INDEX CONCURRENTLY ON catalogdb.ebosstarget_v5 USING BTREE (objc_type); diff --git a/schema/sdss5db/catalogdb/foreignKeys.sql b/schema/sdss5db/catalogdb/foreignKeys.sql index d3f4072b..f4d380d3 100644 --- a/schema/sdss5db/catalogdb/foreignKeys.sql +++ b/schema/sdss5db/catalogdb/foreignKeys.sql @@ -9,11 +9,327 @@ https://www.postgresql.org/docs/8.2/static/sql-altertable.html */ -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT original_ext_source_id_fk FOREIGN KEY (original_ext_source_id) REFERENCES catalogdb.twomass_psc (designation) ON UPDATE CASCADE ON DELETE CASCADE; -ALTER TABLE catalogdb.gaia_dr2_wd_candidates_v1 ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; +-- gaia_dr2_clean -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT source_id_fk FOREIGN KEY (source_id) REFERENCES catalogdb.gaia_dr2_source (source_id) ON UPDATE CASCADE ON DELETE CASCADE; -ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD CONSTRAINT tmass_pts_key_fk FOREIGN KEY (tmass_pts_key) REFERENCES catalogdb.twomass_psc (pts_key) ON UPDATE CASCADE ON DELETE CASCADE; +ALTER TABLE catalogdb.gaia_dr2_clean + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; -ALTER TABLE catalogdb.sdss_dr14_specobj ADD CONSTRAINT bestobjid_fk FOREIGN KEY (bestobjid) REFERENCES catalogdb.sdss_dr13_photoobj (objid) ON UPDATE CASCADE ON DELETE CASCADE; + +-- gaiadr2_sdssdr9_best_neighbour + +-- Some objectids are not present in dr13_photooj. It should not matter +-- because the TIC has x-matching with SDSS. + +-- CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_sdssdr9_best_neighbour (sdssdr9_oid); + +-- ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour +-- ADD CONSTRAINT source_id_fk +-- FOREIGN KEY (source_id) +-- REFERENCES catalogdb.gaia_dr2_source (source_id) +-- ON UPDATE CASCADE ON DELETE CASCADE; + +-- ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour +-- ADD CONSTRAINT sdssdr9_oid_fk +-- FOREIGN KEY (sdssdr9_oid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaiadr2_tmass_best_neighbour + +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT tmass_pts_key_fk + FOREIGN KEY (tmass_pts_key) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaiadr2_tmass_best_neighbour + +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_pts_key ASC); + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour + ADD CONSTRAINT tmass_pts_key_fk + FOREIGN KEY (tmass_pts_key) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- sdss_dr14_specobj + +UPDATE catalogdb.sdss_dr14_specobj SET bestobjid = NULL WHERE bestobjid = 0; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj using BTREE (bestobjid ASC); + +-- Cannot be created because some bestobjids are not present in dr13_photoobj. +-- ALTER TABLE catalogdb.sdss_dr14_specobj +-- ADD CONSTRAINT bestobjid_fk +-- FOREIGN KEY (bestobjid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; + +-- sdss_dr16_specobj + +UPDATE catalogdb.sdss_dr16_specobj SET bestobjid = NULL WHERE bestobjid = 0; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj using BTREE (bestobjid ASC); + +-- Cannot be created because some bestobjids are not present in dr13_photoobj. +-- ALTER TABLE catalogdb.sdss_dr16_specobj +-- ADD CONSTRAINT bestobjid_fk +-- FOREIGN KEY (bestobjid) +-- REFERENCES catalogdb.sdss_dr13_photoobj (objid) +-- ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaia_unwise_agn + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_unwise_agn USING BTREE (unwise_objid); + +ALTER TABLE catalogdb.gaia_unwise_agn + ADD CONSTRAINT source_id_fk + FOREIGN KEY (gaia_sourceid) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.gaia_unwise_agn + ADD CONSTRAINT unwise_objid_fk + FOREIGN KEY (unwise_objid) + REFERENCES catalogdb.unwise (unwise_objid) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- sdss_dr14_apogeeStarVisit + +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit + ADD CONSTRAINT visit_id_fk + FOREIGN KEY (visit_id) + REFERENCES catalogdb.sdss_dr14_apogeeVisit (visit_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit + ADD CONSTRAINT apstar_id_fk + FOREIGN KEY (apstar_id) + REFERENCES catalogdb.sdss_dr14_apogeeStar (apstar_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- sdss_dr14_ascapStar + +ALTER TABLE catalogdb.sdss_dr14_ascapStar + ADD CONSTRAINT apstar_id_fk + FOREIGN KEY (apstar_id) + REFERENCES catalogdb.sdss_dr14_apogeeStar (apstar_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- tic_v8 + +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (sdss); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (tyc); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (twomass_psc); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (kic); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (allwise); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gaia_int); + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT sdss_fk + FOREIGN KEY (sdss) + REFERENCES catalogdb.sdss_dr13_photoobj (objid) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT tyc_fk + FOREIGN KEY (tyc) + REFERENCES catalogdb.tycho2 (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT twomass_psc_fk + FOREIGN KEY (twomass_psc) + REFERENCES catalogdb.twomass_psc (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT allwise_fk + FOREIGN KEY (allwise) + REFERENCES catalogdb.allwise (designation) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT kic_fk + FOREIGN KEY (kic) + REFERENCES catalogdb.kepler_input_10 (kic_kepler_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.tic_v8 + ADD CONSTRAINT gaia_int_fk + FOREIGN KEY (gaia_int) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- bhm_spiders_agn_superset, bhm_spiders_clusters_superset + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (gaia_dr2_source_id); +UPDATE catalogdb.bhm_spiders_agn_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_agn_superset USING BTREE (ls_id); + +ALTER TABLE catalogdb.bhm_spiders_agn_superset + ADD CONSTRAINT gaia_dr2_source_id_fk + FOREIGN KEY (gaia_dr2_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.bhm_spiders_agn_superset + ADD CONSTRAINT ls_id_fk + FOREIGN KEY (ls_id) + REFERENCES catalogdb.legacy_survey_dr8 (ls_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (gaia_dr2_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_spiders_clusters_superset USING BTREE (ls_id); +UPDATE catalogdb.bhm_spiders_clusters_superset SET gaia_dr2_source_id = NULL WHERE gaia_dr2_source_id = 0; + +ALTER TABLE catalogdb.bhm_spiders_clusters_superset + ADD CONSTRAINT gaia_dr2_source_id_fk + FOREIGN KEY (gaia_dr2_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.bhm_spiders_clusters_superset + ADD CONSTRAINT ls_id_fk + FOREIGN KEY (ls_id) + REFERENCES catalogdb.legacy_survey_dr8 (ls_id) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- skymapper_dr1_1 + +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (allwise_cntr); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id1); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 USING BTREE (gaia_dr2_id2); + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT allwise_cntr_fk + FOREIGN KEY (allwise_cntr) + REFERENCES catalogdb.allwise (cntr) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT gaia_dr2_id1_fk + FOREIGN KEY (gaia_dr2_id1) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT gaia_dr2_id2_fk + FOREIGN KEY (gaia_dr2_id2) + REFERENCES catalogdb.gaia_dr2_source (source_id) + + +-- glimpse + +CREATE INDEX CONCURRENTLY ON catalogdb.glimpse USING BTREE (tmass_cntr); + +ALTER TABLE catalogdb.glimpse + ADD CONSTRAINT tmass_cntr_fk + FOREIGN KEY (tmass_cntr) + REFERENCES catalogdb.twomass_psc (pts_key) + ON UPDATE CASCADE ON DELETE CASCADE; + + +-- gaia_dr2_wd + +ALTER TABLE catalogdb.gaia_dr2_wd + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- legacy_survey_dr8 + +ALTER TABLE catalogdb.legacy_survey_dr8 + ADD CONSTRAINT gaia_sourceid_fk + FOREIGN KEY (gaia_sourceid) + REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- yso_clustering + +ALTER TABLE catalogdb.yso_clustering + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- mipsgal + +-- Prepare the columns +UPDATE catalogdb.mipsgal SET twomass_name = TRIM(twomass_name); +UPDATE catalogdb.mipsgal SET twomass_name = NULL where twomass_name = ''; +UPDATE catalogdb.mipsgal SET glimpse = NULL where TRIM(glimpse) = ''; + +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal USING BTREE (twomass_name); +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal USING BTREE (glimpse); + +ALTER TABLE catalogdb.mipsgal + ADD CONSTRAINT twomass_name_fk + FOREIGN KEY (twomass_name) + REFERENCES catalogdb.twomass_psc (designation); + + +-- TESS-TOI + +CREATE INDEX CONCURRENTLY ON catalogdb.tess_toi USING BTREE (ticid); + +ALTER TABLE catalogdb.tess_toi + ADD CONSTRAINT ticid_fk + FOREIGN KEY (ticid) + REFERENCES catalogdb.tic_v8 (id); + + +-- geometric_distances_gaia_dr2 + +ALTER TABLE catalogdb.geometric_distances_gaia_dr2 + ADD CONSTRAINT source_id_fk + FOREIGN KEY (source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id); + + +-- bhm_rm_v0 + +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (source_id_gaia) +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (objid_sdss) +CREATE INDEX CONCURRENTLY ON catalogdb.bhm_rm_v0 (objid_unwise) + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT source_id_gaia_fk + FOREIGN KEY (source_id_gaia) + REFERENCES catalogdb.gaia_dr2_source (source_id); + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT objid_sdss_fk + FOREIGN KEY (objid_sdss) + REFERENCES catalogdb.sdss_dr13_photoobj (objid); + +ALTER TABLE catalogdb.bhm_rm_v0 + ADD CONSTRAINT objid_unwise_fk + FOREIGN KEY (objid_unwise) + REFERENCES catalogdb.unwise (unwise_objid); diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql index b09cb428..f19f8d2c 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source.sql @@ -19,7 +19,7 @@ drop schema catalogdb cascade; CREATE TABLE catalogdb.gaia_dr2_source ( solution_id bigint, designation text, - source_id bigint, + source_id bigint primary key, random_index bigint, ref_epoch double precision, ra double precision, @@ -111,7 +111,3 @@ CREATE TABLE catalogdb.gaia_dr2_source ( lum_val real, lum_percentile_lower real, lum_percentile_upper real); - - - - diff --git a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql index 127ee6b5..b61d5c85 100644 --- a/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql +++ b/schema/sdss5db/catalogdb/gaia/dr2/gaia_dr2_source_index.sql @@ -11,19 +11,11 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -alter table catalogdb.gaia_dr2_source add primary key(source_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (l); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (b); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ecl_lon); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (ecl_lat); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (phot_g_mean_flux); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (phot_g_mean_mag); -CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source using BTREE (solution_id); - - -create index on catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); -CLUSTER gaia_dr2_source_q3c_ang2ipix_idx on catalogdb.gaia_dr2_source; -analyze catalogdb.gaia_dr2_source; \ No newline at end of file +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_flux); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (phot_g_mean_mag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source USING BTREE (solution_id); + + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_source (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_source_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_source; +ANALYZE catalogdb.gaia_dr2_source; diff --git a/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql new file mode 100644 index 00000000..853e9910 --- /dev/null +++ b/schema/sdss5db/catalogdb/gaia_qso/gaia_qso.sql @@ -0,0 +1,65 @@ +/* + +Title: Compilation of known QSOs for the Gaia mission (Liao+, 2019) +URL: http://vizier.u-strasbg.fr/viz-bin/VizieR?-source=J/other/RAA/19.29 + +The ;-separated file downloaded from Vizier has been modified by removing +all the comments except the header and then opening and saving it with +Pandas using the skipinitialspace=True option. + +Columns: + +Name (a23) Name (1) +RAJ2000 (F15.11) Right ascension (J2000.0) (2) +DEJ2000 (F15.11) Declination (J2000.0) (2) +e_RAJ2000 (F13.11) Error in Right ascension +e_DEJ2000 (F13.11) Error in Declination +z (F13.10) Redshift (3) +umag (F11.8) u photographic magnitude (4) +gmag (F11.8) g photographic magnitude (4) +rmag (F11.8) r photographic magnitude (4) +imag (F11.8) i photographic magnitude (4) +zmag (F11.8) z photographic magnitude (4) +Ref (a15) Reference catalog (5) +W1-W2 (F5.2) W1-W2 photographic magnitude (6) +W2-W3 (F5.2) W2-W3 photographic magnitude (6) +W1mag (F5.2) W1 photographic magnitude (6) + +*/ + +CREATE TABLE catalogdb.gaia_qso ( + name TEXT, + raj2000 DOUBLE PRECISION, + dej2000 DOUBLE PRECISION, + e_raj2000 DOUBLE PRECISION, + e_dej2000 DOUBLE PRECISION, + z DOUBLE PRECISION, + umag REAL, + gmag REAL, + rmag REAL, + imag REAL, + zmag REAL, + ref TEXT, + w1_w2 REAL, + w2_w3 REAL, + w1mag REAL +) WITHOUT OIDS; + +\copy catalogdb.gaia_qso FROM PROGRAM 'cat $CATALOGDB_DIR/gaia_qso/Gaia_QSO_Liao_2019.csv' WITH CSV HEADER; + +ALTER TABLE catalogdb.gaia_qso ADD COLUMN pk BIGSERIAL; +ALTER TABLE catalogdb.gaia_qso ADD PRIMARY KEY (pk); + + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso (q3c_ang2ipix(raj2000, dej2000)); +CLUSTER gaia_qso_q3c_ang2ipix_idx ON catalogdb.gaia_qso; +ANALYZE catalogdb.gaia_qso; + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w1_w2); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w2_w3); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_qso USING BTREE (w1mag); diff --git a/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql b/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql new file mode 100644 index 00000000..5d683b42 --- /dev/null +++ b/schema/sdss5db/catalogdb/geometric_distances/geometric_distances.sql @@ -0,0 +1,25 @@ +/* + +Geometric distances from Bailer-Jones Gaia DR2 + +*/ + + +CREATE TABLE catalogdb.geometric_distances_gaia_dr2 ( + source_id BIGINT, + r_est REAL, + r_lo REAL, + r_hi REAL, + r_len REAL, + result_flag CHAR(1), + modality_flag SMALLINT +) WITHOUT OIDS; + +\COPY catalogdb.geometric_distances_gaia_dr2 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/geometric_distances/cbj_geometric_distances.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.geometric_distances_gaia_dr2 ADD PRIMARY KEY (source_id); + +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_est); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_lo); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_hi); +CREATE INDEX CONCURRENTLY ON catalogdb.geometric_distances_gaia_dr2 (r_len); diff --git a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql index aad425d0..d4b34295 100644 --- a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql +++ b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat.sql @@ -114,7 +114,7 @@ columns below are order in which they appearn in csvs */ CREATE TABLE catalogdb.guvcat( - OBJID bigint, + OBJID bigint PRIMARY KEY, PHOTOEXTRACTID bigint, MPSTYPE text, AVASPRA double precision, @@ -210,8 +210,3 @@ CREATE TABLE catalogdb.guvcat( INLARGEOBJ text, LARGEOBJSIZE real ); - - - - - diff --git a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql index 0e969642..c443fab0 100644 --- a/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql +++ b/schema/sdss5db/catalogdb/guvcat/fov055/guvcat_index.sql @@ -11,9 +11,7 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -CREATE INDEX CONCURRENTLY ON catalogdb.guvcat using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.guvcat using BTREE (dec); - -alter table catalogdb.guvcat add primary key(objid); - +CREATE INDEX CONCURRENTLY ON catalogdb.guvcat (q3c_ang2ipix(ra, dec)); +CLUSTER guvcat_q3c_ang2ipix_idx ON catalogdb.guvcat; +ANALYZE catalogdb.guvcat; diff --git a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql index 3292079b..db8ad8f5 100644 --- a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql +++ b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10.sql @@ -28,7 +28,7 @@ CREATE TABLE catalogdb.kepler_input_10 ( kic_hmag double precision, kic_kmag double precision, kic_kepmag double precision, - kic_kepler_id bigint, + kic_kepler_id bigint PRIMARY KEY, kic_tmid bigint, kic_scpid bigint, kic_altid bigint, diff --git a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql index 06e0fc80..b92438e7 100644 --- a/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql +++ b/schema/sdss5db/catalogdb/kic/v10/kepler_input_10_index.sql @@ -11,21 +11,20 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; -- Indices -alter table catalogdb.kepler_input_10 add primary key(kic_kepler_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_ra); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_dec); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_umag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_gmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_rmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_imag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_zmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_jmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_hmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_kmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_kepmag); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_glon); -CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 using BTREE (kic_glat); - - - +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 (q3c_ang2ipix(kic_ra, kic_dec)); +CLUSTER kepler_input_10_q3c_ang2ipix_idx ON catalogdb.kepler_input_10; +ANALYZE catalogdb.kepler_input_10; + +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_ra); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_dec); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_umag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_imag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_kmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_kepmag); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_glon); +CREATE INDEX CONCURRENTLY ON catalogdb.kepler_input_10 USING BTREE (kic_glat); diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql new file mode 100644 index 00000000..01fbc9e4 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey.sql @@ -0,0 +1,131 @@ +/* + +Schema for Legacy Survey. + +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/legacysurvey/dr8 + +*/ + +CREATE TABLE catalogdb.legacy_survey_dr8 ( + release INTEGER, + brickid BIGINT, + brickname TEXT, + objid BIGINT, + type TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + ra_ivar REAL, + dec_ivar REAL, + dchisq REAL[5], + ebv REAL, + flux_g REAL, + flux_r REAL, + flux_z REAL, + flux_w1 REAL, + flux_w2 REAL, + flux_w3 REAL, + flux_w4 REAL, + flux_ivar_g REAL, + flux_ivar_r REAL, + flux_ivar_z REAL, + flux_ivar_w1 REAL, + flux_ivar_w2 REAL, + flux_ivar_w3 REAL, + flux_ivar_w4 REAL, + mw_transmission_g REAL, + mw_transmission_r REAL, + mw_transmission_z REAL, + mw_transmission_w1 REAL, + mw_transmission_w2 REAL, + mw_transmission_w3 REAL, + mw_transmission_w4 REAL, + nobs_g INTEGER, + nobs_r INTEGER, + nobs_z INTEGER, + nobs_w1 INTEGER, + nobs_w2 INTEGER, + nobs_w3 INTEGER, + nobs_w4 INTEGER, + rchisq_g REAL, + rchisq_r REAL, + rchisq_z REAL, + rchisq_w1 REAL, + rchisq_w2 REAL, + rchisq_w3 REAL, + rchisq_w4 REAL, + fracflux_g REAL, + fracflux_r REAL, + fracflux_z REAL, + fracflux_w1 REAL, + fracflux_w2 REAL, + fracflux_w3 REAL, + fracflux_w4 REAL, + fracmasked_g REAL, + fracmasked_r REAL, + fracmasked_z REAL, + fracin_g REAL, + fracin_r REAL, + fracin_z REAL, + anymask_g INTEGER, + anymask_r INTEGER, + anymask_z INTEGER, + allmask_g INTEGER, + allmask_r INTEGER, + allmask_z INTEGER, + wisemask_w1 SMALLINT, + wisemask_w2 SMALLINT, + psfsize_g REAL, + psfsize_r REAL, + psfsize_z REAL, + psfdepth_g REAL, + psfdepth_r REAL, + psfdepth_z REAL, + galdepth_g REAL, + galdepth_r REAL, + galdepth_z REAL, + psfdepth_w1 REAL, + psfdepth_w2 REAL, + wise_coadd_id TEXT, + fracdev REAL, + fracdev_ivar REAL, + shapedev_r REAL, + shapedev_r_ivar REAL, + shapedev_e1 REAL, + shapedev_e1_ivar REAL, + shapedev_e2 REAL, + shapedev_e2_ivar REAL, + shapeexp_r REAL, + shapeexp_r_ivar REAL, + shapeexp_e1 REAL, + shapeexp_e1_ivar REAL, + shapeexp_e2 REAL, + shapeexp_e2_ivar REAL, + fiberflux_g REAL, + fiberflux_r REAL, + fiberflux_z REAL, + fibertotflux_g REAL, + fibertotflux_r REAL, + fibertotflux_z REAL, + ref_cat TEXT, + ref_id BIGINT, + ref_epoch REAL, + gaia_phot_g_mean_mag REAL, + gaia_phot_g_mean_flux_over_error REAL, + gaia_phot_bp_mean_mag REAL, + gaia_phot_bp_mean_flux_over_error REAL, + gaia_phot_rp_mean_mag REAL, + gaia_phot_rp_mean_flux_over_error REAL, + gaia_astrometric_excess_noise REAL, + gaia_duplicated_source BOOL, + gaia_phot_bp_rp_excess_factor REAL, + gaia_astrometric_sigma5d_max REAL, + gaia_astrometric_params_solved SMALLINT, + parallax REAL, + parallax_ivar REAL, + pmra REAL, + pmra_ivar REAL, + pmdec REAL, + pmdec_ivar REAL, + maskbits INTEGER, + ls_id BIGINT GENERATED ALWAYS AS (objid + brickid * 2^16 + release * 2^40) STORED +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_generate_csv.py b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_generate_csv.py new file mode 100755 index 00000000..d02afcb9 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_generate_csv.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-12 +# @Filename: legacy_survey_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import glob +import multiprocessing +import sys + +import astropy.table +import progressbar + +from sdssdb.utils.ingest import to_csv + + +def convert_to_csv(file_): + + data = astropy.table.Table.read(file_, memmap=True) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True, convert_arrays=True, overwrite=True) + del data + + +if __name__ == '__main__': + + if len(sys.argv) > 1: + convert_to_csv(sys.argv[1]) + sys.exit(0) + + files = glob.glob('sweep*.fits') + + with multiprocessing.Pool(10) as pool: + + for _ in progressbar.progressbar(pool.imap_unordered(convert_to_csv, files), + max_value=len(files), poll_interval=1): + pass diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql new file mode 100644 index 00000000..3b652f56 --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_index.sql @@ -0,0 +1,15 @@ + +ALTER TABLE catalogdb.legacy_survey_dr8 ADD PRIMARY KEY (ls_id); + +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (ref_id); +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (ref_cat); + +-- Create new column with the source_id for Gaia where ref_cat=G2 +ALTER TABLE catalogdb.legacy_survey_dr8 ADD COLUMN gaia_sourceid BIGINT; +UPDATE catalogdb.legacy_survey_dr8 SET gaia_sourceid = ref_id + WHERE ref_cat = 'G2'; +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (gaia_sourceid); + +CREATE INDEX CONCURRENTLY ON catalogdb.legacy_survey_dr8 (q3c_ang2ipix(ra, dec)); +CLUSTER legacy_survey_dr8_q3c_ang2ipix_idx ON catalogdb.legacy_survey_dr8; +VACUUM ANALYZE catalogdb.legacy_survey_dr8; diff --git a/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load new file mode 100755 index 00000000..8291c4fd --- /dev/null +++ b/schema/sdss5db/catalogdb/legacy_survey/dr8/legacy_survey_load @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# encoding: utf-8 + +ls $CATALOGDB_DIR/legacysurvey/dr8/north/sweep/*.fits.csv | xargs -I {} sh -c "echo {} && psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM {} WITH HEADER DELIMITER ',' NULL '\N' CSV;\"" +ls $CATALOGDB_DIR/legacysurvey/dr8/south/sweep/*.fits.csv | xargs -I {} sh -c "echo {} && psql -U sdss sdss5db -c \"\copy catalogdb.legacy_survey_dr8 FROM {} WITH HEADER DELIMITER ',' NULL '\N' CSV;\"" diff --git a/schema/sdss5db/catalogdb/mipsgal/mipsgal.py b/schema/sdss5db/catalogdb/mipsgal/mipsgal.py new file mode 100755 index 00000000..8363a8c9 --- /dev/null +++ b/schema/sdss5db/catalogdb/mipsgal/mipsgal.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: mipsgal.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/MIPSGAL.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'mipsgal', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql new file mode 100644 index 00000000..e9b2df0b --- /dev/null +++ b/schema/sdss5db/catalogdb/mipsgal/mipsgal.sql @@ -0,0 +1,77 @@ +/* + +MIPSGAL - https://irsa.ipac.caltech.edu/data/SPITZER/MIPSGAL/ + +*/ + +CREATE TABLE catalogdb.mipsgal ( + mipsgal VARCHAR(18) PRIMARY KEY, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + s24 DOUBLE PRECISION, + e_s24 DOUBLE PRECISION, + mag_24 DOUBLE PRECISION, + e_mag_24 DOUBLE PRECISION, + twomass_name VARCHAR(17), + sj DOUBLE PRECISION, + e_sj DOUBLE PRECISION, + sh DOUBLE PRECISION, + e_sh DOUBLE PRECISION, + sk DOUBLE PRECISION, + e_sk DOUBLE PRECISION, + glimpse VARCHAR(17), + s3_6 DOUBLE PRECISION, + e_s3_6 DOUBLE PRECISION, + s4_5 DOUBLE PRECISION, + e_s4_5 DOUBLE PRECISION, + s5_8 DOUBLE PRECISION, + e_s5_8 DOUBLE PRECISION, + s8_0 DOUBLE PRECISION, + wise VARCHAR(19), + s3_4 DOUBLE PRECISION, + e_s3_4 DOUBLE PRECISION, + s4_6 DOUBLE PRECISION, + e_s4_6 DOUBLE PRECISION, + s12 DOUBLE PRECISION, + e_s12 DOUBLE PRECISION, + s22 DOUBLE PRECISION, + e_s22 DOUBLE PRECISION, + jmag DOUBLE PRECISION, + e_jmag DOUBLE PRECISION, + hmag DOUBLE PRECISION, + e_hmag DOUBLE PRECISION, + kmag DOUBLE PRECISION, + e_kmag DOUBLE PRECISION, + mag_3_6 DOUBLE PRECISION, + e_mag_3_6 DOUBLE PRECISION, + mag_4_5 DOUBLE PRECISION, + e_mag_4_5 DOUBLE PRECISION, + mag_5_8 DOUBLE PRECISION, + e_mag_5_8 DOUBLE PRECISION, + mag_8_0 DOUBLE PRECISION, + e_mag_8_0 DOUBLE PRECISION, + w1mag DOUBLE PRECISION, + e_w1mag DOUBLE PRECISION, + w2mag DOUBLE PRECISION, + e_w2mag DOUBLE PRECISION, + w3mag DOUBLE PRECISION, + e_w3mag DOUBLE PRECISION, + w4mag DOUBLE PRECISION, + e_w4mag DOUBLE PRECISION, + dnn DOUBLE PRECISION, + ng INTEGER, + n2m INTEGER, + nw INTEGER, + fwhm DOUBLE PRECISION, + sky DOUBLE PRECISION, + lim1 DOUBLE PRECISION, + lim2 DOUBLE PRECISION, + f1 INTEGER, + f2 INTEGER +); + +CREATE INDEX CONCURRENTLY ON catalogdb.mipsgal (q3c_ang2ipix(radeg, dedeg)); +CLUSTER mipsgal_q3c_ang2ipix_idx ON catalogdb.mipsgal; +ANALYZE catalogdb.mipsgal; diff --git a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py new file mode 100755 index 00000000..07c1ba64 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: atnf.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/atnfpsr_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'atnf', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql new file mode 100644 index 00000000..29700293 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/atnf/atnf.sql @@ -0,0 +1,32 @@ +/* + +ATNF Pulsar Catalog https://bit.ly/3aajZF7 + +*/ + +CREATE TABLE catalogdb.atnf ( + name TEXT PRIMARY KEY, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT +); + +CREATE INDEX CONCURRENTLY ON catalogdb.atnf (q3c_ang2ipix(radeg, decdeg)); +CLUSTER atnf_q3c_ang2ipix_idx ON catalogdb.atnf; +ANALYZE catalogdb.atnf; + +CREATE INDEX CONCURRENTLY ON catalogdb.atnf USING BTREE (gaia_source_id); + +ALTER TABLE catalogdb.atnf + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py b/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py new file mode 100755 index 00000000..fc169b68 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/blackcat/backcat.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: blackcat.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/blackcat_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'blackcat', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql new file mode 100644 index 00000000..206a3c82 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/blackcat/blackcat.sql @@ -0,0 +1,30 @@ +/* + +BlackCAT Stellar-Mass Black Holes in X-ray Binaries http://www.astro.puc.cl/BlackCAT/ + +*/ + +CREATE TABLE catalogdb.blackcat ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.blackcat (q3c_ang2ipix(radeg, decdeg)); +CLUSTER blackcat_q3c_ang2ipix_idx ON catalogdb.blackcat; +ANALYZE catalogdb.blackcat; + +ALTER TABLE catalogdb.blackcat + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py new file mode 100755 index 00000000..b8a65884 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: galactic_millisecond_pulsars.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/msp_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'galactic_millisecond_pulsars', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql new file mode 100644 index 00000000..859db3a9 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/galactic_millisecond_pulsars/galactic_millisecond_pulsars.sql @@ -0,0 +1,30 @@ +/* + +Galactic Millisecond Pulsar Catalog https://bit.ly/33NXK5C + +*/ + +CREATE TABLE catalogdb.galactic_millisecond_pulsars ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.galactic_millisecond_pulsars (q3c_ang2ipix(radeg, decdeg)); +CLUSTER galactic_millisecond_pulsars_q3c_ang2ipix_idx ON catalogdb.galactic_millisecond_pulsars; +ANALYZE catalogdb.galactic_millisecond_pulsars; + +ALTER TABLE catalogdb.galactic_millisecond_pulsars + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py new file mode 100755 index 00000000..117ed4c6 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: lmxb.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/lmxb_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'lmxb', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql new file mode 100644 index 00000000..35154d07 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/lmxb/lmxb.sql @@ -0,0 +1,30 @@ +/* + +Ritter & Kolb Catalog of Low Mass X-ray Binaries https://bit.ly/3agEcZT + +*/ + +CREATE TABLE catalogdb.lmxb ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.lmxb (q3c_ang2ipix(radeg, decdeg)); +CLUSTER lmxb_q3c_ang2ipix_idx ON catalogdb.lmxb; +ANALYZE catalogdb.lmxb; + +ALTER TABLE catalogdb.lmxb + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql new file mode 100644 index 00000000..95481382 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps.sql @@ -0,0 +1,30 @@ +/* + +Accreting X-ray Binary Pulsars http://www.iasfbo.inaf.it/~mauro/pulsar_list.html + +*/ + +CREATE TABLE catalogdb.transitional_msps ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.transitional_msps (q3c_ang2ipix(radeg, decdeg)); +CLUSTER transitional_msps_q3c_ang2ipix_idx ON catalogdb.transitional_msps; +ANALYZE catalogdb.transitional_msps; + +ALTER TABLE catalogdb.transitional_msps + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py new file mode 100755 index 00000000..f6ae7f33 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/transitional_msps/transitional_msps_load.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: transitional_msps.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/slavko_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'transitional_msps', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py new file mode 100755 index 00000000..501f3057 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: watchdog.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/watchdog_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'watchdog', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql new file mode 100644 index 00000000..0492c647 --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/watchdog/watchdog.sql @@ -0,0 +1,30 @@ +/* + +WATCHDOG Galactic Black Hole Binaries (B. E. Tetarenko et al 2016 ApJS 222 15) + +*/ + +CREATE TABLE catalogdb.watchdog ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.watchdog (q3c_ang2ipix(radeg, decdeg)); +CLUSTER watchdog_q3c_ang2ipix_idx ON catalogdb.watchdog; +ANALYZE catalogdb.watchdog; + +ALTER TABLE catalogdb.watchdog + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py new file mode 100755 index 00000000..7af1a39e --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: xray_pulsars.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/mwm_small/xraypsr_good.fits' + + data = astropy.table.Table.read(file_) + source_id = list(map(lambda x: int(x.split()[2]), data['GAIA_NAME'])) + data.add_column(astropy.table.Column(source_id, 'gaia_source_id')) + + copy_data(data, database, 'xray_pulsars', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql new file mode 100644 index 00000000..84b362bb --- /dev/null +++ b/schema/sdss5db/catalogdb/mwm_small/xray_pulsars/xray_pulsars.sql @@ -0,0 +1,30 @@ +/* + +Accreting X-ray Binary Pulsars http://www.iasfbo.inaf.it/~mauro/pulsar_list.html + +*/ + +CREATE TABLE catalogdb.xray_pulsars ( + name TEXT, + rastr TEXT, + decstr TEXT, + radeg DOUBLE PRECISION, + decdeg DOUBLE PRECISION, + ngaia REAL, + gaia_name TEXT, + gaia_ra DOUBLE PRECISION, + gaia_dec DOUBLE PRECISION, + gaia_gmag REAL, + gaia_dist REAL, + gaia_source_id BIGINT PRIMARY KEY +); + +CREATE INDEX CONCURRENTLY ON catalogdb.xray_pulsars (q3c_ang2ipix(radeg, decdeg)); +CLUSTER xray_pulsars_q3c_ang2ipix_idx ON catalogdb.xray_pulsars; +ANALYZE catalogdb.xray_pulsars; + +ALTER TABLE catalogdb.xray_pulsars + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql new file mode 100644 index 00000000..d491494b --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge.sql @@ -0,0 +1,84 @@ +/* + +SDSS AllStarMerge r13 + +Provided by Nicholas Troup - https://bit.ly/39emJ31 + +At Utah: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/sdssApogeeAllStarMerge/r13/ +Catalogue generated via IDL script: aporbitprep_mergestar.pro + +*/ + +CREATE TABLE catalogdb.sdss_apogeeAllStarMerge_r13 ( + apogee_id VARCHAR(18) PRIMARY KEY, + nvisits SMALLINT, + nentries INTEGER, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + pm_src VARCHAR(4), + j REAL, + j_err REAL, + h REAL, + h_err REAL, + k REAL, + k_err REAL, + ak REAL, + vhelio_avg REAL, + vhelio_err REAL, + vscatter REAL, + sig_rvvar REAL, + baseline REAL, + mean_fiber REAL, + sig_fiber REAL, + apstar_ids TEXT, + nvisit_apstar SMALLINT[20], + weights REAL[20], + visits TEXT, + fields TEXT, + surveys TEXT, + telescopes TEXT, + location_id SMALLINT[20], + commiss SMALLINT[20], + targflags TEXT, + starflags TEXT, + aspcapflags TEXT, + teff REAL, + teff_err REAL, + logg REAL, + logg_err REAL, + feh REAL, + feh_err REAL, + param REAL[9], + param_cov REAL[9][9], + fparam REAL[9], + fparam_cov REAL[9][9], + param_symbol VARCHAR(15)[9], + x_h REAL[26], + x_h_err REAL[26], + felem REAL[26], + felem_err REAL[26], + elem_symbol VARCHAR(4)[26], + felemtoh SMALLINT[31], + startype VARCHAR(1), + vjitter REAL, + dist REAL, + dist_err REAL, + dist_src VARCHAR(1), + dist_srclist VARCHAR(1), + mstar REAL, + mstar_err REAL, + rstar REAL, + rstar_err REAL, + mstar_src VARCHAR(1), + jd DOUBLE PRECISION[60], + rv DOUBLE PRECISION[60], + rv_err DOUBLE PRECISION[60], + rv_src VARCHAR(6)[60], + snr REAL[60], + fiberid SMALLINT[60], + visit_pk INTEGER[60] +); diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql new file mode 100644 index 00000000..58b300ba --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_index.sql @@ -0,0 +1,10 @@ + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; +ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; + +CREATE INDEX sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13 (q3c_ang2ipix(glon, glat)); +CLUSTER sdss_apogeeAllStarMerge_r13_q3c_ang2ipix_gal_idx ON catalogdb.sdss_apogeeAllStarMerge_r13; +ANALYZE catalogdb.sdss_apogeeAllStarMerge_r13; + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_apogeeAllStarMerge_r13 USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py new file mode 100755 index 00000000..a260ad69 --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssApogeeAllStarMerge/r13/AllStarMerge_load.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-13 +# @Filename: AllStarMerge_load.py.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import to_csv + + +assert database.connected + + +def main(): + + file_ = os.environ['CATALOGDB_DIR'] + 'sdssApogeeAllStarMerge/r13/allStarMerge-r13-l33-58814.fits' # noqa + + data = astropy.table.Table.read(file_) + data.meta = {} + data.rename_columns(data.colnames, list(map(lambda x: x.lower(), data.colnames))) + to_csv(data, file_ + '.csv', header=True) + del data + + cursor = database.cursor() + fileobj = open(file_ + '.csv') + fileobj.readline() # Read header + cursor.copy_from(fileobj, 'catalogdb.sdss_apogeeAllStarMerge_r13', sep='\t') + database.commit() + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql index 879bd578..56793980 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStar/dr14/sdss_dr14_apogeeStar.sql @@ -58,7 +58,7 @@ htmID bigint 8 HTM ID */ CREATE TABLE catalogdb.sdss_dr14_apogeeStar( - apstar_id varchar(64), + apstar_id varchar(64) PRIMARY KEY, target_id varchar(64), reduction_id varchar(32), file varchar(128), @@ -112,11 +112,13 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStar( \copy catalogdb.sdss_dr14_apogeeStar FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeStar/dr14/src/sqlApogeeStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeStar add primary key(apstar_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (glon); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar using BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_apogeeStar_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeStar; +ANALYZE catalogdb.sdss_dr14_apogeeStar; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (glat); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql index 3890b098..c83a3b1d 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeStarVisit/dr14/sdss_dr14_apogeeStarVisit.sql @@ -20,8 +20,7 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeStarVisit( \copy catalogdb.sdss_dr14_apogeeStarVisit FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeStarVisit/dr14/src/sqlApogeeStarVisit.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeStarVisit add primary key(visit_id); +ALTER TABLE catalogdb.sdss_dr14_apogeeStarVisit ADD COLUMN pk BIGSERIAL PRIMARY KEY; CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (apstar_id); - - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeStarVisit using BTREE (visit_id); diff --git a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql index e46999c7..52b10a37 100644 --- a/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql +++ b/schema/sdss5db/catalogdb/sdssApogeeVisit/dr14/sdss_dr14_apogeeVisit.sql @@ -1,6 +1,6 @@ /* -schema for dr 14 apogeeStar table. +schema for dr 14 apogeeVisit table. Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx @@ -60,7 +60,7 @@ synthvhelio real 4 km/s Heliocentric radial velocity from cross-corr */ CREATE TABLE catalogdb.sdss_dr14_apogeeVisit( - visit_id varchar(64), + visit_id varchar(64) PRIMARY KEY, apred_version varchar(32), apogee_id varchar(64), target_id varchar(64), @@ -115,11 +115,11 @@ CREATE TABLE catalogdb.sdss_dr14_apogeeVisit( \copy catalogdb.sdss_dr14_apogeeVisit FROM program 'bzcat $CATALOGDB_DIR/sdssApogeeVisit/dr14/src/sqlApogeeVisit.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_apogeeVisit add primary key(visit_id); - -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (glon); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit using BTREE (glat); - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_apogeeVisit_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_apogeeVisit; +ANALYZE catalogdb.sdss_dr14_apogeeVisit; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (ra); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (dec); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glon); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_apogeeVisit USING BTREE (glat); diff --git a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql index 1d82eda2..5e9c824a 100644 --- a/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql +++ b/schema/sdss5db/catalogdb/sdssAscapStar/dr14/sdss_dr14_ascapStar.sql @@ -171,7 +171,7 @@ felem_nd_h_err real 4 dex FERRE pipeline uncalibratied ratio original CREATE TABLE catalogdb.sdss_dr14_ascapStar( apstar_id varchar(64), target_id varchar(64), - aspcap_id varchar(64), + aspcap_id varchar(64) PRIMARY KEY, apogee_id varchar(32), aspcap_version varchar(32), results_version varchar(32), @@ -331,12 +331,7 @@ CREATE TABLE catalogdb.sdss_dr14_ascapStar( felem_nd_h_err real ); - \copy catalogdb.sdss_dr14_ascapStar FROM program 'bzcat $CATALOGDB_DIR/sdssAscapStar/dr14/src/sqlaspcapStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_ascapStar add primary key(apstar_id); - -CREATE INDEX CONCURRENTLY sdss_dr14_ascapStar_target_id_index ON catalogdb.sdss_dr14_ascapStar using BTREE (target_id); -CREATE INDEX CONCURRENTLY sdss_dr14_ascapStar_apogee_id_index ON catalogdb.sdss_dr14_ascapStar using BTREE (apogee_id); - - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_ascapStar USING BTREE (target_id); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_ascapStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql index e3719be5..56b21882 100644 --- a/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql +++ b/schema/sdss5db/catalogdb/sdssCannonStar/dr14/sdss_dr14_cannonStar.sql @@ -89,7 +89,7 @@ r_chi_sq real 4 reduced chi^2 from Cannon analysis CREATE TABLE catalogdb.sdss_dr14_cannonStar( apogee_id varchar(32), - cannon_id varchar(64), + cannon_id varchar(64) PRIMARY KEY, filename varchar(128), location_id bigint, field varchar(128), @@ -169,11 +169,6 @@ CREATE TABLE catalogdb.sdss_dr14_cannonStar( r_chi_sq real ); - \copy catalogdb.sdss_dr14_cannonStar FROM program 'bzcat $CATALOGDB_DIR/sdssCannonStar/dr14/src/sqlcannonStar.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_cannonStar add primary key(cannon_id); - -CREATE INDEX CONCURRENTLY sdss_dr14_cannonStar_apogee_id_index ON catalogdb.sdss_dr14_cannonStar using BTREE (apogee_id); - - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_cannonStar USING BTREE (apogee_id); diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql index 45cb0177..a653a195 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj.sql @@ -521,7 +521,7 @@ TAI_z */ CREATE TABLE catalogdb.sdss_dr13_photoobj( - objID bigint, + objID bigint PRIMARY KEY, run smallint, rerun smallint, camcol smallint, @@ -735,8 +735,3 @@ CREATE TABLE catalogdb.sdss_dr13_photoobj( b double precision, l double precision ); - - - - - diff --git a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql index 56439e1e..6e05baa0 100644 --- a/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql +++ b/schema/sdss5db/catalogdb/sdssPhotoObj/dr13/sdss_dr13_photoobj_index.sql @@ -3,19 +3,14 @@ indices for catalogdb tables, to be run after bulk uploads */ -alter table catalogdb.sdss_dr13_photoobj add primary key(objid); - -- Indices -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (dec); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (l); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (b); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_u); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_g); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_r); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_i); -CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj using BTREE (psfmag_z); - - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr13_photoobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr13_photoobj; +VACUUM ANALYZE catalogdb.sdss_dr13_photoobj; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_u); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_g); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_r); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_i); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr13_photoobj USING BTREE (psfmag_z); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql index aa68de15..54f481e7 100644 --- a/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr14/sdss_dr14_specObj.sql @@ -205,7 +205,7 @@ img varbinary -1 IMAGE? Spectrum Image */ CREATE TABLE catalogdb.sdss_dr14_specobj( - specObjID numeric(20), + specObjID numeric(20) PRIMARY KEY, bestObjID bigint, fluxObjID bigint, targetObjID bigint, @@ -404,14 +404,11 @@ CREATE TABLE catalogdb.sdss_dr14_specobj( \copy catalogdb.sdss_dr14_specobj FROM program 'bzcat $CATALOGDB_DIR/sdssSpecObj/dr14/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; -alter table catalogdb.sdss_dr14_specobj add primary key(specObjID); - -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_bestObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (bestObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_fluxObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (fluxObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_targetObjID_index ON catalogdb.sdss_dr14_specobj using BTREE (targetObjID); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_ra_index ON catalogdb.sdss_dr14_specobj using BTREE (ra); -CREATE INDEX CONCURRENTLY sdss_dr14_sdss_dr14_specobj_dec_index ON catalogdb.sdss_dr14_specobj using BTREE (dec); - - - +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_specobj; +ANALYZE catalogdb.sdss_dr14_specobj; +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj USING BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj USING BTREE (targetObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql new file mode 100644 index 00000000..20e072ee --- /dev/null +++ b/schema/sdss5db/catalogdb/sdssSpecObj/dr16/sdss_dr16_specObj.sql @@ -0,0 +1,415 @@ +/* + +Schema for sdssSpecObj DR16 table. + +Model can be found at http://skyserver.sdss.org/CasJobs/SchemaBrowser.aspx + +File is /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj + +specObjID decimal 16 ID_CATALOG Unique database ID based on PLATE, MJD, FIBERID, RUN2D +bestObjID bigint 8 ID_MAIN Object ID of photoObj match (position-based) +fluxObjID bigint 8 ID_MAIN Object ID of photoObj match (flux-based) +targetObjID bigint 8 ID_CATALOG Object ID of original target +plateID decimal 16 Database ID of Plate +sciencePrimary smallint 2 Best version of spectrum at this location (defines default view SpecObj) +sdssPrimary smallint 2 Best version of spectrum at this location among SDSS plates (defines default view SpecObj) +legacyPrimary smallint 2 Best version of spectrum at this location, among Legacy plates +seguePrimary smallint 2 Best version of spectrum at this location, among SEGUE plates +segue1Primary smallint 2 Best version of spectrum at this location, among SEGUE-1 plates +segue2Primary smallint 2 Best version of spectrum at this location, among SEGUE-2 plates +bossPrimary smallint 2 Best version of spectrum at this location, among BOSS plates +bossSpecObjID int 4 Index of BOSS observation in spAll flat file +firstRelease varchar 32 Name of first release this object was associated with +survey varchar 32 Survey name +instrument varchar 32 Instrument used (SDSS or BOSS spectrograph) +programname varchar 32 Program name +chunk varchar 32 Chunk name +platerun varchar 32 Plate drill run name +mjd int 4 days MJD of observation +plate smallint 2 Plate number +fiberID smallint 2 Fiber ID +run1d varchar 32 1D Reduction version of spectrum +run2d varchar 32 2D Reduction version of spectrum +tile int 4 Tile number +designID int 4 Design ID number +legacy_target1 bigint 8 for Legacy program, target selection information at plate design +legacy_target2 bigint 8 for Legacy program target selection information at plate design, secondary/qa/calibration +special_target1 bigint 8 for Special program target selection information at plate design +special_target2 bigint 8 for Special program target selection information at plate design, secondary/qa/calibration +segue1_target1 bigint 8 SEGUE-1 target selection information at plate design, primary science selection +segue1_target2 bigint 8 SEGUE-1 target selection information at plate design, secondary/qa/calib selection +segue2_target1 bigint 8 SEGUE-2 target selection information at plate design, primary science selection +segue2_target2 bigint 8 SEGUE-2 target selection information at plate design, secondary/qa/calib selection +boss_target1 bigint 8 BOSS target selection information at plate +eboss_target0 bigint 8 EBOSS target selection information, for SEQUELS plates +eboss_target1 bigint 8 EBOSS target selection information, for eBOSS plates +eboss_target2 bigint 8 EBOSS target selection information, for TDSS, SPIDERS, ELG, etc. plates +eboss_target_id bigint 8 EBOSS unique target identifier for every spectroscopic target, +ancillary_target1 bigint 8 BOSS ancillary science target selection information at plate design +ancillary_target2 bigint 8 BOSS ancillary target selection information at plate design +thing_id_targeting bigint 8 thing_id value from the version of resolve from which the targeting was created +thing_id int 4 Unique identifier from global resolve +primTarget bigint 8 target selection information at plate design, primary science selection (for backwards compatibility) +secTarget bigint 8 target selection information at plate design, secondary/qa/calib selection (for backwards compatibility) +spectrographID smallint 2 which spectrograph (1,2) +sourceType varchar 128 For Legacy, SEGUE-2 and BOSS science targets, type of object targeted as (target bits contain full information and are recommended) +targetType varchar 128 Nature of target: SCIENCE, STANDARD, or SKY +ra float 8 deg DR8 Right ascension of fiber, J2000 +dec float 8 deg DR8 Declination of fiber, J2000 +cx float 8 POS_EQ_CART_X x of Normal unit vector in J2000 +cy float 8 POS_EQ_CART_Y y of Normal unit vector in J2000 +cz float 8 POS_EQ_CART_Z z of Normal unit vector in J2000 +xFocal real 4 mm X focal plane position (+RA direction) +yFocal real 4 mm Y focal plane position (+Dec direction) +lambdaEff real 4 Angstroms Effective wavelength that hole was drilled for (accounting for atmopheric refraction) +blueFiber int 4 Set to 1 if this hole was designated a "blue fiber", 0 if designated a "red fiber" (high redshift LRGs are preferentially in "red fibers") +zOffset real 4 microns Washer thickness used (for backstopping BOSS quasar targets, so they are closer to 4000 Angstrom focal plan) +z real 4 Final Redshift +zErr real 4 Redshift error +zWarning int 4 Bitmask of warning values; 0 means all is well +class varchar 32 Spectroscopic class (GALAXY, QSO, or STAR) +subClass varchar 32 Spectroscopic subclass +rChi2 real 4 Reduced chi-squared of best fit +DOF real 4 Degrees of freedom in best fit +rChi2Diff real 4 Difference in reduced chi-squared between best and second best fit +z_noqso real 4 Best redshift when excluding QSO fit in BOSS spectra (right redshift to use for galaxy targets) +zErr_noqso real 4 Error in "z_noqso" redshift (BOSS spectra only) +zWarning_noqso int 4 Warnings in "z_noqso" redshift (BOSS spectra only) +class_noqso varchar 32 Classification in "z_noqso" redshift +subClass_noqso varchar 32 Sub-classification in "z_noqso" redshift +rChi2Diff_noqso real 4 Reduced chi-squared difference from next best redshift, for "z_noqso" redshift +z_person real 4 Person-assigned redshift, if this object has been inspected +class_person varchar 32 Person-assigned classification, if this object has been inspected +comments_person varchar 200 Comments from person for inspected objects +tFile varchar 32 File name of best fit template source +tColumn_0 smallint 2 Which column of the template file corresponds to template #0 +tColumn_1 smallint 2 Which column of the template file corresponds to template #1 +tColumn_2 smallint 2 Which column of the template file corresponds to template #2 +tColumn_3 smallint 2 Which column of the template file corresponds to template #3 +tColumn_4 smallint 2 Which column of the template file corresponds to template #4 +tColumn_5 smallint 2 Which column of the template file corresponds to template #5 +tColumn_6 smallint 2 Which column of the template file corresponds to template #6 +tColumn_7 smallint 2 Which column of the template file corresponds to template #7 +tColumn_8 smallint 2 Which column of the template file corresponds to template #8 +tColumn_9 smallint 2 Which column of the template file corresponds to template #9 +nPoly real 4 Number of polynomial terms used in the fit +theta_0 real 4 Coefficient for template #0 of fit +theta_1 real 4 Coefficient for template #1 of fit +theta_2 real 4 Coefficient for template #2 of fit +theta_3 real 4 Coefficient for template #3 of fit +theta_4 real 4 Coefficient for template #4 of fit +theta_5 real 4 Coefficient for template #5 of fit +theta_6 real 4 Coefficient for template #6 of fit +theta_7 real 4 Coefficient for template #7 of fit +theta_8 real 4 Coefficient for template #8 of fit +theta_9 real 4 Coefficient for template #9 of fit +velDisp real 4 km/s Velocity dispersion +velDispErr real 4 km/s Error in velocity dispersion +velDispZ real 4 Redshift associated with best fit velocity dispersion +velDispZErr real 4 Error in redshift associated with best fit velocity dispersion +velDispChi2 real 4 Chi-squared associated with velocity dispersion fit +velDispNPix int 4 Number of pixels overlapping best template in velocity dispersion fit +velDispDOF int 4 Number of degrees of freedom in velocity dispersion fit +waveMin real 4 Angstroms Minimum observed (vacuum) wavelength +waveMax real 4 Angstroms Maximum observed (vacuum) wavelength +wCoverage real 4 Coverage in wavelength, in units of log10 wavelength +snMedian_u real 4 Median signal-to-noise over all good pixels in u-band +snMedian_g real 4 Median signal-to-noise over all good pixels in g-band +snMedian_r real 4 Median signal-to-noise over all good pixels in r-band +snMedian_i real 4 Median signal-to-noise over all good pixels in i-band +snMedian_z real 4 Median signal-to-noise over all good pixels in z-band +snMedian real 4 Median signal-to-noise over all good pixels +chi68p real 4 68-th percentile value of abs(chi) of the best-fit synthetic spectrum to the actual spectrum (around 1.0 for a good fit) +fracNSigma_1 real 4 Fraction of pixels deviant by more than 1 sigma relative to best-fit +fracNSigma_2 real 4 Fraction of pixels deviant by more than 2 sigma relative to best-fit +fracNSigma_3 real 4 Fraction of pixels deviant by more than 3 sigma relative to best-fit +fracNSigma_4 real 4 Fraction of pixels deviant by more than 4 sigma relative to best-fit +fracNSigma_5 real 4 Fraction of pixels deviant by more than 5 sigma relative to best-fit +fracNSigma_6 real 4 Fraction of pixels deviant by more than 6 sigma relative to best-fit +fracNSigma_7 real 4 Fraction of pixels deviant by more than 7 sigma relative to best-fit +fracNSigma_8 real 4 Fraction of pixels deviant by more than 8 sigma relative to best-fit +fracNSigma_9 real 4 Fraction of pixels deviant by more than 9 sigma relative to best-fit +fracNSigma_10 real 4 Fraction of pixels deviant by more than 10 sigma relative to best-fit +fracNSigHi_1 real 4 Fraction of pixels high by more than 1 sigma relative to best-fit +fracNSigHi_2 real 4 Fraction of pixels high by more than 2 sigma relative to best-fit +fracNSigHi_3 real 4 Fraction of pixels high by more than 3 sigma relative to best-fit +fracNSigHi_4 real 4 Fraction of pixels high by more than 4 sigma relative to best-fit +fracNSigHi_5 real 4 Fraction of pixels high by more than 5 sigma relative to best-fit +fracNSigHi_6 real 4 Fraction of pixels high by more than 6 sigma relative to best-fit +fracNSigHi_7 real 4 Fraction of pixels high by more than 7 sigma relative to best-fit +fracNSigHi_8 real 4 Fraction of pixels high by more than 8 sigma relative to best-fit +fracNSigHi_9 real 4 Fraction of pixels high by more than 9 sigma relative to best-fit +fracNSigHi_10 real 4 Fraction of pixels high by more than 10 sigma relative to best-fit +fracNSigLo_1 real 4 Fraction of pixels low by more than 1 sigma relative to best-fit +fracNSigLo_2 real 4 Fraction of pixels low by more than 2 sigma relative to best-fit +fracNSigLo_3 real 4 Fraction of pixels low by more than 3 sigma relative to best-fit +fracNSigLo_4 real 4 Fraction of pixels low by more than 4 sigma relative to best-fit +fracNSigLo_5 real 4 Fraction of pixels low by more than 5 sigma relative to best-fit +fracNSigLo_6 real 4 Fraction of pixels low by more than 6 sigma relative to best-fit +fracNSigLo_7 real 4 Fraction of pixels low by more than 7 sigma relative to best-fit +fracNSigLo_8 real 4 Fraction of pixels low by more than 8 sigma relative to best-fit +fracNSigLo_9 real 4 Fraction of pixels low by more than 9 sigma relative to best-fit +fracNSigLo_10 real 4 Fraction of pixels low by more than 10 sigma relative to best-fit +spectroFlux_u real 4 nanomaggies Spectrum projected onto u filter +spectroFlux_g real 4 nanomaggies Spectrum projected onto g filter +spectroFlux_r real 4 nanomaggies Spectrum projected onto r filter +spectroFlux_i real 4 nanomaggies Spectrum projected onto i filter +spectroFlux_z real 4 nanomaggies Spectrum projected onto z filter +spectroSynFlux_u real 4 nanomaggies Best-fit template spectrum projected onto u filter +spectroSynFlux_g real 4 nanomaggies Best-fit template spectrum projected onto g filter +spectroSynFlux_r real 4 nanomaggies Best-fit template spectrum projected onto r filter +spectroSynFlux_i real 4 nanomaggies Best-fit template spectrum projected onto i filter +spectroSynFlux_z real 4 nanomaggies Best-fit template spectrum projected onto z filter +spectroFluxIvar_u real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto u filter +spectroFluxIvar_g real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto g filter +spectroFluxIvar_r real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto r filter +spectroFluxIvar_i real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto i filter +spectroFluxIvar_z real 4 1/nanomaggies^2 Inverse variance of spectrum projected onto z filter +spectroSynFluxIvar_u real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto u filter +spectroSynFluxIvar_g real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto g filter +spectroSynFluxIvar_r real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto r filter +spectroSynFluxIvar_i real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto i filter +spectroSynFluxIvar_z real 4 1/nanomaggies^2 Inverse variance of best-fit template spectrum projected onto z filter +spectroSkyFlux_u real 4 nanomaggies Sky spectrum projected onto u filter +spectroSkyFlux_g real 4 nanomaggies Sky spectrum projected onto g filter +spectroSkyFlux_r real 4 nanomaggies Sky spectrum projected onto r filter +spectroSkyFlux_i real 4 nanomaggies Sky spectrum projected onto i filter +spectroSkyFlux_z real 4 nanomaggies Sky spectrum projected onto z filter +anyAndMask int 4 For each bit, records whether any pixel in the spectrum has that bit set in its ANDMASK +anyOrMask int 4 For each bit, records whether any pixel in the spectrum has that bit set in its ORMASK +plateSN2 real 4 Overall signal-to-noise-squared measure for plate (only set for SDSS spectrograph) +deredSN2 real 4 Dereddened signal-to-noise-squared measure for plate (only set for BOSS spectrograph) +snTurnoff real 4 Signal to noise measure for MS turnoff stars on plate (-9999 if not appropriate) +sn1_g real 4 (S/N)^2 at g=20.20 for spectrograph #1 +sn1_r real 4 (S/N)^2 at r=20.25 for spectrograph #1 +sn1_i real 4 (S/N)^2 at i=19.90 for spectrograph #1 +sn2_g real 4 (S/N)^2 at g=20.20 for spectrograph #2 +sn2_r real 4 (S/N)^2 at r=20.25 for spectrograph #2 +sn2_i real 4 (S/N)^2 at i=19.90 for spectrograph #2 +elodieFileName varchar 32 File name for best-fit Elodie star +elodieObject varchar 32 Star name (mostly Henry Draper names) +elodieSpType varchar 32 Spectral type +elodieBV real 4 mag (B-V) color +elodieTEff real 4 Kelvin Effective temperature +elodieLogG real 4 log10(gravity) +elodieFeH real 4 Metallicity ([Fe/H]) +elodieZ real 4 Redshift +elodieZErr real 4 Redshift error (negative for invalid fit) +elodieZModelErr real 4 Standard deviation in redshift among the 12 best-fit stars +elodieRChi2 real 4 Reduced chi^2 +elodieDOF real 4 Degrees of freedom for fit +htmID bigint 8 CODE_HTM 20 deep Hierarchical Triangular Mesh ID +loadVersion int 4 ID_TRACER Load Version +img varbinary -1 IMAGE? Spectrum Image + +*/ + +CREATE TABLE catalogdb.sdss_dr16_specobj( + specObjID numeric(20) PRIMARY KEY, + bestObjID bigint, + fluxObjID bigint, + targetObjID bigint, + plateID numeric(20), + sciencePrimary smallint, + sdssPrimary smallint, + legacyPrimary smallint, + seguePrimary smallint, + segue1Primary smallint, + segue2Primary smallint, + bossPrimary smallint, + bossSpecObjID integer, + firstRelease varchar(32), + survey varchar(32), + instrument varchar(32), + programname varchar(32), + chunk varchar(32), + platerun varchar(32), + mjd integer, + plate smallint, + fiberID smallint, + run1d varchar(32), + run2d varchar(32), + tile integer, + designID integer, + legacy_target1 bigint, + legacy_target2 bigint, + special_target1 bigint, + special_target2 bigint, + segue1_target1 bigint, + segue1_target2 bigint, + segue2_target1 bigint, + segue2_target2 bigint, + boss_target1 bigint, + eboss_target0 bigint, + eboss_target1 bigint, + eboss_target2 bigint, + eboss_target_id bigint, + ancillary_target1 bigint, + ancillary_target2 bigint, + thing_id_targeting bigint, + thing_id integer, + primTarget bigint, + secTarget bigint, + spectrographID smallint, + sourceType varchar(128), + targetType varchar(128), + ra double precision, + dec double precision, + cx double precision, + cy double precision, + cz double precision, + xFocal real, + yFocal real, + lambdaEff real, + blueFiber integer, + zOffset real, + z real, + zErr real, + zWarning integer, + class varchar(32), + subClass varchar(32), + rChi2 real, + DOF real, + rChi2Diff real, + z_noqso real, + zErr_noqso real, + zWarning_noqso integer, + class_noqso varchar(32), + subClass_noqso varchar(32), + rChi2Diff_noqso real, + z_person real, + class_person varchar(32), + comments_person varchar(200), + tFile varchar(32), + tColumn_0 smallint, + tColumn_1 smallint, + tColumn_2 smallint, + tColumn_3 smallint, + tColumn_4 smallint, + tColumn_5 smallint, + tColumn_6 smallint, + tColumn_7 smallint, + tColumn_8 smallint, + tColumn_9 smallint, + nPoly real, + theta_0 real, + theta_1 real, + theta_2 real, + theta_3 real, + theta_4 real, + theta_5 real, + theta_6 real, + theta_7 real, + theta_8 real, + theta_9 real, + velDisp real, + velDispErr real, + velDispZ real, + velDispZErr real, + velDispChi2 real, + velDispNPix integer, + velDispDOF integer, + waveMin real, + waveMax real, + wCoverage real, + snMedian_u real, + snMedian_g real, + snMedian_r real, + snMedian_i real, + snMedian_z real, + snMedian real, + chi68p real, + fracNSigma_1 real, + fracNSigma_2 real, + fracNSigma_3 real, + fracNSigma_4 real, + fracNSigma_5 real, + fracNSigma_6 real, + fracNSigma_7 real, + fracNSigma_8 real, + fracNSigma_9 real, + fracNSigma_10 real, + fracNSigHi_1 real, + fracNSigHi_2 real, + fracNSigHi_3 real, + fracNSigHi_4 real, + fracNSigHi_5 real, + fracNSigHi_6 real, + fracNSigHi_7 real, + fracNSigHi_8 real, + fracNSigHi_9 real, + fracNSigHi_10 real, + fracNSigLo_1 real, + fracNSigLo_2 real, + fracNSigLo_3 real, + fracNSigLo_4 real, + fracNSigLo_5 real, + fracNSigLo_6 real, + fracNSigLo_7 real, + fracNSigLo_8 real, + fracNSigLo_9 real, + fracNSigLo_10 real, + spectroFlux_u real, + spectroFlux_g real, + spectroFlux_r real, + spectroFlux_i real, + spectroFlux_z real, + spectroSynFlux_u real, + spectroSynFlux_g real, + spectroSynFlux_r real, + spectroSynFlux_i real, + spectroSynFlux_z real, + spectroFluxIvar_u real, + spectroFluxIvar_g real, + spectroFluxIvar_r real, + spectroFluxIvar_i real, + spectroFluxIvar_z real, + spectroSynFluxIvar_u real, + spectroSynFluxIvar_g real, + spectroSynFluxIvar_r real, + spectroSynFluxIvar_i real, + spectroSynFluxIvar_z real, + spectroSkyFlux_u real, + spectroSkyFlux_g real, + spectroSkyFlux_r real, + spectroSkyFlux_i real, + spectroSkyFlux_z real, + anyAndMask integer, + anyOrMask integer, + plateSN2 real, + deredSN2 real, + snTurnoff real, + sn1_g real, + sn1_r real, + sn1_i real, + sn2_g real, + sn2_r real, + sn2_i real, + elodieFileName varchar(32), + elodieObject varchar(32), + elodieSpType varchar(32), + elodieBV real, + elodieTEff real, + elodieLogG real, + elodieFeH real, + elodieZ real, + elodieZErr real, + elodieZModelErr real, + elodieRChi2 real, + elodieDOF real, + htmID bigint, + loadVersion integer +); + + +\copy catalogdb.sdss_dr16_specobj FROM program 'bzcat /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/sdssSpecObj/dr16/src/sqlSpecObj.csv.bz2' WITH CSV HEADER; + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr16_specobj_q3c_ang2ipix_idx ON catalogdb.sdss_dr16_specobj; +ANALYZE catalogdb.sdss_dr16_specobj; + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj USING BTREE (fluxObjID); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj USING BTREE (targetObjID); + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr16_specobj (mjd, plate, fiberid, run2d); diff --git a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql similarity index 98% rename from schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql rename to schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql index 81bc23f9..8096ef9c 100644 --- a/schema/sdss5db/catalogdb/dr14q_v4_4/dr14q_v4_4.sql +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS "catalogdb"."dr14q_v4_4" ( +CREATE TABLE IF NOT EXISTS "catalogdb"."sdss_dr14_qso" ( "sdss_name" VARCHAR(18), "ra" DOUBLE PRECISION, "dec" DOUBLE PRECISION, diff --git a/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql new file mode 100644 index 00000000..134edfe2 --- /dev/null +++ b/schema/sdss5db/catalogdb/sdss_qso/dr14/sdss_dr14_qso_index.sql @@ -0,0 +1,18 @@ + +ALTER TABLE catalogdb.sdss_dr14_qso ADD PRIMARY KEY (pk); + +-- ALTER TABLE catalogdb.sdss_dr14_qso ADD COLUMN specobjid BIGINT; +-- UPDATE catalogdb.sdss_dr14_qso +-- SET specobjid = (plate::bigint<<50 ) + (fiberid::bigint<<38) + +-- ((mjd-50000)::bigint<<24) + (26::bigint<<10); + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (plate); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (mjd); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso USING BTREE (fiberid); +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (mjd, plate, fiberid); + +-- CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (specobjid); + +CREATE INDEX CONCURRENTLY ON catalogdb.sdss_dr14_qso (q3c_ang2ipix(ra, dec)); +CLUSTER sdss_dr14_qso_q3c_ang2ipix_idx ON catalogdb.sdss_dr14_qso; +ANALYZE catalogdb.sdss_dr14_qso; diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql new file mode 100644 index 00000000..2dfe1d2a --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper.sql @@ -0,0 +1,117 @@ +/* + +SkyMapper Southern Sky Survey DR1.1 - http://skymapper.anu.edu.au + +To download the file see: http://skymapper.anu.edu.au/_data/DR1.1/ +Table columns: http://skymapper.anu.edu.au/table-browser/ (different order as the CSVs!) + +At Utah: /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/skymapper/dr1.1 + +*/ + +CREATE TABLE catalogdb.skymapper_dr1_1 ( + object_id BIGINT, + raj2000 DOUBLE PRECISION, + dej2000 DOUBLE PRECISION, + e_raj2000 INTEGER, + e_dej2000 INTEGER, + smss_j VARCHAR(18), + mean_epoch DOUBLE PRECISION, + rms_epoch REAL, + glon REAL, + glat REAL, + flags SMALLINT, + nimaflags INTEGER, + ngood SMALLINT, + ngood_min SMALLINT, + nch_max SMALLINT, + u_flags SMALLINT, + u_nimaflags INTEGER, + u_ngood SMALLINT, + u_nch SMALLINT, + u_nvisit SMALLINT, + v_flags SMALLINT, + v_nimaflags INTEGER, + v_ngood SMALLINT, + v_nch SMALLINT, + v_nvisit SMALLINT, + g_flags SMALLINT, + g_nimaflags INTEGER, + g_ngood SMALLINT, + g_nch SMALLINT, + g_nvisit SMALLINT, + r_flags SMALLINT, + r_nimaflags INTEGER, + r_ngood SMALLINT, + r_nch SMALLINT, + r_nvisit SMALLINT, + i_flags SMALLINT, + i_nimaflags INTEGER, + i_ngood SMALLINT, + i_nch SMALLINT, + i_nvisit SMALLINT, + z_flags SMALLINT, + z_nimaflags INTEGER, + z_ngood SMALLINT, + z_nch SMALLINT, + z_nvisit SMALLINT, + class_star REAL, + radius_petro REAL, + a REAL, + e_a REAL, + b REAL, + e_b REAL, + pa REAL, + e_pa REAL, + u_psf REAL, + e_u_psf REAL, + u_petro REAL, + e_u_petro REAL, + v_psf REAL, + e_v_psf REAL, + v_petro REAL, + e_v_petro REAL, + g_psf REAL, + e_g_psf REAL, + g_petro REAL, + e_g_petro REAL, + r_psf REAL, + e_r_psf REAL, + r_petro REAL, + e_r_petro REAL, + i_psf REAL, + e_i_psf REAL, + i_petro REAL, + e_i_petro REAL, + z_psf REAL, + e_z_psf REAL, + z_petro REAL, + e_z_petro REAL, + ebmv_sfd REAL, + prox REAL, + prox_id BIGINT, + edr_id BIGINT, + edr_dist REAL, + twomass_key1 BIGINT, + twomass_dist1 REAL, + twomass_cat1 VARCHAR(3), + twomass_key2 BIGINT, + twomass_dist2 REAL, + twomass_cat2 VARCHAR(3), + allwise_cntr BIGINT, + allwise_dist REAL, + ucac4_mpos BIGINT, + ucac4_dist REAL, + apass_recno BIGINT, + apass_dist REAL, + gaia_dr1_id BIGINT, + gaia_dr1_dist REAL, + ps1_dr1_id BIGINT, + ps1_dr1_dist REAL, + galex_bcs_id BIGINT, + galex_bcs_dist REAL, + gaia_dr2_id1 BIGINT, + gaia_dr2_dist1 REAL, + gaia_dr2_id2 BIGINT, + gaia_dr2_dist2 REAL +); diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql new file mode 100644 index 00000000..b3b31d5e --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_indexes.sql @@ -0,0 +1,14 @@ + +-- Doing it this way prevent blocks: https://bit.ly/2UDbpbz + +CREATE UNIQUE INDEX CONCURRENTLY object_id_pk + ON catalogdb.skymapper_dr1_1 (object_id); -- takes a long time, but doesn’t block queries + +ALTER TABLE catalogdb.skymapper_dr1_1 + ADD CONSTRAINT object_id_pk PRIMARY KEY + USING INDEX object_id_pk; -- blocks queries, but only very briefly + + +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_dr1_1 (q3c_ang2ipix(raj2000, dej2000)); +CLUSTER skymapper_dr1_1_q3c_ang2ipix_idx ON catalogdb.skymapper_dr1_1; +ANALYZE catalogdb.skymapper_dr1_1; diff --git a/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load new file mode 100755 index 00000000..64bc620d --- /dev/null +++ b/schema/sdss5db/catalogdb/skymapper/dr1.1/skymapper_load @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +ls $CATALOGDB_DIR/skymapper/dr1.1/*.csv.gz | parallel -j10 "psql -U sdss sdss5db -c \"\copy catalogdb.skymapper_dr1_1 FROM program 'zcat {}' WITH CSV HEADER;\"" diff --git a/schema/sdss5db/catalogdb/smgaia/smgaia.sql b/schema/sdss5db/catalogdb/smgaia/smgaia.sql new file mode 100644 index 00000000..4939a120 --- /dev/null +++ b/schema/sdss5db/catalogdb/smgaia/smgaia.sql @@ -0,0 +1,38 @@ +/* + +SkyMapper-Gaia + +http://skymapper.anu.edu.au/_data/sm-gaia/ + +*/ + + +CREATE TABLE catalogdb.skymapper_gaia ( + skymapper_object_id BIGINT, + gaia_source_id BIGINT, + teff REAL, + e_teff REAL, + feh REAL, + e_feh REAL +); + + +\COPY catalogdb.skymapper_gaia FROM PROGRAM 'zcat /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/SMGaia/SMGaia.csv.gz' WITH CSV HEADER DELIMITER ','; + + +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (gaia_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (teff); +CREATE INDEX CONCURRENTLY ON catalogdb.skymapper_gaia (feh); + + +ALTER TABLE catalogdb.skymapper_gaia + ADD CONSTRAINT skymapper_object_id_fk + FOREIGN KEY (skymapper_object_id) + REFERENCES catalogdb.skymapper_dr1_1 (object_id) + ON DELETE CASCADE; + +ALTER TABLE catalogdb.skymapper_gaia + ADD CONSTRAINT gaia_source_id_fk + FOREIGN KEY (gaia_source_id) + REFERENCES catalogdb.gaia_dr2_source (source_id) + ON DELETE CASCADE; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql index 2a94772b..ca9a4086 100644 --- a/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8.sql @@ -1,7 +1,135 @@ /* -schema for TESS Input Catalog: +schema for tess: -https://outerspace.stsci.edu/display/TESS/TIC+v8+and+CTL+v8.xx+Data+Release+Notes +/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tic/v8/tic_column_description.txt */ + +CREATE TABLE catalogdb.tic_v8 ( + id BIGINT PRIMARY KEY, + version VARCHAR(8), + hip INTEGER, + tyc VARCHAR(12), + ucac VARCHAR(10), + twomass VARCHAR(20), + sdss BIGINT, + allwise VARCHAR(20), + gaia BIGINT, + apass VARCHAR(30), + kic INTEGER, + objtype VARCHAR(16), + typesrc VARCHAR(16), + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + posflag VARCHAR(12), + pmra REAL, + e_pmra REAL, + pmdec REAL, + e_pmdec REAL, + pmflag VARCHAR(12), + plx REAL, + e_plx REAL, + parflag VARCHAR(12), + gallong DOUBLE PRECISION, + gallat DOUBLE PRECISION, + eclong DOUBLE PRECISION, + eclat DOUBLE PRECISION, + bmag REAL, + e_bmag REAL, + vmag REAL, + e_vmag REAL, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + jmag REAL, + e_jmag REAL, + hmag REAL, + e_hmag REAL, + kmag REAL, + e_kmag REAL, + twomflag VARCHAR(20), + prox REAL, + w1mag REAL, + e_w1mag REAL, + w2mag REAL, + e_w2mag REAL, + w3mag REAL, + e_w3mag REAL, + w4mag REAL, + e_w4mag REAL, + gaiamag REAL, + e_gaiamag REAL, + tmag REAL, + e_tmag REAL, + tessflag VARCHAR(20), + spflag VARCHAR(20), + teff REAL, + e_teff REAL, + logg REAL, + e_logg REAL, + mh REAL, + e_mh REAL, + rad REAL, + e_rad REAL, + mass REAL, + e_mass REAL, + rho REAL, + e_rho REAL, + lumclass VARCHAR(10), + lum REAL, + e_lum REAL, + d REAL, + e_d REAL, + ebv REAL, + e_ebv REAL, + numcont INTEGER, + contratio REAL, + disposition VARCHAR(10), + duplicate_id BIGINT, + priority REAL, + eneg_ebv REAL, + epos_ebv REAL, + ebvflag VARCHAR(20), + eneg_mass REAL, + epos_mass REAL, + eneg_rad REAL, + epos_rad REAL, + eneg_rho REAL, + epos_rho REAL, + eneg_logg REAL, + epos_logg REAL, + eneg_lum REAL, + epos_lum REAL, + eneg_dist REAL, + epos_dist REAL, + distflag VARCHAR(20), + eneg_teff REAL, + epos_teff REAL, + tefflag VARCHAR(20), + gaiabp REAL, + e_gaiabp REAL, + gaiarp REAL, + e_gaiarp REAL, + gaiaqflag INTEGER, + starchareflag VARCHAR(20), + vmagflag VARCHAR(20), + bmagflag VARCHAR(20), + splits VARCHAR(20), + e_ra DOUBLE PRECISION, + e_dec DOUBLE PRECISION, + ra_orig DOUBLE PRECISION, + dec_orig DOUBLE PRECISION, + e_ra_orig DOUBLE PRECISION, + e_dec_orig DOUBLE PRECISION, + raddflag INTEGER, + wdflag INTEGER, + objid BIGINT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql new file mode 100644 index 00000000..f98e312f --- /dev/null +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_index.sql @@ -0,0 +1,29 @@ + +-- Convert SDSS and GAIA to bigint for more efficient FKs + +ALTER TABLE catalogdb.tic_v8 ADD COLUMN gaia_int BIGINT; +UPDATE catalogdb.tic_v8 SET gaia_int = gaia::BIGINT; + + +-- Indices + +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 (q3c_ang2ipix(ra, dec)); +CLUSTER tic_v8_q3c_ang2ipix_idx ON catalogdb.tic_v8; +VACUUM ANALYZE catalogdb.tic_v8; + +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Bmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Vmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (umag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (gmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (rmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (imag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (zmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Jmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Hmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (Kmag); +CREATE INDEX CONCURRENTLY ON catalogdb.tic_v8 USING BTREE (posflag); + +ALTER TABLE catalogdb.tic_v8 ADD COLUMN twomass_psc TEXT; +UPDATE catalogdb.tic_v8 + SET twomass_psc = twomass + WHERE twomass IS NOT NULL AND posflag != '2MASSEXT'; diff --git a/schema/sdss5db/catalogdb/tic/v8/tic_v8_load b/schema/sdss5db/catalogdb/tic/v8/tic_v8_load new file mode 100644 index 00000000..9c16d9b6 --- /dev/null +++ b/schema/sdss5db/catalogdb/tic/v8/tic_v8_load @@ -0,0 +1,4 @@ +#! /usr/bin/bash +# encoding: utf-8 + +ls /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tic/v8/csv/*.csv.gz | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.tic_v8 FROM program 'zcat {}' WITH CSV;\"" diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql index e4a10355..f7337278 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc.sql @@ -37,7 +37,7 @@ CREATE TABLE catalogdb.twomass_psc ( pxcntr integer, gal_contam smallint, mp_flg smallint, - pts_key integer, + pts_key integer PRIMARY KEY, hemis character(1), date date, scan smallint, @@ -529,6 +529,3 @@ CREATE TABLE catalogdb.twomass_xsc ( k_seetrack real, ext_key integer ) WITHOUT OIDS; - - - diff --git a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql index c5bc5415..0d9f9985 100644 --- a/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql +++ b/schema/sdss5db/catalogdb/twomass/psc/twomass_psc_index.sql @@ -8,24 +8,22 @@ drop index catalogdb.gaia_dr1_tgas_dec_index; */ - -- Indices -alter table catalogdb.twomass_psc add primary key(pts_key); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (j_m); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (h_m); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (k_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (ra); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (decl); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (j_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (h_m); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (k_m); -create index on catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); -CLUSTER twomass_psc_q3c_ang2ipix_idx on catalogdb.twomass_psc; -analyze catalogdb.twomass_psc; +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc (q3c_ang2ipix(ra, decl)); +CLUSTER twomass_psc_q3c_ang2ipix_idx ON catalogdb.twomass_psc; +ANALYZE catalogdb.twomass_psc; -ALTER TABLE catalogdb.twomass_psc ADD CONSTRAINT twomass_psc_desig_unique UNIQUE (designation); +ALTER TABLE catalogdb.twomass_psc + ADD CONSTRAINT catalogdb_twomass_psc_designation_unique + UNIQUE (designation); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (designation); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (ph_qual); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (cc_flg); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (gal_contam); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (rd_flg); -CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc using BTREE (designation); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (ph_qual); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (cc_flg); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (gal_contam); +CREATE INDEX CONCURRENTLY ON catalogdb.twomass_psc USING BTREE (rd_flg); diff --git a/schema/sdss5db/catalogdb/tycho2/tycho2.sql b/schema/sdss5db/catalogdb/tycho2/tycho2.sql new file mode 100644 index 00000000..ded2be37 --- /dev/null +++ b/schema/sdss5db/catalogdb/tycho2/tycho2.sql @@ -0,0 +1,395 @@ +/* + +The Tycho-2 catalogue including the supplemental table 1. + +/uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/tycho2/Tycho-2 + +Downloaded from ftp://cdsarc.u-strasbg.fr/pub/cats/I/259/ + +Note 1: we add an id_tycho column defined as in the Gaia cross-matching + (http://bit.ly/2TZD4Eg): (TYC1*1000000.0d0)+(TYC2*10.0d0)+(TYC3*1.0d0). + +Note 2: The columns HIP and CCDM are missing the delimiter in the files. This + is fixed in the loading process. + +I/259 The Tycho-2 Catalogue (Hog+ 2000) +================================================================================ +The Tycho-2 Catalogue of the 2.5 Million Brightest Stars + Hog E., Fabricius C., Makarov V.V., Urban S., Corbin T., + Wycoff G., Bastian U., Schwekendiek P., Wicenec A. + + =2000A&A...355L..27H +================================================================================ +ADC_Keywords: Positional data ; Proper motions ; Surveys ; + Photometry ; Stars, double and multiple +Mission_Name: Hipparcos +Keywords: astrometry - stars: fundamental parameters - catalogs + +Abstract: + The Tycho-2 Catalogue is an astrometric reference catalogue containing + positions and proper motions as well as two-colour photometric data + for the 2.5 million brightest stars in the sky. The Tycho-2 positions + and magnitudes are based on precisely the same observations as the + original Tycho Catalogue (hereafter Tycho-1; see Cat. )) + collected by the star mapper of the ESA Hipparcos satellite, but + Tycho-2 is much bigger and slightly more precise, owing to a more + advanced reduction technique. Components of double stars with + separations down to 0.8 arcsec are included. Proper motions precise to + about 2.5 mas/yr are given as derived from a comparison with the + Astrographic Catalogue and 143 other ground-based astrometric + catalogues, all reduced to the Hipparcos celestial coordinate system. + Tycho-2 supersedes in most applications Tycho-1, as well as the ACT + (Cat. ) and TRC (Cat. ) catalogues based on Tycho-1. + Supplement-1 lists stars from the Hipparcos and Tycho-1 Catalogues + which are not in Tycho-2. Supplement-2 lists 1146 Tycho-1 stars which + are probably either false or heavily disturbed. + + For more information, please consult the Tycho-2 home page: + http://www.astro.ku.dk/~erik/Tycho-2 + +Catalogue Characteristics: + + The principal characteristics of the Tycho-2 Catalogue are + summarized below. By means of proper motions the positions + are transferred to the year 2000.0, the epoch of the catalogue. + The median values of internal standard errors are given. + ------------------------------------------------------------ + Mean satellite observation epoch ~J1991.5 + Epoch of the Tycho-2 Catalogue J2000.0 + Reference system ICRS + coincidence with ICRS (1) +/-0.6 mas + deviation from inertial (1) +/-0.25 mas/yr + Number of entries 2,539,913 + Astrometric standard errors (2) + V_T_ < 9 mag 7 mas + all stars, positions 60 mas + all stars, proper motions 2.5 mas/yr + Photometric std. errors (3) on V_T_ + V_T_ < 9 mag 0.013 mag + all stars 0.10 mag + Star density + b= 0 deg 150 stars/sq.deg. + b= +/-30 deg 50 stars/sq.deg. + b= +/-90 deg 25 stars/sq.deg. + Completeness to 90 per cent V ~ 11.5 mag + Completeness to 99 per cent V ~ 11.0 mag + Number of Tycho observations ~300 10^6^ + ------------------------------------------------------------ + Note (1): about all 3 axes + Note (2): + ratio of external to internal standard errors is ~1.0 + for positions and for proper motions. Systematic errors + are less than 1 mas and 0.5 mas/yr + Note (3): + ratio of photometric external to internal standard errors + at V_T_ > 9 mag is below 1.5 + ------------------------------------------------------------ + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +tyc2.dat 206 2539913 *The Tycho-2 main catalogue +suppl_1.dat 122 17588 The Tycho-2 supplement-1 +suppl_2.dat 115 1146 The Tycho-2 supplement-2 +index.dat 42 9538 Index to Tycho-2 and supplement-1 +guide.ps 120 7837 Guide to Tycho-2 (postscript) +guide.pdf 1 428724 Guide to Tycho-2 (pdf) +-------------------------------------------------------------------------------- +Note on tyc2.dat: + This huge file is split into 20 smaller files named tyc2.dat.00, + tyc2.dat.01, ... tyc2.dat.18, each containing 127000 lines, + and tyc2.dat.19 which contains the last 126913 lines. +-------------------------------------------------------------------------------- + +See also: + I/239 : The Hipparcos and Tycho Catalogues (ESA 1997) + I/211 : CCDM (Components of Double and Multiple stars) (Dommanget+ 1994) + I/246 : The ACT Reference Catalog (Urban+ 1997) + I/250 : The Tycho Reference Catalogue (Hog+ 1998) + http://www.astro.ku.dk/~erik/Tycho-2 : Tycho-2 home page + +Nomenclature Notes: + The TYC identifier is constructed from the GSC region number (TYC1), + the running number within the region (TYC2) and a component identifier + (TYC3) which is normally 1. Some non-GSC running numbers were + constructed for the first Tycho Catalogue and for Tycho-2. + The recommended star designation contains a hyphen between the TYC + numbers, e.g. TYC 1-13-1. + +Field separator in the data files: + In the data files, the fields in each record are separated by + a vertical bar, |. In this connection the TYC identifier (TYC1, TYC2 + and TYC3) constitutes one field and the pair of a HIP number with a + CCDM identifier is also considered one field. + + +Byte-by-byte Description of file: tyc2.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 4 I4 --- TYC1 [1,9537]+= TYC1 from TYC or GSC (1) + 6- 10 I5 --- TYC2 [1,12121] TYC2 from TYC or GSC (1) + 12 I1 --- TYC3 [1,3] TYC3 from TYC (1) + 14 A1 --- pflag [ PX] mean position flag (2) + 16- 27 F12.8 deg RAmdeg []? Mean Right Asc, ICRS, epoch=J2000 (3) + 29- 40 F12.8 deg DEmdeg []? Mean Decl, ICRS, at epoch=J2000 (3) + 42- 48 F7.1 mas/yr pmRA ? Proper motion in RA*cos(dec) (12) + 50- 56 F7.1 mas/yr pmDE ? Proper motion in Dec (12) + 58- 60 I3 mas e_RAmdeg [3,183]? s.e. RA*cos(dec),at mean epoch (5) + 62- 64 I3 mas e_DEmdeg [1,184]? s.e. of Dec at mean epoch (5) + 66- 69 F4.1 mas/yr e_pmRA [0.2,11.5]? s.e. prop mot in RA*cos(dec)(5) + 71- 74 F4.1 mas/yr e_pmDE [0.2,10.3]? s.e. of proper motion in Dec(5) + 76- 82 F7.2 yr EpRAm [1915.95,1992.53]? mean epoch of RA (4) + 84- 90 F7.2 yr EpDEm [1911.94,1992.01]? mean epoch of Dec (4) + 92- 93 I2 --- Num [2,36]? Number of positions used + 95- 97 F3.1 --- q_RAmdeg [0.0,9.9]? Goodness of fit for mean RA (6) + 99-101 F3.1 --- q_DEmdeg [0.0,9.9]? Goodness of fit for mean Dec (6) + 103-105 F3.1 --- q_pmRA [0.0,9.9]? Goodness of fit for pmRA (6) + 107-109 F3.1 --- q_pmDE [0.0,9.9]? Goodness of fit for pmDE (6) + 111-116 F6.3 mag BTmag [2.183,16.581]? Tycho-2 BT magnitude (7) + 118-122 F5.3 mag e_BTmag [0.014,1.977]? s.e. of BT (7) + 124-129 F6.3 mag VTmag [1.905,15.193]? Tycho-2 VT magnitude (7) + 131-135 F5.3 mag e_VTmag [0.009,1.468]? s.e. of VT (7) + 137-139 I3 0.1arcsec prox [3,999] proximity indicator (8) + 141 A1 --- TYC [T] Tycho-1 star (9) + 143-148 I6 --- HIP [1,120404]? Hipparcos number + 149-151 A3 --- CCDM CCDM component identifier for HIP stars(10) + 153-164 F12.8 deg RAdeg Observed Tycho-2 Right Ascension, ICRS + 166-177 F12.8 deg DEdeg Observed Tycho-2 Declination, ICRS + 179-182 F4.2 yr EpRA-1990 [0.81,2.13] epoch-1990 of RAdeg + 184-187 F4.2 yr EpDE-1990 [0.72,2.36] epoch-1990 of DEdeg + 189-193 F5.1 mas e_RAdeg s.e.RA*cos(dec), of observed Tycho-2 RA (5) + 195-199 F5.1 mas e_DEdeg s.e. of observed Tycho-2 Dec (5) + 201 A1 --- posflg [ DP] type of Tycho-2 solution (11) + 203-206 F4.1 --- corr [-1,1] correlation (RAdeg,DEdeg) +-------------------------------------------------------------------------------- + +Note (1): The TYC identifier is constructed from the GSC region number + (TYC1), the running number within the region (TYC2) and a component + identifier (TYC3) which is normally 1. Some non-GSC running numbers + were constructed for the first Tycho Catalogue and for Tycho-2. + The recommended star designation contains a hyphen between the + TYC numbers, e.g. TYC 1-13-1. + +Note (2): + ' ' = normal mean position and proper motion. + 'P' = the mean position, proper motion, etc., refer to the + photocentre of two Tycho-2 entries, where the BT magnitudes + were used in weighting the positions. + 'X' = no mean position, no proper motion. + +Note (3): + The mean position is a weighted mean for the catalogues contributing + to the proper motion determination. This mean has then been brought to + epoch 2000.0 by the computed proper motion. See Note(2) above for + details. Tycho-2 is one of the several catalogues used to determine + the mean position and proper motion. The observed Tycho-2 position is + given in the fields RAdeg and DEdeg. + +Note (4): + The mean epochs are given in Julian years. + +Note (5): + The errors are based on error models. + +Note (6): + This goodness of fit is the ratio of the scatter-based and the + model-based error. It is only defined when Num > 2. Values + exceeding 9.9 are truncated to 9.9. + +Note (7): + Blank when no magnitude is available. Either BTmag or VTmag is + always given. Approximate Johnson photometry may be obtained as: + V = VT -0.090*(BT-VT) + B-V = 0.850*(BT-VT) + Consult Sect 1.3 of Vol 1 of "The Hipparcos and Tycho Catalogues", + ESA SP-1200, 1997, for details. + +Note (8): + Distance in units of 100 mas to the nearest entry in the Tycho-2 + main catalogue or supplement. The distance is computed for the + epoch 1991.25. A value of 999 (i.e. 99.9 arcsec) is given if the + distance exceeds 99.9 arcsec. + +Note (9): + ' ' = no Tycho-1 star was found within 0.8 arcsec (quality 1-8) + or 2.4 arcsec (quality 9). + 'T' = this is a Tycho-1 star. The Tycho-1 identifier is given in the + beginning of the record. For Tycho-1 stars, resolved in + Tycho-2 as a close pair, both components are flagged as + a Tycho-1 star and the Tycho-1 TYC3 is assigned to the + brightest (VT) component. + The HIP-only stars given in Tycho-1 are not flagged as Tycho-1 stars. + +Note (10): + The CCDM component identifiers for double or multiple Hipparcos stars + contributing to this Tycho-2 entry. For photocentre solutions, all + components within 0.8 arcsec contribute. For double star solutions any + unresolved component within 0.8 arcsec contributes. For single star + solutions, the predicted signal from close stars were normally + subtracted in the analysis of the photon counts and such stars + therefore do not contribute to the solution. The components are given + in lexical order. + +Note (11): + ' ' = normal treatment, close stars were subtracted when possible. + 'D' = double star treatment. Two stars were found. The companion is + normally included as a separate Tycho-2 entry, but may have + been rejected. + 'P' = photocentre treatment, close stars were not subtracted. This + special treatment was applied to known or suspected doubles + which were not successfully (or reliably) resolved in the + Tycho-2 double star processing. + +Note (12): Some Hipparcos stars (having a positive number in the HIP column) + have no proper motions; these are virtually all in multiple systems. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: suppl_1.dat, suppl_2.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 4 I4 --- TYC1 [2,9529]+= TYC1 from TYC (1) + 6- 10 I5 --- TYC2 [1,12112] TYC2 from TYC (1) + 12 I1 --- TYC3 [1,4] TYC3 from TYC (1) + 14 A1 --- flag [HT] data from Hipparcos or Tycho-1 (2) + 16- 27 F12.8 deg RAdeg Right Asc, ICRS, at epoch=J1991.25 + 29- 40 F12.8 deg DEdeg Decl, ICRS, at epoch=J1991.25 + 42- 48 F7.1 mas/yr pmRA []? Proper motion in RA*cos(dec) + 50- 56 F7.1 mas/yr pmDE []? Proper motion in Dec + 58- 62 F5.1 mas e_RAdeg s.e. RA*cos(dec) + 64- 68 F5.1 mas e_DEdeg s.e. of Dec + 70- 74 F5.1 mas/yr e_pmRA []? s.e. prop mot in RA * cos(dec) + 76- 80 F5.1 mas/yr e_pmDE []? s.e. of proper motion in Dec + 82 A1 --- mflag [ BVH] Note about Tycho magnitudes (3) + 84- 89 F6.3 mag BTmag []? Tycho-1 BT magnitude (4) + 91- 95 F5.3 mag e_BTmag []? s.e. of BT (4) + 97-102 F6.3 mag VTmag []? Tycho-1 VT or Hp magnitude (4) + 104-108 F5.3 mag e_VTmag []? s.e. of VT (4) + 110-112 I3 0.1arcsec prox [1,999] proximity indicator (5) + 114 A1 --- TYC [ T] Tycho-1 star + 116-121 I6 --- HIP [1,120404]? Hipparcos number + 122 A1 --- CCDM CCDM component identifier for HIP stars +-------------------------------------------------------------------------------- + +Note (1): The TYC identifier is constructed from the GSC region number (TYC1), + the running number within the region (TYC2) and a component identifier + (TYC3) which is normally 1. The numbers are copied from Tycho-1. + (see the "Nomenclature Notes" section above) + +Note (2): + 'H' = data are from Hipparcos and include proper motion. + 'T' = data are from Tycho-1. No proper motion is given. + +Note (3): + ' ' = both BT and VT given. + 'B' = only BT given. + 'V' = only VT given. + 'H' = Hp is given instead of VT. BT is blank. + +Note (4): + Blank when no magnitude is available. + For Hipparcos stars with no VT, Hp is given instead of VT, and BT is blank. + +Note (5): + Distance in units of 100 mas to nearest Tycho-2 main or supplement + entry. (Computed for the epoch 1991.25). A value of 999 (i.e. 99.9 + arcsec) is given if the distance exceeds 99.9 arcsec. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: index.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 7 I7 --- rec_t2 + Tycho-2 rec. of 1st star in region (1) + 9- 14 I6 --- rec_s1 += Suppl-1 rec. of 1st star in region (1) + 16- 21 F6.2 deg RAmin [-0.01,] smallest RA in region (2) + 23- 28 F6.2 deg RAmax [,360.00] largest RA in region (2) + 30- 35 F6.2 deg DEmin smallest Dec in this region (2) + 37- 42 F6.2 deg DEmax largest Dec in this region (2) +-------------------------------------------------------------------------------- + +Note (1): The catalogue is sorted according to the GSC region numbers. + The line i of the index file gives the record number in Tycho-2 of + the first star in GSC region i. Line i+1 gives the record number +1 + of the last star in GSC region i. For Supplement-1, some regions are + empty and line i and line i+1 give the same record number. + +Note (2): a safe rounding was applied. Minimum values are always + rounded down and maximum values up. +-------------------------------------------------------------------------------- + +References: + Hog E., Fabricius C., Makarov V.V., Bastian U., Schwekendiek P., Wicenec A., + Urban S., Corbin T., Wycoff G., "Construction and verification of the + Tycho-2 Catalogue." =2000A&A...357..367H +================================================================================ +(End) C. Fabricius [Niels Bohr Institute] 19-Jan-2000 + +This script loads the file tycho2_all.csv that was generated by loading +the tyc2.dat and suplement files using +astropy.io.ascii.read(file, format='csv', readme='./ReadMe') and then concatenating +them with astropy.table.vstack. The resulting table was then saved as CSV. + +*/ + + +CREATE TABLE catalogdb.tycho2 ( + tyc1 INTEGER, + tyc2 INTEGER, + tyc3 INTEGER, + pflag VARCHAR(1), + ramdeg DOUBLE PRECISION, + demdeg DOUBLE PRECISION, + pmra DOUBLE PRECISION, + pmde DOUBLE PRECISION, + e_ramdeg INTEGER, + e_demdeg INTEGER, + e_pmra DOUBLE PRECISION, + e_pmde DOUBLE PRECISION, + epram DOUBLE PRECISION, + epdem DOUBLE PRECISION, + num INTEGER, + q_ramdeg DOUBLE PRECISION, + q_demdeg DOUBLE PRECISION, + q_pmra DOUBLE PRECISION, + q_pmde DOUBLE PRECISION, + btmag REAL, + e_btmag REAL, + vtmag REAL, + e_vtmag REAL, + prox INTEGER, + tyc VARCHAR(1), + hip BIGINT, + ccdm VARCHAR(3), + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + epra_1990 DOUBLE PRECISION, + epde_1990 DOUBLE PRECISION, + e_radeg DOUBLE PRECISION, + e_dedeg DOUBLE PRECISION, + posflg VARCHAR(1), + corr REAL, + flag VARCHAR(1), + mflag VARCHAR(1) +) WITHOUT OIDS; + +\COPY catalogdb.tycho2 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/tycho2/Tycho-2/tycho2_all.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.tycho2 ADD COLUMN designation TEXT; +UPDATE catalogdb.tycho2 SET designation = LPAD(tyc1::TEXT, 4, '0') || '-' || LPAD(tyc2::TEXT, 5, '0') || '-' || tyc3; + +-- Need to remove duplicates in the supplement tables. This assumes that the supplement tables +-- are towards the end of the table, which may not be true, but seems to work. +ALTER TABLE catalogdb.tycho2 ADD COLUMN pk BIGSERIAL; +DELETE FROM catalogdb.tycho2 a USING catalogdb.tycho2 b + WHERE a.designation = b.designation AND a.pk > b.pk; +ALTER TABLE catalogdb.tycho2 DROP COLUMN pk; + +ALTER TABLE catalogdb.tycho2 ADD PRIMARY KEY (designation); + +CREATE INDEX CONCURRENTLY ON catalogdb.tycho2 (q3c_ang2ipix(ramdeg, demdeg)); +CLUSTER tycho2_q3c_ang2ipix_idx on catalogdb.tycho2; +VACUUM ANALYZE catalogdb.tycho2; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE.sql b/schema/sdss5db/catalogdb/unWISE/unWISE.sql new file mode 100644 index 00000000..5543ae22 --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE.sql @@ -0,0 +1,60 @@ +/* + +Schema for unWISE + +Docs: http://catalog.unwise.me/catalogs.html +Files: /uufs/chpc.utah.edu/common/home/sdss10/sdss5/target/catalogs/unwise/release/band-merged + +*/ + +CREATE TABLE catalogdb.unwise ( + x_w1 DOUBLE PRECISION, + x_w2 DOUBLE PRECISION, + y_w1 DOUBLE PRECISION, + y_w2 DOUBLE PRECISION, + flux_w1 REAL, + flux_w2 REAL, + dx_w1 REAL, + dx_w2 REAL, + dy_w1 REAL, + dy_w2 REAL, + dflux_w1 REAL, + dflux_w2 REAL, + qf_w1 REAL, + qf_w2 REAL, + rchi2_w1 REAL, + rchi2_w2 REAL, + fracflux_w1 REAL, + fracflux_w2 REAL, + fluxlbs_w1 REAL, + fluxlbs_w2 REAL, + dfluxlbs_w1 REAL, + dfluxlbs_w2 REAL, + fwhm_w1 REAL, + fwhm_w2 REAL, + spread_model_w1 REAL, + spread_model_w2 REAL, + dspread_model_w1 REAL, + dspread_model_w2 REAL, + sky_w1 REAL, + sky_w2 REAL, + ra12_w1 DOUBLE PRECISION, + ra12_w2 DOUBLE PRECISION, + dec12_w1 DOUBLE PRECISION, + dec12_w2 DOUBLE PRECISION, + coadd_id TEXT, + unwise_detid_w1 TEXT, + unwise_detid_w2 TEXT, + nm_w1 INTEGER, + nm_w2 INTEGER, + primary12_w1 INTEGER, + primary12_w2 INTEGER, + flags_unwise_w1 INTEGER, + flags_unwise_w2 INTEGER, + flags_info_w1 INTEGER, + flags_info_w2 INTEGER, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + primary_status INTEGER, + unwise_objid TEXT PRIMARY KEY +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py new file mode 100755 index 00000000..44e3aee3 --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_generate_csv.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-12 +# @Filename: unWISE_generate_csv.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + + +import glob +import multiprocessing + +from astropy import table + + +def unWISE_to_CSV(filename): + """Converts FITS to CSV for ingestion.""" + + new_table = table.Table() + + file_table = table.Table.read(filename) + + for colname in file_table.colnames: + column = file_table[colname] + if column.ndim == 1: + new_table.add_column(column, copy=False) + else: + w1 = table.Column(column[:, 0], colname + '_w1') + w2 = table.Column(column[:, 1], colname + '_w2') + new_table.add_columns([w1, w2]) + + new_table.write(filename + '.csv', format='ascii.fast_no_header', + delimiter=',', overwrite=True) + + +if __name__ == '__main__': + + files = glob.glob('./*.cat.fits') + + pool = multiprocessing.Pool(25) + pool.map(unWISE_to_CSV, files) diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql new file mode 100644 index 00000000..c2946f75 --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_index.sql @@ -0,0 +1,4 @@ + +CREATE INDEX CONCURRENTLY ON catalogdb.unwise (q3c_ang2ipix(ra, dec)); +CLUSTER unwise_q3c_ang2ipix_idx ON catalogdb.unwise; +ANALYZE catalogdb.unwise; diff --git a/schema/sdss5db/catalogdb/unWISE/unWISE_load b/schema/sdss5db/catalogdb/unWISE/unWISE_load new file mode 100755 index 00000000..326d8524 --- /dev/null +++ b/schema/sdss5db/catalogdb/unWISE/unWISE_load @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# encoding: utf-8 + +ls ./*cat.fits.csv | parallel -j25 "psql -U sdss sdss5db -c \"\copy catalogdb.unwise FROM '{}' WITH DELIMITER ',' NULL '\N' CSV;;\"" diff --git a/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql new file mode 100644 index 00000000..6f19bad9 --- /dev/null +++ b/schema/sdss5db/catalogdb/uvotssc1/uvotssc1.sql @@ -0,0 +1,382 @@ +/* + +II/339 Swift/UVOT Serendipitous Source Catalog (Yershov, 2015) +================================================================================ +Serendipitous UV source catalogues for 10 years of XMM and 5 years of Swift + Yershov V.N. + + =2014Ap&SS.354...97Y +The Swift UVOT Serendipitous Source Catalogue - UVOTSSC (2005-2010), Version 1 + Page M., Yershov V., Breeveld A., Kuin N.P.M., Mignani R.P., + Smith P.J., Rawlings J.I., Oates S.R., Siegel M., Roming P.W.A. + + =2015yCat.2339....0Y + =2014styd.confE..37P (2015arXiv150306597P) +================================================================================ +ADC_Keywords: Photometry, ultraviolet ; Photometry, UBV ; X-ray sources +Mission_Name: Swift + +Description: + The first version of the Swift UVOT serendipitous source catalogue + (UVOTSSC) provides positions and magnitudes, as well as errors and + upper limits of confirmed sources for observations taken from start of + operations in 2005 until October 1st of 2010. + + The first version of the Swift UVOT Serendipitous Source Catalogue + (UVOTSSC) has been produced by processing the image data obtained from + the Swift Ultraviolet and Optical Telescope (UVOT) from the beginning + of the mission (2005) until 1st of October of 2010. The data + processing was performed at the Mullard Space Science Laboratory + (MSSL, University College London, U.K.) using Swift FTOOLS from NASA's + High Energy Astrophysics Software (HEASoft-6.11), with some + customising of the UVOT packages in order to get more complete source + detection and properly apply quality flags to those sources that were + detected within the UVOT image artefacts. The total number of + observations with 17'x17' images used for version 1 of the catalogue + is 23,059, giving 6,200,016 sources in total, of which 2,027,265 have + multiple entries in the source table because they have been detected + in more than one observation. Some sources were only observed in one + filter. The total number of entries in the source table is 13,860,568. + The S/N ratio for all sources exceeds 5 for at least one UVOT filter, + the rest of the filters having a S/N greater than 3. + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +uvotssc1.dat 618 13860568 The Catalogue (6200016 individual sources) +summary.dat 268 23059 Summary information and per image upper limits +uvotssc1_1.fit 2880 1613850 The catalogue (FITS version) +-------------------------------------------------------------------------------- + +See also: + http://www.ucl.ac.uk/mssl/astro/space_missions/swift/uvotssc : Swift/UVOT + Serendipitous Source Catalog home page + J/ApJ/725/1215 : Faint UV standards from Swift, GALEX and SDSS (Siegel+, 2010) + J/AJ/137/4517 : UVOT light curves of supernovae (Brown+, 2009) + J/AJ/141/205 : UVOT imaging of M81 and Holmberg IX (Hoversten+, 2011) + J/MNRAS/424/1636 : Swift/UVOT sources in NGC4321 (M100) (Ferreras+, 2012) + J/other/ATel/5200 : VizieR Online Data Catalog: Swift Galactic Plane Survey: + sourcelist v3 (Reynolds+, 2013) + +Byte-by-byte Description of file: uvotssc1.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 8 A8 --- --- [UVOTSSC1] + 10- 26 A17 --- Name UVOTSSC1 name (JHHMMSS.s+DDMMSSa) (IAUNAME) + 28- 32 I5 --- Oseq [1/23059] Reference number in the observation + table (N_SUMMARY) + 34- 44 I011 --- ObsID Unique Swift observation ID (OBSID) + 46 I1 --- Nf [1/6] Number of filters included in this + observation (NFILT) + 48- 54 I7 --- SrcID [1/6200016] Unique source number (SRCNUM) + 56- 65 F10.6 deg RAdeg Right ascension (J2000) (RA) (1) + 67- 76 F10.6 deg DEdeg Declination (J2000) (DEC) (1) + 78- 84 F7.3 arcsec e_RAdeg [0.001/15] Right ascension error (RA_ERR) (1) + 86- 92 F7.3 arcsec e_DEdeg [0.001/21] Declination error (DEC_ERR) (1) + 94- 99 F6.3 arcsec rUVW2 [0/30]? Distance to closest UVW2 source + (UVW2_SRCDIST) + 101-106 F6.3 arcsec rUVM2 [0/30]? Distance to closest UVM2 source + (UVM2_SRCDIST) + 108-113 F6.3 arcsec rUVW1 [0/30]? Distance to closest UVW1 source + (UVW1_SRCDIST) + 115-120 F6.3 arcsec rU [0/30]? Distance to closest U source + (U_SRCDIST) + 122-127 F6.3 arcsec rB [0/30]? Distance to closest B source + (B_SRCDIST) + 129-134 F6.3 arcsec rV [0/30]? Distance to closest V source + (V_SRCDIST) + 136-138 I3 --- Nd [1/208] Number of individual observations + (N_OBSID) (2) + 140-145 F6.1 --- sUVW2 [3/5813]? Significance (S/N) in UVW2 + (UVW2_SIGNIF) + 147-152 F6.1 --- sUVM2 [3/5000]? Significance (S/N) in UVM2 + (UVM2_SIGNIF) + 154-159 F6.1 --- sUVW1 [3/5000]? Significance (S/N) in UVW1 + (UVW1_SIGNIF) + 161-166 F6.1 --- sU [3/5000]? Significance (S/N) in U (U_SIGNIF) + 168-173 F6.1 --- sB [3/5000]? Significance (S/N) in B (B_SIGNIF) + 175-180 F6.1 --- sV [3/5000]? Significance (S/N) in V (V_SIGNIF) + 182-188 F7.4 mag UVW2 ? UVOT/UVW2 Vega magnitude (UVW2_VEGAMAG) (3) + 190-196 F7.4 mag UVM2 ? UVOT/UVM2 Vega magnitude (UVM2_VEGAMAG) (3) + 198-204 F7.4 mag UVW1 ? UVOT/UVW1 Vega magnitude (UVW1_VEGAMAG) (3) + 206-212 F7.4 mag Umag ? UVOT/U Vega magnitude (U_VEGAMAG) (3) + 214-220 F7.4 mag Bmag ? UVOT/N Vega magnitude (B_VEGAMAG) (3) + 222-228 F7.4 mag Vmag ? UVOT/V Vega magnitude (V_VEGAMAG) (3) + 230-236 F7.4 mag UVW2-AB ? UVOT/UVW2 AB magnitude (UVW2_ABMAG) (3) + 238-244 F7.4 mag UVM2-AB ? UVOT/UVM2 AB magnitude (UVM2_ABMAG) (3) + 246-252 F7.4 mag UVW1-AB ? UVOT/UVW1 AB magnitude (UVW1_ABMAG) (3) + 254-260 F7.4 mag U-AB ? UVOT/U AB magnitude (U_ABMAG) (3) + 262-268 F7.4 mag B-AB ? UVOT/B AB magnitude (B_ABMAG) (3) + 270-276 F7.4 mag V-AB ? UVOT/V AB magnitude (V_ABMAG) (3) + 278-283 F6.4 mag e_UVW2 ? Error on UVW2 magnitude (UVW2_MAG_ERR) + 285-290 F6.4 mag e_UVM2 ? Error on UVM2 magnitude (UVM2_MAG_ERR) + 292-297 F6.4 mag e_UVW1 ? Error on UVW2 magnitude (UVW1_MAG_ERR) + 299-304 F6.4 mag e_Umag ? Error on U magnitude (U_MAG_ERR) + 306-311 F6.4 mag e_Bmag ? Error on B magnitude (B_MAG_ERR) + 313-318 F6.4 mag e_Vmag ? Error on V magnitude (V_MAG_ERR) + 320-331 E12.6 cW/m2/nm F.UVW2 ? UVOT/UVW2 Flux (UVW2_FLUX) (3) + 333-344 E12.6 cW/m2/nm F.UVM2 ? UVOT/UVW2 Flux (UVM2_FLUX) (3) + 346-357 E12.6 cW/m2/nm F.UVW1 ? UVOT/UVW2 Flux (UVW1_FLUX) (3) + 359-370 E12.6 cW/m2/nm F.U ? UVOT/UVW2 Flux (U_FLUX) (3) + 372-383 E12.6 cW/m2/nm F.B ? UVOT/UVW2 Flux (B_FLUX) (3) + 385-396 E12.6 cW/m2/nm F.V ? UVOT/UVW2 Flux (V_FLUX) (3) + 398-407 E10.4 cW/m2/nm e_F.UVW2 ? Error on F.UVW2 (UVW2_FLUX_ERR) + 409-418 E10.4 cW/m2/nm e_F.UVM2 ? Error on F.UVM2 (UVM2_FLUX_ERR) + 420-429 E10.4 cW/m2/nm e_F.UVW1 ? Error on F.UVW1 (UVW1_FLUX_ERR) + 431-440 E10.4 cW/m2/nm e_F.U ? Error on F.U (U_FLUX_ERR) + 442-451 E10.4 cW/m2/nm e_F.B ? Error on F.B (B_FLUX_ERR) + 453-462 E10.4 cW/m2/nm e_F.V ? Error on F.V (V_FLUX_ERR) + 464-469 F6.3 arcsec aUVW2 ? Major axis in UVW2 (UVW2_MAJOR) + 471-476 F6.3 arcsec aUVM2 ? Major axis in UVM2 (UVM2_MAJOR) + 478-483 F6.3 arcsec aUVW1 ? Major axis in UVW1 (UVW1_MAJOR) + 485-490 F6.3 arcsec aU ? Major axis in U (U_MAJOR) + 492-497 F6.3 arcsec aB ? Major axis in B (B_MAJOR) + 499-504 F6.3 arcsec aV ? Major axis in V (V_MAJOR) + 506-511 F6.3 arcsec bUVW2 ? Minor axis in UVW2 (UVW2_MINOR) + 513-518 F6.3 arcsec bUVM2 ? Minor axis in UVM2 (UVM2_MINOR) + 520-525 F6.3 arcsec bUVW1 ? Minor axis in UVW1 (UVW1_MINOR) + 527-532 F6.3 arcsec bU ? Minor axis in U (U_MINOR) + 534-539 F6.3 arcsec bB ? Minor axis in B (B_MINOR) + 541-546 F6.3 arcsec bV ? Minor axis in V (V_MINOR) + 548-552 F5.2 deg paUVW2 [0/90]? Position angle of major axis in UVW2 + (UVW2_POSANG) + 554-558 F5.2 deg paUVM2 [0/90]? Position angle of major axis in UVM2 + (UVM2_POSANG) + 560-564 F5.2 deg paUVW1 [0/90]? Position angle of major axis in UVW1 + (UVW1_POSANG) + 566-570 F5.2 deg paU [0/90]? Position angle of major axis in U + (U_POSANG) + 572-576 F5.2 deg paB [0/90]? Position angle of major axis in B + (B_POSANG) + 578-582 F5.2 deg paV [0/90]? Position angle of major axis in V + (V_POSANG) + 584 I1 --- xUVW2 [0/1] Extended flag in UVW2 (UVW2_EXTENDED) + 586 I1 --- xUVM2 [0/1] Extended flag in UVW2 (UVM2_EXTENDED) + 588 I1 --- xUVW1 [0/1] Extended flag in UVW2 (UVW1_EXTENDED) + 590 I1 --- xU [0/1] Extended flag in UVW2 (U_EXTENDED) + 592 I1 --- xB [0/1] Extended flag in UVW2 (B_EXTENDED) + 594 I1 --- xV [0/1] Extended flag in UVW2 (V_EXTENDED) + 596-598 I3 --- fUVW2 [0/511]?=- Quality flags in UVW2 + (UVW2_QUALITY_FLAG) (4) + 600-602 I3 --- fUVM2 [0/511]?=- Quality flags in UVM2 + (UVM2_QUALITY_FLAG) (4) + 604-606 I3 --- fUVW1 [0/511]?=- Quality flags in UVW1 + (UVW1_QUALITY_FLAG) (4) + 608-610 I3 --- fU [0/511]?=- Quality flags in U (U_QUALITY_FLAG) + 612-614 I3 --- fB [0/511]?=- Quality flags in B (B_QUALITY_FLAG) + 616-618 I3 --- fV [0/511]?=- Quality flags in V (V_QUALITY_FLAG) +-------------------------------------------------------------------------------- +Note (1): standard error of position in RA and Dec, i.e. (1/n)sqrt({Sigma}err) + +Note (2): this number should correspond to the number of entries for that + particular source in the table. + +Note (3): Fluxes are expressed in erg/cm^2^/s/{AA}=cW/m^2^/nm. + The fluxes were derived using the count rate to flux conversion + factors from Poole et al. (2008MNRAS.383..627P) which have been based + on GRB spectra. This differs slightly from the count rate to flux + conversion factor based on Pickles star spectra expressed as a flux + ratio (rightmost column); note that the flux conversion factors are + valid for UVOT B-V > -0.36 + ------------------------------------------------ + Filter Central Wavelength FWHM F.filter + (nm) (nm) (Pickles/grb) + ------------------------------------------------ + V 546.8 76.9 0.998 + B 439.2 97.5 0.893 + U 346.5 78.5 0.940 + UVW1 260.0 69.3 0.963 + UVM2 224.6 49.8 0.884 + UVW2 192.8 65.7 0.965 + ------------------------------------------------ + The UVOT UV filters are physically similar to those in XMM-OM but have + greater throughput. The UVOT U,B,V are not the Johnson U,B,V passbands. + +Note (4): Quality flags are; + 1 = Cosmetic defects (BAD PIXELS) within the source region + 2 = Source on a READOUT STREAK + 4 = Source on a "SMOKE RING" + 8 = Source on a DIFFRACTION SPIKE + 16 = Source affected by MOD-8 noise pattern + 32 = Source within a "HALO RING" + 64 = Source near to a BRIGHT source + 128 = MULTIPLE EXPOSURE values within photometry aperture + 256 = Source within an EXTENDED FEATURE + Multiple flags are summed to give the final quality flag value. +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: summary.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 5 I5 --- Oseq [1/23059] Reference number (N_SUMMARY) + 7- 17 I011 --- ObsID Unique Swift observation ID (OBSID) (5) + 19- 36 A18 --- Target Swift original target identifier (TARGET_ID) + 38- 57 A20 --- Filters List of filters used this OBSID (FILTERS) + 59- 77 A19 "datime" StartDate Earliest observation date and time, UT format + (DATE_MIN) + 79- 88 F10.6 deg ptRAdeg Pointing RA (ICRS) of image centre (RA_PNT) + 90- 99 F10.6 deg ptDEdeg Pointing Dec (ICRS) of image centre (DEC_PNT) + 101-105 F5.1 deg ptPA [0/360] Position angle image centre (PA_PNT) + 107-113 F7.1 s exp.UVW2 ?Total exposure time in UVW2 (EXP_UVW2) + 115-121 F7.1 s exp.UVM2 ?Total exposure time in UVM2 (EXP_UVM2) + 123-129 F7.1 s exp.UVW1 ?Total exposure time in UVW1(EXP_UVW1) + 131-137 F7.1 s exp.U ?Total exposure time in U (EXP_U) + 139-145 F7.1 s exp.B ?Total exposure time in B(EXP_B) + 147-153 F7.1 s exp.V ?Total exposure time in V (EXP_V) + 155-158 I4 --- n.UVW2 ?Number of UVW2 sources (NSOURCES_UVW2) + 160-163 I4 --- n.UVM2 ?Number of UVM2 sources (NSOURCES_UVM2) + 165-168 I4 --- n.UVW1 ?Number of UVW1 sources (NSOURCES_UVW1) + 170-173 I4 --- n.U ?Number of U sources (NSOURCES_U) + 175-178 I4 --- n.B ?Number of B sources (NSOURCES_B) + 180-184 I5 --- n.V ?Number of V sources (NSOURCES_V) + 186-191 F6.3 mag ul.UVW2 ?Limiting mag UVW2 (VEGAMAG_LIM_UVW2) (6) + 193-198 F6.3 mag ul.UVM2 ?Limiting mag UVM2 (VEGAMAG_LIM_UVM2) (6) + 200-205 F6.3 mag ul.UVW1 ?Limiting mag UVW1 (VEGAMAG_LIM_UVW1) (6) + 207-212 F6.3 mag ul.U ?Limiting mag U (VEGAMAG_LIM_U) (6) + 214-219 F6.3 mag ul.B ?Limiting mag B (VEGAMAG_LIM_B) (6) + 221-226 F6.3 mag ul.V ?Limiting mag V (VEGAMAG_LIM_V) (6) + 228-233 F6.3 mag ul.UVW2-AB ?Limiting AB-mag UVW2 (ABMAG_LIM_UVW2) (6) + 235-240 F6.3 mag ul.UVM2-AB ?Limiting AB-mag UVM2 (ABMAG_LIM_UVM2) (6) + 242-247 F6.3 mag ul.UVW1-AB ?Limiting AB-mag UVW1 (ABMAG_LIM_UVW1) (6) + 249-254 F6.3 mag ul.U-AB ?Limiting AB-mag U (ABMAG_LIM_U) (6) + 256-261 F6.3 mag ul.B-AB ?Limiting AB-mag B (ABMAG_LIM_B) (6) + 263-268 F6.3 mag ul.V-AB ?Limiting AB-mag V (ABMAG_LIM_V) (6) +-------------------------------------------------------------------------------- +Note (5): One Swift OBSID can consist of one or more images, which for this + catalogue have been summed, yielding the quoted total exposure time. + The original uvot images can be found in the on-line archives at + MAST, the Swift archive at swift.ac.uk and swift.gsfc.nasa.gov using + the OBSID as search key. For higher temporal resolution the original + images need to be used because the catalogue data sum over all + images within an OBSID. + +Note (6): The upper limits per filter of the summed image constructed + for each OBSID because the sensitivity hardly varies over the + detector. Usually the images within one OBSID share the same + pointing, however, whereas the quoted upper limits apply always for + sources near the pointing direction given, if the images had small + offsets in pointing they may not apply to sources near the edge of + the summed image, which is typically about 8 arc seconds from the + pointing. +-------------------------------------------------------------------------------- + +History and notes: + The initially released version of the catalogue (2015) was done + with the source identifier "SWIFTUVOT" for each source, and was + made available in that form. The decision was made to rename the + catalogue sources by including the catalogue version number. In + addition, in a few instances multiple SOURCE IDs shared the same + IAU NAME. They will be distinguished by having a letter a,b,c,.. + appended to their NAME. Sources brighter than 0.96 counts per frame + have not been included. Their coincidence loss is too large to + correct for. +================================================================================ +(End) Paul Kuin [MSSL/UCL], Francois Ochsenbein [CDS] 27-November-2015 + +This uvotssc1.csv file was generating by taking uvotssc1.dat and ReadMe from +http://cdsarc.u-strasbg.fr/ftp/cats/II/339/, reading the file with astropy as + +table = astropy.io.ascii.read('./uvotssc1.dat', readme='./ReadMe', + format='cds', fill_values=[('---', '0')]) +table.remove_column('---') +table.write('./uvotssc1.csv', format='ascii.csv') + +*/ + +CREATE TABLE catalogdb.uvotssc1 ( + name VARCHAR(17), + oseq BIGINT, + obsid BIGINT, + nf BIGINT, + srcid BIGINT, + radeg DOUBLE PRECISION, + dedeg DOUBLE PRECISION, + e_radeg DOUBLE PRECISION, + e_dedeg DOUBLE PRECISION, + ruvw2 REAL, + ruvm2 REAL, + ruvw1 REAL, + ru REAL, + rb REAL, + rv REAL, + nd BIGINT, + suvw2 REAL, + suvm2 REAL, + suvw1 REAL, + su REAL, + sb REAL, + sv REAL, + uvw2 DOUBLE PRECISION, + uvm2 DOUBLE PRECISION, + uvw1 DOUBLE PRECISION, + umag DOUBLE PRECISION, + bmag DOUBLE PRECISION, + vmag DOUBLE PRECISION, + uvw2_ab DOUBLE PRECISION, + uvm2_ab DOUBLE PRECISION, + uvw1_ab DOUBLE PRECISION, + u_ab DOUBLE PRECISION, + b_ab DOUBLE PRECISION, + v_ab DOUBLE PRECISION, + e_uvw2 DOUBLE PRECISION, + e_uvm2 DOUBLE PRECISION, + e_uvw1 DOUBLE PRECISION, + e_umag DOUBLE PRECISION, + e_bmag DOUBLE PRECISION, + e_vmag DOUBLE PRECISION, + f_uvw2 DOUBLE PRECISION, + f_uvm2 DOUBLE PRECISION, + f_uvw1 DOUBLE PRECISION, + f_u DOUBLE PRECISION, + f_b DOUBLE PRECISION, + f_v DOUBLE PRECISION, + e_f_uvw2 DOUBLE PRECISION, + e_f_uvm2 DOUBLE PRECISION, + e_f_uvw1 DOUBLE PRECISION, + e_f_u DOUBLE PRECISION, + e_f_b DOUBLE PRECISION, + e_f_v DOUBLE PRECISION, + auvw2 DOUBLE PRECISION, + auvm2 DOUBLE PRECISION, + auvw1 DOUBLE PRECISION, + au DOUBLE PRECISION, + ab DOUBLE PRECISION, + av DOUBLE PRECISION, + buvw2 DOUBLE PRECISION, + buvm2 DOUBLE PRECISION, + buvw1 DOUBLE PRECISION, + bu DOUBLE PRECISION, + bb DOUBLE PRECISION, + bv DOUBLE PRECISION, + pauvw2 REAL, + pauvm2 REAL, + pauvw1 REAL, + pau REAL, + pab REAL, + pav REAL, + xuvw2 INTEGER, + xuvm2 INTEGER, + xuvw1 INTEGER, + xu INTEGER, + xb INTEGER, + xv INTEGER, + fuvw2 INTEGER, + fuvm2 INTEGER, + fuvw1 INTEGER, + fu INTEGER, + fb INTEGER, + fv INTEGER +) WITHOUT OIDS; + +\COPY catalogdb.uvotssc1 FROM /uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/uvotssc1.csv WITH CSV HEADER DELIMITER ','; + +ALTER TABLE catalogdb.uvotssc1 ADD id BIGSERIAL PRIMARY KEY; + +CREATE INDEX CONCURRENTLY ON catalogdb.uvotssc1 (q3c_ang2ipix(radeg, dedeg)); +CLUSTER uvotssc1_q3c_ang2ipix_idx ON catalogdb.uvotssc1; +VACUUM ANALYZE catalogdb.uvotssc1; diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1.sql b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1.sql similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1.sql rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1.sql diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_index.sql b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_index.sql similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_index.sql rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_index.sql diff --git a/schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_load b/schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_load similarity index 100% rename from schema/sdss5db/catalogdb/wd/v1/gaia_dr2_wd_candidates_v1_load rename to schema/sdss5db/catalogdb/wd/deprecated/gaia_dr2_wd_candidates_v1_load diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql new file mode 100644 index 00000000..bba188d2 --- /dev/null +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2.sql @@ -0,0 +1,349 @@ +/* + +From Gentile Fusillo N.P. , Tremblay P-E, Gaensicke B.T. et..al, 2019MNRAS.482.4570G + +Downloaded from https://cdsarc.unistra.fr/viz-bin/cat/J/MNRAS/482/4570 + +J/MNRAS/482/4570 Gaia DR2 white dwarf candidates (Gentile Fusillo+, 2019) +================================================================================ +A Gaia Data Release 2 catalogue of white dwarfs and a comparison with SDSS. + Gentile Fusillo N.P., Tremblay P-E, Gaensicke B.T., Manser C.J., + Cunningham T., Cukanovaite E., Hollands M., Marsh T., Raddi R., + Jordan S., Toonen S., Geier S., Barstow M., Cummings J.D. + + =2019MNRAS.482.4570G (SIMBAD/NED BibCode) +================================================================================ +ADC_Keywords: Surveys ; Stars, white dwarf ; Parallaxes, trigonometric; + Photometry +Keywords: catalogues - surveys - white dwarfs + +Abstract: + We present a catalogue of white dwarf candidates selected from the + second data release of Gaia (DR2). We used a sample of + spectroscopically confirmed white dwarfs from the Sloan Digital Sky + Survey (SDSS) to map the entire space spanned by these objects in the + Gaia Hertzsprung-Russell diagram. + + We then defined a set of cuts in absolute magnitude, colour, and a + number of Gaia quality flags to remove the majority of contaminating + objects. Finally, we adopt a method analogous to the one presented in + our earlier SDSS photometric catalogues to calculate a probability of + being a white dwarf (PWD) for all Gaia sources which passed the + initial selection. The final catalogue is composed of 486641 stars + with calculated PWD from which it is possible to select a sample + of~260000 high-confidence white dwarf candidates in the magnitude + range 87000K, at high + Galactic latitudes (|b|>20deg). However, the completeness drops at + low Galactic latitudes, and the magnitude limit of the catalogue + varies significantly across the sky as a function of Gaia's scanning + law. We also provide the list of objects within our sample with + available SDSS spectroscopy. + +Description: + The main catalogue provides 486,641 stars selected from Gaia DR2 with + calculated probabilities of being a white dwarf (PWD). The PWD values + can used to reliably select high-confidence white dwarf candidates + with a flexible compromise between completeness and level of potential + contamination. As a generic guideline selecting objects with PWD>0.75 + recovers 96 per cent of the spectroscopically confirmed white dwarfs + from SDSS and only 1 per cent of the contaminant (non white dwarfs) + objects. + + All Gaia sources in the catalogue have also been cross matched with + SDSS DR14 taking into account the difference in epoch of observation + and proper motions. Whether available we include SDSS ugriz + photometry. In a separate table we provide informations on all the + available SDSS spectra for the Gaia sources in the main catalogue. + +File Summary: +-------------------------------------------------------------------------------- + FileName Lrecl Records Explanations +-------------------------------------------------------------------------------- +ReadMe 80 . This file +gaia2wd.dat 1008 486641 DR2 white dwarf candidates, corrected version + (gaia_dr2_white_dwarf_candidates.dat) +gaiasdss.dat 344 37259 Available SDSS spectra of candidates, corrected + version (gaia-sdss_white_dwarf_catalogue.dat) +-------------------------------------------------------------------------------- + +See also: + B/wd : Spectroscopically identified white dwarfs (McCook+, 2014) + I/345 : Gaia DR2 (Gaia Collaboration, 2018) + V/147 : The SDSS Photometric Catalogue, Release 12 (Alam+, 2015) + J/MNRAS/465/2849 : Gaia DR1 mass-radius relation of WD (Tremblay+ 2017) + +Byte-by-byte Description of file: gaia2wd.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 23 A23 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 25- 52 A28 --- DR2Name Unique Gaia source designation (designation) + 54- 72 I19 --- Source Unique Gaia source identifier (source_id) + 74- 96 F23.19 deg RAdeg Gaia DR2 barycentric right ascension (ICRS) + at epoch J2015.5 (ra) + 98- 117 F20.18 mas e_RAdeg Standard error of right ascension (ra_error) + 119- 140 E22.18 deg DEdeg Gaia DR2 barycentric declination (ICRS) + at epoch J2015.5 (dec) + 142- 161 F20.18 mas e_DEdeg Standard error of declination (dec_error) + 163- 183 F21.17 mas Plx Absolute stellar parallax of the source + at J2015.5 (parallax) + 185- 204 F20.18 mas e_Plx Standard error of parallax (parallax_error) + 206- 227 E22.19 mas/yr pmRA Proper motion in right ascension + (pmRAxcosDE) (pmra) + 229- 248 F20.18 mas/yr e_pmRA Standard error of proper motion in + right ascension (pmra_error) + 250- 271 E22.19 mas/yr pmDE Proper motion in declination (pmdec) + 273- 292 F20.18 mas/yr e_pmDE Standard error of proper motion + in declination (pmdec_error) + 294- 315 F22.19 mas epsi Measure of the residuals in the astrometric + solution for the source + (astrometric_excess_noise) + 317- 327 F11.9 mas amax Five-dimensional equivalent to the + semi-major axis of the Gaia position error + ellipse (astrometric_sigma5d_max) + 329- 350 F22.14 e-/s FG Gaia G-band mean flux (phot_g_mean_flux) + 352- 374 F23.17 e-/s e_FG Error on G-band mean flux + (phot_g_mean_flux_error) + 376- 385 F10.7 mag Gmag Gaia G-band mean magnitude (Vega scale) + (phot_g_mean_mag) + 387- 409 F23.15 e-/s FBP Integrated GBP mean flux (phot_bp_mean_flux) + 411- 435 F25.18 e-/s e_FBP Error on GBP-band mean flux + (phot_bp_mean_flux_error) + 437- 446 F10.7 mag BPmag Gaia GBP-band mean magnitude (Vega scale) + (phot_bp_mean_mag) + 448- 470 F23.15 e-/s FRP Integrated GRP mean flux (phot_rp_mean_flux) + 472- 497 F26.19 e-/s e_FRP Error on GRP-band mean flux + (phot_rp_mean_flux_error) + 499- 508 F10.7 mag RPmag Gaia GRP-band mean magnitude (Vega scale) + (phot_rp_mean_mag) + 510- 520 F11.9 --- E(BR/RP) GBP/GRP excess factor estimated from the + comparison of the sum of integrated GBP and + GRP fluxes with respect to the flux in the + G-band (phot_bp_rp_excess_factor) + 522- 542 E21.19 deg GLON Galactic longitude (l) + 544- 565 E22.19 deg GLAT Galactic latitude (b) + 567- 578 F12.4 --- Density The number of Gaia sources per square degree + around this object (density) + 580- 602 F23.19 mag AG Extinction in the Gaia G-band band derived + from E(B-V) values from Schlafly and + Finkbeiner (2011ApJ...737..103S) (AG) + 604- 622 A19 --- SDSS SDSS object name if available + (JHHMMSS,.ss+DDMMSS.s, J2000) (SDSS_name) + 624- 641 F18.15 mag umag ? SDSS u band magnitude (umag) + 643- 664 F22.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 666- 683 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 685- 706 F22.19 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 708- 725 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 727- 747 F21.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 749- 766 F18.15 mag imag ? SDSS i band magnitude (imag) + 768- 789 F22.19 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 791- 808 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 810- 830 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 832- 852 E21.19 --- Pwd [0/1]? The probability of being a white + dwarf (Pwd) + 854 I1 --- f_Pwd [0/1] If 1 it indicates the PWD value + could be unreliable (Pwd_flag) + 856- 868 F13.6 K TeffH ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-H model atmospheres (Teff_H) + 870- 882 F13.6 K e_TeffH ? Uncertainty on Teff_H (eTeff_H) + 884- 891 F8.6 --- loggH ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-H model atmospheres (log_g_H) + 893- 900 E8.6 --- e_loggH ? Uncertainty on log_g_H (elog_g_H) + 902- 909 F8.6 Msun MassH ? Stellar mass resulting from the adopted + mass-radius relation (mass_H) + 911- 919 E9.6 Msun e_MassH ? Uncertainty on the mass (emass_H) + 921- 931 E11.6 --- chi2H ? chi2 value of the fit (pure-H) (chi2_H) + 933- 944 F12.6 K TeffHe ? Effective temperature from fitting the + dereddened G,GBP, and GRP absolute fluxes + with pure-He model atmospheres (Teff_He) + 946- 959 F14.6 K e_TeffHe ? Uncertainty on Teff_He (eTeff_He) + 961- 968 F8.6 --- loggHe ? Surface gravity from fitting the + dereddened G,GBP,and GRP absolute fluxes + with pure-He model atmospheres (log_g_He) + 970- 977 E8.6 --- e_loggHe ? Uncertainty on log_g_He (elog_g_He) + 979- 986 F8.6 Msun MassHe ? Stellar mass resulting from the adopted + mass-radius relation (mass_He) + 988- 996 E9.6 Msun e_MassHe ? Uncertainty on the mass (emass_He) + + 998-1008 E11.6 --- chisqHe ? chi2 value of the fit (pure-H) (chisq_He) +-------------------------------------------------------------------------------- + +Byte-by-byte Description of file: gaiasdss.dat +-------------------------------------------------------------------------------- + Bytes Format Units Label Explanations +-------------------------------------------------------------------------------- + 1- 22 A22 --- WD WD name (WDJHHMMSS.ss+DDMMSS.ss, J2000) + (white_dwarf_name) + 23 A1 --- n_WD [a] Multiplicity index on WD + 24- 42 I19 --- Source Unique Gaia source identifier (source_id) + 44- 62 A19 --- SDSS SDSS object name (JHHMMSS,.ss+DDMMSS.s, J2000) + (SDSS_name) + 64- 83 F20.16 deg RAdeg Right ascension (J2000) of the spectrum source + from SDSS DR14 (SDSS_ra) + 85- 98 E14.10 deg DEdeg Declination (J2000) of the spectrum source + from SDSS DR14 (SDSS_dec) + 100-117 F18.15 mag umag ? SDSS u band magnitude (umag) + 119-139 F21.18 mag e_umag ? SDSS u band magnitude uncertainty (e_umag) + 141-158 F18.15 mag gmag ? SDSS g band magnitude (gmag) + 160-179 F20.18 mag e_gmag ? SDSS g band magnitude uncertainty (e_gmag) + 181-198 F18.15 mag rmag ? SDSS r band magnitude (rmag) + 200-219 F20.18 mag e_rmag ? SDSS r band magnitude uncertainty (e_rmag) + 221-238 F18.15 mag imag ? SDSS i band magnitude (imag) + 240-260 F21.18 mag e_imag ? SDSS i band magnitude uncertainty (e_imag) + 262-279 F18.15 mag zmag ? SDSS z band magnitude (zmag) + 281-301 F21.18 mag e_zmag ? SDSS z band magnitude uncertainty (e_zmag) + 303-306 I4 --- Plate Identifier of the plate used in the + observation of the spectrum (Plate) + 308-312 I5 --- MJD Modified Julian date of the observation of + the spectrum (mjd) + 314-317 I4 --- Fiber Identifier of the fiber used in the + observation of the spectrum (fiberID) + 319-333 F15.11 --- S/N Signal-to-noise ratio of the spectrum + calculated in the range 4500-5500 A (S/N) + 335-344 A10 --- SpClass Classification of the object based on a visual + inspection of the SDSS spectrum + (spectral_class) (1) +-------------------------------------------------------------------------------- +Note (1): spectral classes are as follows: + CV = Cataclysmic Variable star + DA = white dwarfs with H lines only + DAB/DAB = white dwarfs with H lines and neutral He lines + DABZ/DBAZ/DZAB/DZBA = white dwarfs with H lines, neutral He lines and + Ca H & K lines + DAO/DO = white dwarfs with H lines and He II lines + DAZ/DZA = white dwarfs with H lines and Ca H & K lines + DB = white dwarfs with neutral He lines only + DBZ/DZB = white dwarfs with neutral He lines and Ca H & K lines + DC = white dwarfs with feature-less spectra + DQ = white dwarfs with molecular C absorptions + hotDQ = white dwarfs with atomic C absorption + DQpec = Peculiar white dwarfs with molecular C absorptions + DZ = white dwarfs with Ca H & K lines + DAH = white dwarfs with Zeeman split H lines + DBH = white dwarfs with Zeeman split He lines + DZH = white dwarfs with Zeeman split Ca H & K lines + MWD = Magnetic white dwarfs with unidentified absorption lines + WDPec = Peculiar magnetic white dwarf + PG1159 = Pre-white dwarf star + DB+MS/DA+MS/DC+M = White Dwarf - Main Sequence binary + STAR = non white dwarf stellar object + QSO = Quasar + UNK = Spectra which we were unable to classify + Unreliable = Unreliable spectra +-------------------------------------------------------------------------------- + +Acknowledgements: + Nicola Gentile Fusillo, n.gentile-fusillo(at)warwick.ac.uk + +History: + * 01-Feb-2019: on-line version + * 01-Apr-2019: corrected version (from author) + +================================================================================ +(End) Patricia Vannier [CDS] 31-Jan-2019 + +Modifications to datamodels: + +- All columns to lower case. +- All column except coordinates have been converted from float64 to float32. +- Added a source_id column that is constructed taking the integer of the +- dr2name designation (to be used for foreign keys). +- RAdeg, DEdeg, e_RAdeg, e_DEdeg -> ra, dec, e_ra, e_dec +- pm_de, e_pmde -> pm_dec, e_pmdec +- FG, e_FG Gmag -> fg_gaia, e_fg_gaia, g_gaia_mag +- E(BR/RP) -> e_br_rp +- (S/N) -> snr + +*/ + + +CREATE TABLE catalogdb.gaia_dr2_wd ( + wd TEXT, + dr2name TEXT, + source_id BIGINT PRIMARY KEY, + source INTEGER, + ra DOUBLE PRECISION, + e_ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + e_dec DOUBLE PRECISION, + plx REAL, + e_plx REAL, + pmra DOUBLE PRECISION, + e_pmra DOUBLE PRECISION, + pmdec DOUBLE PRECISION, + e_pmdec DOUBLE PRECISION, + epsi REAL, + amax REAL, + fg_gaia REAL, + e_fg_gaia REAL, + g_gaia_mag REAL, + fbp REAL, + e_fbp REAL, + bpmag REAL, + frp REAL, + e_frp REAL, + rpmag REAL, + e_br_rp REAL, + glon DOUBLE PRECISION, + glat DOUBLE PRECISION, + density REAL, + ag REAL, + sdss TEXT, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + pwd REAL, + f_pwd INTEGER, + teffh REAL, + e_teffh REAL, + loggh REAL, + e_loggh REAL, + massh REAL, + e_massh REAL, + chi2h REAL, + teffhe REAL, + e_teffhe REAL, + logghe REAL, + e_logghe REAL, + masshe REAL, + e_masshe REAL, + chisqhe REAL +) WITHOUT OIDS; + + +CREATE TABLE catalogdb.gaia_dr2_wd_sdss ( + pk SERIAL PRIMARY KEY, + wd TEXT, + n_wd TEXT, + source INTEGER, + sdss TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + umag REAL, + e_umag REAL, + gmag REAL, + e_gmag REAL, + rmag REAL, + e_rmag REAL, + imag REAL, + e_imag REAL, + zmag REAL, + e_zmag REAL, + plate INTEGER, + mjd INTEGER, + fiber INTEGER, + snr REAL, + spclass TEXT +) WITHOUT OIDS; diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql new file mode 100644 index 00000000..ec97b18f --- /dev/null +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_index.sql @@ -0,0 +1,15 @@ + +CREATE UNIQUE INDEX gaia_dr2_wd_wd_key ON catalogdb.gaia_dr2_wd(wd text_ops); + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd; +ANALYZE catalogdb.gaia_dr2_wd; + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss (q3c_ang2ipix(ra, dec)); +CLUSTER gaia_dr2_wd_sdss_q3c_ang2ipix_idx ON catalogdb.gaia_dr2_wd_sdss; +ANALYZE catalogdb.gaia_dr2_wd_sdss; + +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (plate); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (mjd); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss USING BTREE (fiber); +CREATE INDEX CONCURRENTLY ON catalogdb.gaia_dr2_wd_sdss (mjd, plate, fiber); diff --git a/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py new file mode 100755 index 00000000..eef12050 --- /dev/null +++ b/schema/sdss5db/catalogdb/wd/dr2/wd_dr2_load.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-17 +# @Filename: wd_dr2_load.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import astropy.table +import numpy + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + replacements = {'RAdeg': 'ra', + 'DEdeg': 'dec', + 'e_RAdeg': 'e_ra', + 'e_DEdeg': 'e_dec', + 'pmDE': 'pmdec', + 'e_pmDE': 'e_pmdec', + 'E(BR/RP)': 'e_br_rp', + '(S/N)': 'snr', + 'FG': 'fg_gaia', + 'e_FG': 'e_fg_gaia', + 'Gmag': 'g_gaia_mag'} + + path_ = ('/uufs/chpc.utah.edu/common/home/sdss50/sdsswork/target/catalogs/wd/dr2/') + + for file_, table in (('J_MNRAS_482_4570_gaia2wd.dat.gz.fits.gz', 'gaia_dr2_wd'), + ('J_MNRAS_482_4570_gaiasdss.dat.gz.fits.gz', 'gaia_dr2_wd_sdss')): + + data = astropy.table.Table.read(path_ + file_) + + for column_name in data.colnames: + if column_name in replacements: + new_column_name = replacements[column_name] + else: + new_column_name = column_name.lower() + data.rename_column(column_name, new_column_name) + + data['wd'][:] = numpy.core.defchararray.strip(data['wd'].data) + + if 'gaia2wd' in file_: + name_split = numpy.core.defchararray.split(data['dr2name']) + source_id = numpy.array(list(zip(*name_split))[2], numpy.int64) + data.add_column(astropy.table.Column(source_id, 'source_id'), + data.index_column('source')) + else: + pk = numpy.arange(len(data)) + 1 + data.add_column(astropy.table.Column(pk, 'pk'), 0) + + copy_data(data, database, table, schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql index defc785f..67373d83 100644 --- a/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaia_tmass/gaiaDR2/gaiadr2_tmass_best_neighbour.sql @@ -18,10 +18,9 @@ CREATE TABLE catalogdb.gaiadr2_tmass_best_neighbour( tmass_pts_key integer ); +\COPY catalogdb.gaiadr2_tmass_best_neighbour FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' WITH CSV HEADER; -\copy catalogdb.gaiadr2_tmass_best_neighbour FROM '$CATALOGDB_DIR/xmatch/gaia_tmass/gaiaDR2/tmass_best_neighbour.csv' WITH CSV HEADER; +ALTER TABLE catalogdb.gaiadr2_tmass_best_neighbour ADD PRIMARY KEY (source_id); -alter table catalogdb.gaiadr2_tmass_best_neighbour add primary key(source_id); - -CREATE INDEX CONCURRENTLY gaiadr2_tmass_best_neighbour_tmass_oid_index ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (tmass_oid); -CREATE INDEX CONCURRENTLY gaiadr2_tmass_best_neighbour_tmass_original_ext_source_id_index ON catalogdb.gaiadr2_tmass_best_neighbour using BTREE (original_ext_source_id); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (tmass_oid); +CREATE INDEX CONCURRENTLY ON catalogdb.gaiadr2_tmass_best_neighbour USING BTREE (original_ext_source_id); diff --git a/schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql similarity index 67% rename from schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql rename to schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql index b7c17e6d..bba4905b 100644 --- a/schema/sdss5db/catalogdb/unused/gaiadr2_sdssdr9_best_neighbour.sql +++ b/schema/sdss5db/catalogdb/xmatch/gaiadr2_sdssdr9_best_neighbour.sql @@ -18,8 +18,6 @@ CREATE TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour( ); -\copy catalogdb.gaiadr2_sdssdr9_best_neighbour FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' WITH CSV HEADER; +\COPY catalogdb.gaiadr2_sdssdr9_best_neighbour FROM '/uufs/chpc.utah.edu/common/home/sdss/sdsswork/gaia/dr2/crossmatch/csv_catalogs/sdssdr9_best_neighbour.csv' WITH CSV HEADER; -alter table catalogdb.gaiadr2_sdssdr9_best_neighbour add primary key(source_id); - -CREATE INDEX CONCURRENTLY gaiadr2_sdssdr9_best_neighbour_sdssdr9_oid_index ON catalogdb.gaiadr2_sdssdr9_best_neighbour using BTREE (sdssdr9_oid); \ No newline at end of file +ALTER TABLE catalogdb.gaiadr2_sdssdr9_best_neighbour ADD PRIMARY KEY (source_id); diff --git a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py new file mode 100755 index 00000000..8c716e69 --- /dev/null +++ b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2020-03-24 +# @Filename: yso_clustering.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +import os + +import astropy.table + +from sdssdb.peewee.sdss5db import database +from sdssdb.utils.ingest import copy_data + + +def main(): + + assert database.connected + + file_ = os.environ['CATALOGDB_DIR'] + '/Kounkel_Clustering.fits' + + data = astropy.table.Table.read(file_) + + copy_data(data, database, 'yso_clustering', schema='catalogdb') + + +if __name__ == '__main__': + + main() diff --git a/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql new file mode 100644 index 00000000..af3bc9d3 --- /dev/null +++ b/schema/sdss5db/catalogdb/yso_clustering/yso_clustering.sql @@ -0,0 +1,30 @@ +/* + +AAVSO Cataclysmic Variables + +*/ + +CREATE TABLE catalogdb.yso_clustering ( + source_id BIGINT PRIMARY KEY, + twomass TEXT, + ra DOUBLE PRECISION, + dec DOUBLE PRECISION, + parallax REAL, + id INTEGER, + g DOUBLE PRECISION, + bp DOUBLE PRECISION, + rp DOUBLE PRECISION, + j REAL, + h REAL, + k REAL, + age DOUBLE PRECISION, + eage DOUBLE PRECISION, + av DOUBLE PRECISION, + eav DOUBLE PRECISION, + dist DOUBLE PRECISION, + edist DOUBLE PRECISION +); + +CREATE INDEX CONCURRENTLY ON catalogdb.yso_clustering (q3c_ang2ipix(ra, dec)); +CLUSTER yso_clustering_q3c_ang2ipix_idx ON catalogdb.yso_clustering; +ANALYZE catalogdb.yso_clustering; diff --git a/schema/sdss5db/targetdb/targetdb.sql b/schema/sdss5db/targetdb/targetdb.sql index 0935954c..12ad8373 100644 --- a/schema/sdss5db/targetdb/targetdb.sql +++ b/schema/sdss5db/targetdb/targetdb.sql @@ -229,7 +229,7 @@ CREATE INDEX CONCURRENTLY magnitude_pk_idx ON targetdb.target using BTREE(magnit CREATE INDEX CONCURRENTLY catalogid_idx ON targetdb.target using BTREE(catalogid); -- This doesn't seem to be loaded unless it's run manually inside a PSQL console. -CREATE INDEX ON targetdb.target (q3c_ang2ipix(ra, dec)); +CREATE INDEX CONCURRENTLY ON targetdb.target (q3c_ang2ipix(ra, dec)); CLUSTER target_q3c_ang2ipix_idx on targetdb.target; ANALYZE targetdb.target; @@ -252,5 +252,5 @@ CREATE INDEX CONCURRENTLY field_cadence_pk_idx ON targetdb.field using BTREE(cad CREATE INDEX CONCURRENTLY observatory_pk_idx ON targetdb.field using BTREE(observatory_pk); CREATE INDEX CONCURRENTLY positioner_status_pk_idx ON targetdb.positioner using BTREE(positioner_status_pk); -CREATE INDEX CONCURRENTLY positioner_type_pk_idx ON targetdb.positioner using BTREE(positioner_type_pk); +CREATE INDEX CONCURRENTLY positioner_info_pk_idx ON targetdb.positioner using BTREE(positioner_info_pk); CREATE INDEX CONCURRENTLY positioner_observatory_pk_idx ON targetdb.positioner using BTREE(observatory_pk); diff --git a/setup.cfg b/setup.cfg index b08bd7f8..65acb80c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sdssdb -version = 0.3.3dev +version = 0.3.3b2 author = José Sánchez-Gallego author_email = gallegoj@uw.edu description = SDSS product for database management @@ -38,6 +38,8 @@ install_requires = six>=1.12.0 peewee>=3.9.6 sqlalchemy>=1.3.6 + sdsstools>=0.1.5 + numpy>=1.18.2 [options.packages.find] where = @@ -51,25 +53,27 @@ sdssdb = [options.extras_require] all = - astropy>=3.2.1 progressbar2>=3.46.1 + pydot>=1.4.1 + astropy>=4.0.0 + pandas>=1.0.0 inflect>=4.1.0 dev = pytest>=5.2 pytest-cov>=2.4.0 pytest-sugar>=0.8.0 - pydot>=1.4.1 + sdsstools[dev]>=0.1.5 + ipython>=7.13.0 + ipdb>=0.13.2 pytest-postgresql>=2.2.1 factory_boy>=2.12.0 pytest-factoryboy>=2.0.3 docs = Sphinx>=1.8.0 sphinx_bootstrap_theme>=0.4.12 - releases>=1.6.1 - semantic-version==2.6.0 # Un-pin when releases gets updated. [isort] -line_length = 99 +line_length = 79 lines_after_imports = 2 use_parentheses = true balanced_wrapping = true @@ -80,29 +84,9 @@ ignore = E722 W504 W505 + E116 + E114 max-line-length = 99 -[bumpversion] -current_version = 0.3.3dev -commit = True -tag = False -tag_name = {new_version} -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? -serialize = - {major}.{minor}.{patch}{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = alpha -values = - dev - alpha - -[bumpversion:file:python/sdssdb/__init__.py] - -[bumpversion:file:setup.cfg] - -[bumpversion:file:docs/sphinx/installation.rst] - [coverage:run] branch = true
{table_name}
' + f'{table_name}
({model.__name__})
{}
{}
{}
{ilabel} {column_name}